From fd8deb48c7ad63b47c5d7465ceefcadef0316f93 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Sat, 6 Mar 2021 12:26:28 +0100 Subject: vlib: split vlib_main_t into global and per-thread Type: refactor Change-Id: I8b273bc3bf16aa360f031f1b2692f766e5fc4613 Signed-off-by: Damjan Marion --- src/vlib/buffer.c | 23 ---- src/vlib/cli.c | 33 +++-- src/vlib/cli.h | 42 +++---- src/vlib/global_funcs.h | 14 ++- src/vlib/init.c | 74 +++++++----- src/vlib/init.h | 313 ++++++++++++++++++++++++------------------------ src/vlib/main.c | 56 +++++---- src/vlib/main.h | 95 +++++++++------ src/vlib/node.c | 3 +- src/vlib/node.h | 41 +++---- src/vlib/threads.c | 63 +++++----- src/vlib/threads.h | 50 ++++---- src/vlib/trace_funcs.h | 6 +- src/vlib/unix/main.c | 18 +-- src/vlib/vlib.h | 1 + 15 files changed, 439 insertions(+), 393 deletions(-) (limited to 'src/vlib') diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index f8cfb0357fa..ae88b4e4d44 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -309,29 +309,6 @@ done: return result; } -/* - * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1 - * and vlib_mains[0] = &vlib_global_main from the beginning of time. - * - * The only place which should ever expand vlib_mains is start_workers() - * in threads.c. It knows about the bootstrap vector. - */ -/* *INDENT-OFF* */ -static struct -{ - vec_header_t h; - vlib_main_t *vm; -} __attribute__ ((packed)) __bootstrap_vlib_main_vector - __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) = -{ - .h.len = 1, - .vm = &vlib_global_main, -}; -/* *INDENT-ON* */ - -vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; - - /* When debugging validate that given buffers are either known allocated or known free. */ void diff --git a/src/vlib/cli.c b/src/vlib/cli.c index 3215a3264f5..5cad95ee004 100644 --- a/src/vlib/cli.c +++ b/src/vlib/cli.c @@ -205,10 +205,11 @@ get_sub_command (vlib_cli_main_t * cm, vlib_cli_command_t * parent, u32 si) static uword unformat_vlib_cli_sub_command (unformat_input_t * i, va_list * args) { - vlib_main_t *vm = va_arg (*args, vlib_main_t *); + vlib_main_t __clib_unused *vm = va_arg (*args, vlib_main_t *); + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_cli_command_t *c = va_arg (*args, vlib_cli_command_t *); vlib_cli_command_t **result = va_arg (*args, vlib_cli_command_t **); - vlib_cli_main_t *cm = &vm->cli_main; + vlib_cli_main_t *cm = &vgm->cli_main; uword *match_bitmap, is_unique, index; match_bitmap = vlib_cli_sub_command_match (c, i); @@ -238,8 +239,8 @@ vlib_cli_get_possible_completions (u8 * str) { vlib_cli_command_t *c; vlib_cli_sub_command_t *sc; - vlib_main_t *vm = vlib_get_main (); - vlib_cli_main_t *vcm = &vm->cli_main; + vlib_global_main_t *vgm = vlib_get_global_main (); + vlib_cli_main_t *vcm = &vgm->cli_main; uword *match_bitmap = 0; uword index, is_unique, help_next_level; u8 **result = 0; @@ -388,11 +389,12 @@ vlib_cli_dispatch_sub_commands (vlib_main_t * vm, unformat_input_t * input, uword parent_command_index) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_cli_command_t *parent, *c; clib_error_t *error = 0; unformat_input_t sub_input; u8 *string; - uword is_main_dispatch = cm == &vm->cli_main; + uword is_main_dispatch = cm == &vgm->cli_main; parent = vec_elt_at_index (cm->commands, parent_command_index); if (is_main_dispatch && unformat (input, "help")) @@ -674,6 +676,7 @@ vlib_cli_input (vlib_main_t * vm, unformat_input_t * input, vlib_cli_output_function_t * function, uword function_arg) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_process_t *cp = vlib_get_current_process (vm); clib_error_t *error; vlib_cli_output_function_t *save_function; @@ -688,7 +691,7 @@ vlib_cli_input (vlib_main_t * vm, do { - error = vlib_cli_dispatch_sub_commands (vm, &vm->cli_main, input, + error = vlib_cli_dispatch_sub_commands (vm, &vgm->cli_main, input, /* parent */ 0); } while (!error && !unformat (input, "%U", unformat_eof)); @@ -1086,6 +1089,7 @@ static clib_error_t * restart_cmd_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { + vlib_global_main_t *vgm = vlib_get_global_main (); clib_file_main_t *fm = &file_main; clib_file_t *f; @@ -1102,7 +1106,7 @@ restart_cmd_fn (vlib_main_t * vm, unformat_input_t * input, /* *INDENT-ON* */ /* Exec ourself */ - execve (vm->name, (char **) vm->argv, environ); + execve (vgm->name, (char **) vm->argv, environ); return 0; } @@ -1352,7 +1356,8 @@ vlib_cli_command_is_empty (vlib_cli_command_t * c) clib_error_t * vlib_cli_register (vlib_main_t * vm, vlib_cli_command_t * c) { - vlib_cli_main_t *cm = &vm->cli_main; + vlib_global_main_t *vgm = vlib_get_global_main (); + vlib_cli_main_t *cm = &vgm->cli_main; clib_error_t *error = 0; uword ci, *p; char *normalized_path; @@ -1626,8 +1631,8 @@ sort_cmds_by_path (void *a1, void *a2) { u32 *index1 = a1; u32 *index2 = a2; - vlib_main_t *vm = vlib_get_main (); - vlib_cli_main_t *cm = &vm->cli_main; + vlib_global_main_t *vgm = vlib_get_global_main (); + vlib_cli_main_t *cm = &vgm->cli_main; vlib_cli_command_t *c1, *c2; int i, lmin; @@ -1737,6 +1742,7 @@ static clib_error_t * show_cli_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { + vlib_global_main_t *vgm = vlib_get_global_main (); int show_mp_safe = 0; int show_not_mp_safe = 0; int show_hit = 0; @@ -1760,8 +1766,8 @@ show_cli_command_fn (vlib_main_t * vm, if (clear_hit == 0 && (show_mp_safe + show_not_mp_safe) == 0) show_mp_safe = show_not_mp_safe = 1; - vlib_cli_output (vm, "%U", format_mp_safe, &vm->cli_main, - show_mp_safe, show_not_mp_safe, show_hit, clear_hit); + vlib_cli_output (vm, "%U", format_mp_safe, &vgm->cli_main, show_mp_safe, + show_not_mp_safe, show_hit, clear_hit); if (clear_hit) vlib_cli_output (vm, "hit counters cleared..."); @@ -1821,7 +1827,8 @@ VLIB_CLI_COMMAND (show_cli_command, static) = static clib_error_t * vlib_cli_init (vlib_main_t * vm) { - vlib_cli_main_t *cm = &vm->cli_main; + vlib_global_main_t *vgm = vlib_get_global_main (); + vlib_cli_main_t *cm = &vgm->cli_main; clib_error_t *error = 0; vlib_cli_command_t *cmd; diff --git a/src/vlib/cli.h b/src/vlib/cli.h index c31236f8ab9..83b4eab1851 100644 --- a/src/vlib/cli.h +++ b/src/vlib/cli.h @@ -160,27 +160,27 @@ typedef struct vlib_cli_main_t } vlib_cli_main_t; #ifndef CLIB_MARCH_VARIANT -#define VLIB_CLI_COMMAND(x,...) \ - __VA_ARGS__ vlib_cli_command_t x; \ -static void __vlib_cli_command_registration_##x (void) \ - __attribute__((__constructor__)) ; \ -static void __vlib_cli_command_registration_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - vlib_cli_main_t *cm = &vm->cli_main; \ - x.next_cli_command = cm->cli_command_registrations; \ - cm->cli_command_registrations = &x; \ -} \ -static void __vlib_cli_command_unregistration_##x (void) \ - __attribute__((__destructor__)) ; \ -static void __vlib_cli_command_unregistration_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - vlib_cli_main_t *cm = &vm->cli_main; \ - VLIB_REMOVE_FROM_LINKED_LIST (cm->cli_command_registrations, &x, \ - next_cli_command); \ -} \ -__VA_ARGS__ vlib_cli_command_t x +#define VLIB_CLI_COMMAND(x, ...) \ + __VA_ARGS__ vlib_cli_command_t x; \ + static void __vlib_cli_command_registration_##x (void) \ + __attribute__ ((__constructor__)); \ + static void __vlib_cli_command_registration_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + vlib_cli_main_t *cm = &vgm->cli_main; \ + x.next_cli_command = cm->cli_command_registrations; \ + cm->cli_command_registrations = &x; \ + } \ + static void __vlib_cli_command_unregistration_##x (void) \ + __attribute__ ((__destructor__)); \ + static void __vlib_cli_command_unregistration_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + vlib_cli_main_t *cm = &vgm->cli_main; \ + VLIB_REMOVE_FROM_LINKED_LIST (cm->cli_command_registrations, &x, \ + next_cli_command); \ + } \ + __VA_ARGS__ vlib_cli_command_t x #else /* create unused pointer to silence compiler warnings and get whole function optimized out */ diff --git a/src/vlib/global_funcs.h b/src/vlib/global_funcs.h index 63f8115a181..3c0fdb78364 100644 --- a/src/vlib/global_funcs.h +++ b/src/vlib/global_funcs.h @@ -22,14 +22,14 @@ always_inline u32 vlib_get_n_threads () { - return vec_len (vlib_mains); + return vec_len (vlib_global_main.vlib_mains); } always_inline vlib_main_t * vlib_get_main_by_index (u32 thread_index) { vlib_main_t *vm; - vm = vlib_mains[thread_index]; + vm = vlib_global_main.vlib_mains[thread_index]; ASSERT (vm); return vm; } @@ -46,6 +46,12 @@ vlib_get_first_main (void) return vlib_get_main_by_index (0); } +always_inline vlib_global_main_t * +vlib_get_global_main (void) +{ + return &vlib_global_main; +} + always_inline vlib_thread_main_t * vlib_get_thread_main () { @@ -55,8 +61,8 @@ vlib_get_thread_main () always_inline elog_main_t * vlib_get_elog_main () { - vlib_main_t *vm = vlib_get_first_main (); - return &vm->elog_main; + vlib_global_main_t *vgm = vlib_get_global_main (); + return &vgm->elog_main; } #endif /* included_vlib_global_funcs_h_ */ diff --git a/src/vlib/init.c b/src/vlib/init.c index 9b43ca3cc26..4a56ab20b4a 100644 --- a/src/vlib/init.c +++ b/src/vlib/init.c @@ -330,10 +330,11 @@ again: */ static inline clib_error_t * -call_init_exit_functions_internal (vlib_main_t * vm, - _vlib_init_function_list_elt_t ** headp, - int call_once, int do_sort) +call_init_exit_functions_internal (vlib_main_t *vm, + _vlib_init_function_list_elt_t **headp, + int call_once, int do_sort, int is_global) { + vlib_global_main_t *vgm = vlib_get_global_main (); clib_error_t *error = 0; _vlib_init_function_list_elt_t *i; @@ -343,10 +344,22 @@ call_init_exit_functions_internal (vlib_main_t * vm, i = *headp; while (i) { - if (call_once && !hash_get (vm->init_functions_called, i->f)) + uword *h; + + if (is_global) + h = hash_get (vgm->init_functions_called, i->f); + else + h = hash_get (vm->worker_init_functions_called, i->f); + + if (call_once && !h) { if (call_once) - hash_set1 (vm->init_functions_called, i->f); + { + if (is_global) + hash_set1 (vgm->init_functions_called, i->f); + else + hash_set1 (vm->worker_init_functions_called, i->f); + } error = i->f (vm); if (error) return error; @@ -357,54 +370,60 @@ call_init_exit_functions_internal (vlib_main_t * vm, } clib_error_t * -vlib_call_init_exit_functions (vlib_main_t * vm, - _vlib_init_function_list_elt_t ** headp, - int call_once) +vlib_call_init_exit_functions (vlib_main_t *vm, + _vlib_init_function_list_elt_t **headp, + int call_once, int is_global) { return call_init_exit_functions_internal (vm, headp, call_once, - 1 /* do_sort */ ); + 1 /* do_sort */, is_global); } clib_error_t * -vlib_call_init_exit_functions_no_sort (vlib_main_t * vm, - _vlib_init_function_list_elt_t ** - headp, int call_once) +vlib_call_init_exit_functions_no_sort (vlib_main_t *vm, + _vlib_init_function_list_elt_t **headp, + int call_once, int is_global) { return call_init_exit_functions_internal (vm, headp, call_once, - 0 /* do_sort */ ); + 0 /* do_sort */, is_global); } clib_error_t * vlib_call_all_init_functions (vlib_main_t * vm) { + vlib_global_main_t *vgm = vlib_get_global_main (); /* Call placeholder functions to make sure purely static modules are linked in. */ #define _(f) vlib_##f##_reference (); foreach_vlib_module_reference; #undef _ - return vlib_call_init_exit_functions - (vm, &vm->init_function_registrations, 1 /* call_once */ ); + return vlib_call_init_exit_functions (vm, &vgm->init_function_registrations, + 1 /* call_once */, 1 /* is_global */); } clib_error_t * vlib_call_all_main_loop_enter_functions (vlib_main_t * vm) { - return vlib_call_init_exit_functions - (vm, &vm->main_loop_enter_function_registrations, 1 /* call_once */ ); + vlib_global_main_t *vgm = vlib_get_global_main (); + return vlib_call_init_exit_functions ( + vm, &vgm->main_loop_enter_function_registrations, 1 /* call_once */, + 1 /* is_global */); } clib_error_t * vlib_call_all_main_loop_exit_functions (vlib_main_t * vm) { - return vlib_call_init_exit_functions - (vm, &vm->main_loop_exit_function_registrations, 1 /* call_once */ ); + vlib_global_main_t *vgm = vlib_get_global_main (); + return vlib_call_init_exit_functions ( + vm, &vgm->main_loop_exit_function_registrations, 1 /* call_once */, + 1 /* is_global */); } clib_error_t * vlib_call_all_config_functions (vlib_main_t * vm, unformat_input_t * input, int is_early) { + vlib_global_main_t *vgm = vlib_get_global_main (); clib_error_t *error = 0; vlib_config_function_runtime_t *c, **all; uword *hash = 0, *p; @@ -413,7 +432,7 @@ vlib_call_all_config_functions (vlib_main_t * vm, hash = hash_create_string (0, sizeof (uword)); all = 0; - c = vm->config_function_registrations; + c = vgm->config_function_registrations; while (c) { @@ -450,9 +469,9 @@ vlib_call_all_config_functions (vlib_main_t * vm, continue; /* Already called? */ - if (hash_get (vm->init_functions_called, c->function)) + if (hash_get (vgm->init_functions_called, c->function)) continue; - hash_set1 (vm->init_functions_called, c->function); + hash_set1 (vgm->init_functions_called, c->function); error = c->function (vm, &c->input); if (error) @@ -473,11 +492,11 @@ done: void vlib_init_dump (void) { - vlib_main_t *vm = vlib_get_main (); + vlib_global_main_t *vgm = vlib_get_global_main (); int i = 0; _vlib_init_function_list_elt_t *head, *this; - head = vm->init_function_registrations; + head = vgm->init_function_registrations; this = head; while (this) @@ -492,6 +511,7 @@ show_init_function_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { + vlib_global_main_t *vgm = vlib_get_global_main (); int which = 1; int verbose = 0; int i, n_init_fns; @@ -524,13 +544,13 @@ show_init_function_command_fn (vlib_main_t * vm, switch (which) { case 1: - head = vm->init_function_registrations; + head = vgm->init_function_registrations; break; case 2: - head = vm->main_loop_enter_function_registrations; + head = vgm->main_loop_enter_function_registrations; break; case 3: - head = vm->main_loop_exit_function_registrations; + head = vgm->main_loop_exit_function_registrations; break; default: return clib_error_return (0, "BUG"); diff --git a/src/vlib/init.h b/src/vlib/init.h index c5447308513..e6235652ad1 100644 --- a/src/vlib/init.h +++ b/src/vlib/init.h @@ -120,49 +120,48 @@ typedef struct vlib_config_function_runtime_t be called from other modules to resolve init function depend. */ #ifndef CLIB_MARCH_VARIANT -#define VLIB_DECLARE_INIT_FUNCTION(x, tag) \ -vlib_init_function_t * _VLIB_INIT_FUNCTION_SYMBOL (x, tag) = x; \ -static void __vlib_add_##tag##_function_##x (void) \ - __attribute__((__constructor__)) ; \ -static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x; \ -static void __vlib_add_##tag##_function_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - _vlib_init_function_##tag##_##x.next_init_function \ - = vm->tag##_function_registrations; \ - vm->tag##_function_registrations = &_vlib_init_function_##tag##_##x; \ - _vlib_init_function_##tag##_##x.f = &x; \ - _vlib_init_function_##tag##_##x.name = #x; \ -} \ -static void __vlib_rm_##tag##_function_##x (void) \ - __attribute__((__destructor__)) ; \ -static void __vlib_rm_##tag##_function_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - _vlib_init_function_list_elt_t *this, *prev; \ - this = vm->tag##_function_registrations; \ - if (this == 0) \ - return; \ - if (this->f == &x) \ - { \ - vm->tag##_function_registrations = this->next_init_function; \ - return; \ - } \ - prev = this; \ - this = this->next_init_function; \ - while (this) \ - { \ - if (this->f == &x) \ - { \ - prev->next_init_function = \ - this->next_init_function; \ - return; \ - } \ - prev = this; \ - this = this->next_init_function; \ - } \ -} \ -static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x +#define VLIB_DECLARE_INIT_FUNCTION(x, tag) \ + vlib_init_function_t *_VLIB_INIT_FUNCTION_SYMBOL (x, tag) = x; \ + static void __vlib_add_##tag##_function_##x (void) \ + __attribute__ ((__constructor__)); \ + static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x; \ + static void __vlib_add_##tag##_function_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + _vlib_init_function_##tag##_##x.next_init_function = \ + vgm->tag##_function_registrations; \ + vgm->tag##_function_registrations = &_vlib_init_function_##tag##_##x; \ + _vlib_init_function_##tag##_##x.f = &x; \ + _vlib_init_function_##tag##_##x.name = #x; \ + } \ + static void __vlib_rm_##tag##_function_##x (void) \ + __attribute__ ((__destructor__)); \ + static void __vlib_rm_##tag##_function_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + _vlib_init_function_list_elt_t *this, *prev; \ + this = vgm->tag##_function_registrations; \ + if (this == 0) \ + return; \ + if (this->f == &x) \ + { \ + vgm->tag##_function_registrations = this->next_init_function; \ + return; \ + } \ + prev = this; \ + this = this->next_init_function; \ + while (this) \ + { \ + if (this->f == &x) \ + { \ + prev->next_init_function = this->next_init_function; \ + return; \ + } \ + prev = this; \ + this = this->next_init_function; \ + } \ + } \ + static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x #else /* create unused pointer to silence compiler warnings and get whole function optimized out */ @@ -179,35 +178,30 @@ static __clib_unused void * __clib_unused_##tag##_##x = x VLIB_DECLARE_INIT_FUNCTION(x,main_loop_exit) #ifndef CLIB_MARCH_VARIANT -#define VLIB_CONFIG_FUNCTION(x,n,...) \ - __VA_ARGS__ vlib_config_function_runtime_t \ - VLIB_CONFIG_FUNCTION_SYMBOL(x); \ -static void __vlib_add_config_function_##x (void) \ - __attribute__((__constructor__)) ; \ -static void __vlib_add_config_function_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - VLIB_CONFIG_FUNCTION_SYMBOL(x).next_registration \ - = vm->config_function_registrations; \ - vm->config_function_registrations \ - = &VLIB_CONFIG_FUNCTION_SYMBOL(x); \ -} \ -static void __vlib_rm_config_function_##x (void) \ - __attribute__((__destructor__)) ; \ -static void __vlib_rm_config_function_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - vlib_config_function_runtime_t *p = \ - & VLIB_CONFIG_FUNCTION_SYMBOL (x); \ - VLIB_REMOVE_FROM_LINKED_LIST \ - (vm->config_function_registrations, p, next_registration);\ -} \ - vlib_config_function_runtime_t \ - VLIB_CONFIG_FUNCTION_SYMBOL (x) \ - = { \ - .name = n, \ - .function = x, \ - .is_early = 0, \ +#define VLIB_CONFIG_FUNCTION(x, n, ...) \ + __VA_ARGS__ vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + static void __vlib_add_config_function_##x (void) \ + __attribute__ ((__constructor__)); \ + static void __vlib_add_config_function_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + VLIB_CONFIG_FUNCTION_SYMBOL (x).next_registration = \ + vgm->config_function_registrations; \ + vgm->config_function_registrations = &VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + } \ + static void __vlib_rm_config_function_##x (void) \ + __attribute__ ((__destructor__)); \ + static void __vlib_rm_config_function_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + vlib_config_function_runtime_t *p = &VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + VLIB_REMOVE_FROM_LINKED_LIST (vgm->config_function_registrations, p, \ + next_registration); \ + } \ + vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x) = { \ + .name = n, \ + .function = x, \ + .is_early = 0, \ } #else /* create unused pointer to silence compiler warnings and get whole @@ -223,35 +217,30 @@ static void __vlib_rm_config_function_##x (void) \ #endif #ifndef CLIB_MARCH_VARIANT -#define VLIB_EARLY_CONFIG_FUNCTION(x,n,...) \ - __VA_ARGS__ vlib_config_function_runtime_t \ - VLIB_CONFIG_FUNCTION_SYMBOL(x); \ -static void __vlib_add_config_function_##x (void) \ - __attribute__((__constructor__)) ; \ -static void __vlib_add_config_function_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - VLIB_CONFIG_FUNCTION_SYMBOL(x).next_registration \ - = vm->config_function_registrations; \ - vm->config_function_registrations \ - = &VLIB_CONFIG_FUNCTION_SYMBOL(x); \ -} \ -static void __vlib_rm_config_function_##x (void) \ - __attribute__((__destructor__)) ; \ -static void __vlib_rm_config_function_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - vlib_config_function_runtime_t *p = \ - & VLIB_CONFIG_FUNCTION_SYMBOL (x); \ - VLIB_REMOVE_FROM_LINKED_LIST \ - (vm->config_function_registrations, p, next_registration);\ -} \ - vlib_config_function_runtime_t \ - VLIB_CONFIG_FUNCTION_SYMBOL (x) \ - = { \ - .name = n, \ - .function = x, \ - .is_early = 1, \ +#define VLIB_EARLY_CONFIG_FUNCTION(x, n, ...) \ + __VA_ARGS__ vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + static void __vlib_add_config_function_##x (void) \ + __attribute__ ((__constructor__)); \ + static void __vlib_add_config_function_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + VLIB_CONFIG_FUNCTION_SYMBOL (x).next_registration = \ + vgm->config_function_registrations; \ + vgm->config_function_registrations = &VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + } \ + static void __vlib_rm_config_function_##x (void) \ + __attribute__ ((__destructor__)); \ + static void __vlib_rm_config_function_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + vlib_config_function_runtime_t *p = &VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + VLIB_REMOVE_FROM_LINKED_LIST (vgm->config_function_registrations, p, \ + next_registration); \ + } \ + vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x) = { \ + .name = n, \ + .function = x, \ + .is_early = 1, \ } #else /* create unused pointer to silence compiler warnings and get whole @@ -267,67 +256,71 @@ static void __vlib_rm_config_function_##x (void) \ #endif /* Call given init function: used for init function dependencies. */ -#define vlib_call_init_function(vm, x) \ - ({ \ - extern vlib_init_function_t * VLIB_INIT_FUNCTION_SYMBOL (x); \ - vlib_init_function_t * _f = VLIB_INIT_FUNCTION_SYMBOL (x); \ - clib_error_t * _error = 0; \ - if (! hash_get (vm->init_functions_called, _f)) \ - { \ - hash_set1 (vm->init_functions_called, _f); \ - _error = _f (vm); \ - } \ - _error; \ +#define vlib_call_init_function(vm, x) \ + ({ \ + vlib_global_main_t *vgm = &vlib_global_main; \ + extern vlib_init_function_t *VLIB_INIT_FUNCTION_SYMBOL (x); \ + vlib_init_function_t *_f = VLIB_INIT_FUNCTION_SYMBOL (x); \ + clib_error_t *_error = 0; \ + if (!hash_get (vgm->init_functions_called, _f)) \ + { \ + hash_set1 (vgm->init_functions_called, _f); \ + _error = _f (vm); \ + } \ + _error; \ }) /* Don't call given init function: used to suppress parts of the netstack */ -#define vlib_mark_init_function_complete(vm, x) \ - ({ \ - extern vlib_init_function_t * VLIB_INIT_FUNCTION_SYMBOL (x); \ - vlib_init_function_t * _f = VLIB_INIT_FUNCTION_SYMBOL (x); \ - hash_set1 (vm->init_functions_called, _f); \ +#define vlib_mark_init_function_complete(vm, x) \ + ({ \ + vlib_global_main_t *vgm = &vlib_global_main; \ + extern vlib_init_function_t *VLIB_INIT_FUNCTION_SYMBOL (x); \ + vlib_init_function_t *_f = VLIB_INIT_FUNCTION_SYMBOL (x); \ + hash_set1 (vgm->init_functions_called, _f); \ }) -#define vlib_call_post_graph_init_function(vm, x) \ - ({ \ - extern vlib_init_function_t * VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x); \ - vlib_init_function_t * _f = VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x); \ - clib_error_t * _error = 0; \ - if (! hash_get (vm->init_functions_called, _f)) \ - { \ - hash_set1 (vm->init_functions_called, _f); \ - _error = _f (vm); \ - } \ - _error; \ +#define vlib_call_post_graph_init_function(vm, x) \ + ({ \ + vlib_global_main_t *vgm = &vlib_global_main; \ + extern vlib_init_function_t *VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x); \ + vlib_init_function_t *_f = VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x); \ + clib_error_t *_error = 0; \ + if (!hash_get (vgm->init_functions_called, _f)) \ + { \ + hash_set1 (vgm->init_functions_called, _f); \ + _error = _f (vm); \ + } \ + _error; \ }) -#define vlib_call_config_function(vm, x) \ - ({ \ - vlib_config_function_runtime_t * _r; \ - clib_error_t * _error = 0; \ - extern vlib_config_function_runtime_t \ - VLIB_CONFIG_FUNCTION_SYMBOL (x); \ - \ - _r = &VLIB_CONFIG_FUNCTION_SYMBOL (x); \ - if (! hash_get (vm->init_functions_called, _r->function)) \ - { \ - hash_set1 (vm->init_functions_called, _r->function); \ - _error = _r->function (vm, &_r->input); \ - } \ - _error; \ +#define vlib_call_config_function(vm, x) \ + ({ \ + vlib_global_main_t *vgm = &vlib_global_main; \ + vlib_config_function_runtime_t *_r; \ + clib_error_t *_error = 0; \ + extern vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + \ + _r = &VLIB_CONFIG_FUNCTION_SYMBOL (x); \ + if (!hash_get (vgm->init_functions_called, _r->function)) \ + { \ + hash_set1 (vgm->init_functions_called, _r->function); \ + _error = _r->function (vm, &_r->input); \ + } \ + _error; \ }) -#define vlib_call_main_loop_enter_function(vm, x) \ - ({ \ - extern vlib_init_function_t * VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x); \ - vlib_init_function_t * _f = VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x); \ - clib_error_t * _error = 0; \ - if (! hash_get (vm->init_functions_called, _f)) \ - { \ - hash_set1 (vm->init_functions_called, _f); \ - _error = _f (vm); \ - } \ - _error; \ +#define vlib_call_main_loop_enter_function(vm, x) \ + ({ \ + vlib_global_main_t *vgm = &vlib_global_main; \ + extern vlib_init_function_t *VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x); \ + vlib_init_function_t *_f = VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x); \ + clib_error_t *_error = 0; \ + if (!hash_get (vgm->init_functions_called, _f)) \ + { \ + hash_set1 (vgm->init_functions_called, _f); \ + _error = _f (vm); \ + } \ + _error; \ }) /* External functions. */ @@ -338,12 +331,14 @@ clib_error_t *vlib_call_all_config_functions (struct vlib_main_t *vm, clib_error_t *vlib_call_all_main_loop_enter_functions (struct vlib_main_t *vm); clib_error_t *vlib_call_all_main_loop_exit_functions (struct vlib_main_t *vm); -clib_error_t *vlib_call_init_exit_functions (struct vlib_main_t *vm, - _vlib_init_function_list_elt_t ** - headp, int call_once); -clib_error_t *vlib_call_init_exit_functions_no_sort (struct vlib_main_t *vm, - _vlib_init_function_list_elt_t - ** headp, int call_once); +clib_error_t * +vlib_call_init_exit_functions (struct vlib_main_t *vm, + _vlib_init_function_list_elt_t **headp, + int call_once, int is_global); +clib_error_t * +vlib_call_init_exit_functions_no_sort (struct vlib_main_t *vm, + _vlib_init_function_list_elt_t **headp, + int call_once, int is_global); clib_error_t *vlib_sort_init_exit_functions (_vlib_init_function_list_elt_t **); #define foreach_vlib_module_reference \ diff --git a/src/vlib/main.c b/src/vlib/main.c index 62da7b9ff0a..bee63970f3d 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -614,7 +614,7 @@ vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n) if (n->type == VLIB_NODE_TYPE_PROCESS) { /* Nothing to do for PROCESS nodes except in main thread */ - if (vm != &vlib_global_main) + if (vm != vlib_get_first_main ()) return; vlib_process_t *p = vlib_get_process_from_node (vm, n); @@ -747,10 +747,10 @@ elog_save_buffer (vlib_main_t * vm, void vlib_post_mortem_dump (void) { - vlib_main_t *vm = &vlib_global_main; + vlib_global_main_t *vgm = vlib_get_global_main (); - for (int i = 0; i < vec_len (vm->post_mortem_callbacks); i++) - (vm->post_mortem_callbacks[i]) (); + for (int i = 0; i < vec_len (vgm->post_mortem_callbacks); i++) + (vgm->post_mortem_callbacks[i]) (); } /* *INDENT-OFF* */ @@ -901,7 +901,7 @@ vlib_elog_main_loop_event (vlib_main_t * vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return) { - vlib_main_t *evm = &vlib_global_main; + vlib_main_t *evm = vlib_get_first_main (); elog_main_t *em = vlib_get_elog_main (); int enabled = evm->elog_trace_graph_dispatch | evm->elog_trace_graph_circuit; @@ -1091,7 +1091,8 @@ dispatch_node (vlib_main_t * vm, nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1; nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1; - if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch)) + if (PREDICT_FALSE ( + vlib_get_first_main ()->elog_trace_graph_dispatch)) { vlib_worker_thread_t *w = vlib_worker_threads + vm->thread_index; @@ -1126,7 +1127,8 @@ dispatch_node (vlib_main_t * vm, + vm->thread_index; node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE; - if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch)) + if (PREDICT_FALSE ( + vlib_get_first_main ()->elog_trace_graph_dispatch)) { ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track); @@ -1794,26 +1796,26 @@ vlib_worker_loop (vlib_main_t * vm) vlib_main_or_worker_loop (vm, /* is_main */ 0); } -vlib_main_t vlib_global_main; +vlib_global_main_t vlib_global_main; void vlib_add_del_post_mortem_callback (void *cb, int is_add) { - vlib_main_t *vm = &vlib_global_main; + vlib_global_main_t *vgm = vlib_get_global_main (); int i; if (is_add == 0) { - for (i = vec_len (vm->post_mortem_callbacks) - 1; i >= 0; i--) - if (vm->post_mortem_callbacks[i] == cb) - vec_del1 (vm->post_mortem_callbacks, i); + for (i = vec_len (vgm->post_mortem_callbacks) - 1; i >= 0; i--) + if (vgm->post_mortem_callbacks[i] == cb) + vec_del1 (vgm->post_mortem_callbacks, i); return; } - for (i = 0; i < vec_len (vm->post_mortem_callbacks); i++) - if (vm->post_mortem_callbacks[i] == cb) + for (i = 0; i < vec_len (vgm->post_mortem_callbacks); i++) + if (vgm->post_mortem_callbacks[i] == cb) return; - vec_add1 (vm->post_mortem_callbacks, cb); + vec_add1 (vgm->post_mortem_callbacks, cb); } static void @@ -1836,6 +1838,7 @@ elog_post_mortem_dump (void) static clib_error_t * vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) { + vlib_global_main_t *vgm = vlib_get_global_main (); int turn_on_mem_trace = 0; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) @@ -1844,9 +1847,9 @@ vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) turn_on_mem_trace = 1; else if (unformat (input, "elog-events %d", - &vm->configured_elog_ring_size)) - vm->configured_elog_ring_size = - 1 << max_log2 (vm->configured_elog_ring_size); + &vgm->configured_elog_ring_size)) + vgm->configured_elog_ring_size = + 1 << max_log2 (vgm->configured_elog_ring_size); else if (unformat (input, "elog-post-mortem-dump")) vlib_add_del_post_mortem_callback (elog_post_mortem_dump, /* is_add */ 1); @@ -1923,21 +1926,22 @@ vl_api_get_elog_trace_api_messages (void) int vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) { + vlib_global_main_t *vgm = vlib_get_global_main (); clib_error_t *volatile error; vlib_node_main_t *nm = &vm->node_main; vm->queue_signal_callback = placeholder_queue_signal_callback; /* Reconfigure event log which is enabled very early */ - if (vm->configured_elog_ring_size && - vm->configured_elog_ring_size != vm->elog_main.event_ring_size) - elog_resize (&vm->elog_main, vm->configured_elog_ring_size); + if (vgm->configured_elog_ring_size && + vgm->configured_elog_ring_size != vgm->elog_main.event_ring_size) + elog_resize (&vgm->elog_main, vgm->configured_elog_ring_size); vl_api_set_elog_main (vlib_get_elog_main ()); (void) vl_api_set_elog_trace_api_messages (1); /* Default name. */ - if (!vm->name) - vm->name = "VLIB"; + if (!vgm->name) + vgm->name = "VLIB"; if ((error = vlib_physmem_init (vm))) { @@ -2005,8 +2009,8 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) } /* See unix/main.c; most likely already set up */ - if (vm->init_functions_called == 0) - vm->init_functions_called = hash_create (0, /* value bytes */ 0); + if (vgm->init_functions_called == 0) + vgm->init_functions_called = hash_create (0, /* value bytes */ 0); if ((error = vlib_call_all_init_functions (vm))) goto done; @@ -2048,7 +2052,7 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) vm->damping_constant = exp (-1.0 / 20.0); /* Sort per-thread init functions before we start threads */ - vlib_sort_init_exit_functions (&vm->worker_init_function_registrations); + vlib_sort_init_exit_functions (&vgm->worker_init_function_registrations); /* Call all main loop enter functions. */ { diff --git a/src/vlib/main.h b/src/vlib/main.h index 860192c1aa3..c655560d08c 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -152,9 +152,6 @@ typedef struct vlib_main_t /* Error marker to use when exiting main loop. */ clib_error_t *main_loop_error; - /* Name for e.g. syslog. */ - char *name; - /* Start of the heap. */ void *heap_base; @@ -173,15 +170,9 @@ typedef struct vlib_main_t /* Node graph main structure. */ vlib_node_main_t node_main; - /* Command line interface. */ - vlib_cli_main_t cli_main; - /* Packet trace buffer. */ vlib_trace_main_t trace_main; - /* Packet trace capture filter */ - vlib_trace_filter_t trace_filter; - /* Error handling. */ vlib_error_main_t error_main; @@ -194,11 +185,10 @@ typedef struct vlib_main_t /* Stream index to use for distribution when MC is enabled. */ u32 mc_stream_index; - vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join; + /* Hash table to record which init functions have been called. */ + uword *worker_init_functions_called; - /* Event logger. */ - elog_main_t elog_main; - u32 configured_elog_ring_size; + vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join; /* Event logger trace flags */ int elog_trace_api_messages; @@ -219,22 +209,11 @@ typedef struct vlib_main_t /* Buffer of random data for various uses. */ clib_random_buffer_t random_buffer; - /* Hash table to record which init functions have been called. */ - uword *init_functions_called; - /* thread, cpu and numa_node indices */ u32 thread_index; u32 cpu_id; u32 numa_node; - /* List of init functions to call, setup by constructors */ - _vlib_init_function_list_elt_t *init_function_registrations; - _vlib_init_function_list_elt_t *worker_init_function_registrations; - _vlib_init_function_list_elt_t *main_loop_enter_function_registrations; - _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; - _vlib_init_function_list_elt_t *api_init_function_registrations; - vlib_config_function_runtime_t *config_function_registrations; - /* control-plane API queue signal pending, length indication */ volatile u32 queue_signal_pending; volatile u32 api_queue_nonempty; @@ -251,15 +230,6 @@ typedef struct vlib_main_t /* debugging */ volatile int parked_at_barrier; - /* post-mortem callbacks */ - void (**post_mortem_callbacks) (void); - - /* - * Need to call vlib_worker_thread_node_runtime_update before - * releasing worker thread barrier. Only valid in vlib_global_main. - */ - int need_vlib_worker_thread_node_runtime_update; - /* Dispatch loop time accounting */ u64 loops_this_reporting_interval; f64 loop_interval_end; @@ -301,8 +271,53 @@ typedef struct vlib_main_t #endif } vlib_main_t; +typedef struct vlib_global_main_t +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + + /* Per-thread Mains */ + vlib_main_t **vlib_mains; + + /* Name for e.g. syslog. */ + char *name; + + /* post-mortem callbacks */ + void (**post_mortem_callbacks) (void); + + /* + * Need to call vlib_worker_thread_node_runtime_update before + * releasing worker thread barrier. + */ + int need_vlib_worker_thread_node_runtime_update; + + /* Command line interface. */ + vlib_cli_main_t cli_main; + + /* Node registrations added by constructors */ + vlib_node_registration_t *node_registrations; + + /* Event logger. */ + elog_main_t elog_main; + u32 configured_elog_ring_size; + + /* Packet trace capture filter */ + vlib_trace_filter_t trace_filter; + + /* List of init functions to call, setup by constructors */ + _vlib_init_function_list_elt_t *init_function_registrations; + _vlib_init_function_list_elt_t *main_loop_enter_function_registrations; + _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; + _vlib_init_function_list_elt_t *worker_init_function_registrations; + _vlib_init_function_list_elt_t *api_init_function_registrations; + vlib_config_function_runtime_t *config_function_registrations; + + /* Hash table to record which init functions have been called. */ + uword *init_functions_called; + +} vlib_global_main_t; + /* Global main structure. */ -extern vlib_main_t vlib_global_main; +extern vlib_global_main_t vlib_global_main; void vlib_worker_loop (vlib_main_t * vm); @@ -441,6 +456,18 @@ always_inline void vlib_set_queue_signal_callback vm->queue_signal_callback = fp; } +always_inline void +vlib_main_init () +{ + vlib_global_main_t *vgm = &vlib_global_main; + vlib_main_t *vm; + + vgm->init_functions_called = hash_create (0, /* value bytes */ 0); + + vm = clib_mem_alloc_aligned (sizeof (*vm), CLIB_CACHE_LINE_BYTES); + vec_add1 (vgm->vlib_mains, vm); +} + /* Main routine. */ int vlib_main (vlib_main_t * vm, unformat_input_t * input); diff --git a/src/vlib/node.c b/src/vlib/node.c index 9de4c690c3f..f4329e7c503 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -572,6 +572,7 @@ vlib_register_all_node_march_variants (vlib_main_t *vm) void vlib_register_all_static_nodes (vlib_main_t * vm) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_node_registration_t *r; static char *null_node_error_strings[] = { @@ -590,7 +591,7 @@ vlib_register_all_static_nodes (vlib_main_t * vm) real node */ register_node (vm, &null_node_reg); - r = vm->node_main.node_registrations; + r = vgm->node_registrations; while (r) { register_node (vm, r); diff --git a/src/vlib/node.h b/src/vlib/node.h index aae5103908d..21a2022858f 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -166,25 +166,25 @@ typedef struct _vlib_node_registration } vlib_node_registration_t; #ifndef CLIB_MARCH_VARIANT -#define VLIB_REGISTER_NODE(x,...) \ - __VA_ARGS__ vlib_node_registration_t x; \ -static void __vlib_add_node_registration_##x (void) \ - __attribute__((__constructor__)) ; \ -static void __vlib_add_node_registration_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - x.next_registration = vm->node_main.node_registrations; \ - vm->node_main.node_registrations = &x; \ -} \ -static void __vlib_rm_node_registration_##x (void) \ - __attribute__((__destructor__)) ; \ -static void __vlib_rm_node_registration_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - VLIB_REMOVE_FROM_LINKED_LIST (vm->node_main.node_registrations, \ - &x, next_registration); \ -} \ -__VA_ARGS__ vlib_node_registration_t x +#define VLIB_REGISTER_NODE(x, ...) \ + __VA_ARGS__ vlib_node_registration_t x; \ + static void __vlib_add_node_registration_##x (void) \ + __attribute__ ((__constructor__)); \ + static void __vlib_add_node_registration_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + x.next_registration = vgm->node_registrations; \ + vgm->node_registrations = &x; \ + } \ + static void __vlib_rm_node_registration_##x (void) \ + __attribute__ ((__destructor__)); \ + static void __vlib_rm_node_registration_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + VLIB_REMOVE_FROM_LINKED_LIST (vgm->node_registrations, &x, \ + next_registration); \ + } \ + __VA_ARGS__ vlib_node_registration_t x #else #define VLIB_REGISTER_NODE(x,...) \ STATIC_ASSERT (sizeof(# __VA_ARGS__) != 7,"node " #x " must not be declared as static"); \ @@ -730,9 +730,6 @@ typedef struct /* Time of last node runtime stats clear. */ f64 time_last_runtime_stats_clear; - /* Node registrations added by constructors */ - vlib_node_registration_t *node_registrations; - /* Node index from error code */ u32 *node_by_error; diff --git a/src/vlib/threads.c b/src/vlib/threads.c index cf82e1802d6..278838798a9 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -670,6 +670,7 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) static clib_error_t * start_workers (vlib_main_t * vm) { + vlib_global_main_t *vgm = vlib_get_global_main (); int i, j; vlib_worker_thread_t *w; vlib_main_t *vm_clone; @@ -694,19 +695,17 @@ start_workers (vlib_main_t * vm) vlib_set_thread_name ((char *) w->name); } - vm->elog_main.lock = + vgm->elog_main.lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); - vm->elog_main.lock[0] = 0; + vgm->elog_main.lock[0] = 0; clib_callback_data_init (&vm->vlib_node_runtime_perf_callbacks, &vm->worker_thread_main_loop_callback_lock); - /* Replace hand-crafted length-1 vector with a real vector */ - vlib_mains = 0; - - vec_validate_aligned (vlib_mains, n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); - _vec_len (vlib_mains) = 0; - vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + _vec_len (vgm->vlib_mains) = 0; + vec_add1_aligned (vgm->vlib_mains, vm, CLIB_CACHE_LINE_BYTES); if (n_vlib_mains > 1) { @@ -727,7 +726,7 @@ start_workers (vlib_main_t * vm) /* Without update or refork */ *vlib_worker_threads->node_reforks_required = 0; - vm->need_vlib_worker_thread_node_runtime_update = 0; + vgm->need_vlib_worker_thread_node_runtime_update = 0; /* init timing */ vm->barrier_epoch = 0; @@ -780,14 +779,13 @@ start_workers (vlib_main_t * vm) vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone), CLIB_CACHE_LINE_BYTES); - clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone)); + clib_memcpy (vm_clone, vlib_get_first_main (), + sizeof (*vm_clone)); vm_clone->thread_index = worker_thread_index; vm_clone->heap_base = w->thread_mheap; vm_clone->heap_aligned_base = (void *) (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1)); - vm_clone->init_functions_called = - hash_create (0, /* value bytes */ 0); vm_clone->pending_rpc_requests = 0; vec_validate (vm_clone->pending_rpc_requests, 0); _vec_len (vm_clone->pending_rpc_requests) = 0; @@ -799,7 +797,7 @@ start_workers (vlib_main_t * vm) (&vm_clone->vlib_node_runtime_perf_callbacks, &vm_clone->worker_thread_main_loop_callback_lock); - nm = &vlib_mains[0]->node_main; + nm = &vlib_get_first_main ()->node_main; nm_clone = &vm_clone->node_main; /* fork next frames array, preserving node runtime indices */ nm_clone->next_frames = vec_dup_aligned (nm->next_frames, @@ -899,18 +897,20 @@ start_workers (vlib_main_t * vm) /* Packet trace buffers are guaranteed to be empty, nothing to do here */ clib_mem_set_heap (oldheap); - vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES); + vec_add1_aligned (vgm->vlib_mains, vm_clone, + CLIB_CACHE_LINE_BYTES); /* Switch to the stats segment ... */ void *oldheap = vlib_stats_push_heap (0); - vm_clone->error_main.counters = vec_dup_aligned - (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES); + vm_clone->error_main.counters = + vec_dup_aligned (vlib_get_first_main ()->error_main.counters, + CLIB_CACHE_LINE_BYTES); vlib_stats_pop_heap2 (vm_clone->error_main.counters, worker_thread_index, oldheap, 1); - vm_clone->error_main.counters_last_clear = vec_dup_aligned - (vlib_mains[0]->error_main.counters_last_clear, - CLIB_CACHE_LINE_BYTES); + vm_clone->error_main.counters_last_clear = vec_dup_aligned ( + vlib_get_first_main ()->error_main.counters_last_clear, + CLIB_CACHE_LINE_BYTES); worker_thread_index++; } @@ -1386,9 +1386,10 @@ vnet_main_fixup (vlib_fork_fixup_t which) void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_main_t *vm = vlib_get_main (); - if (vlib_mains == 0) + if (vgm->vlib_mains == 0) return; ASSERT (vlib_get_thread_index () == 0); @@ -1557,6 +1558,7 @@ vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name) void vlib_worker_thread_barrier_release (vlib_main_t * vm) { + vlib_global_main_t *vgm = vlib_get_global_main (); f64 deadline; f64 now; f64 minimum_open; @@ -1581,7 +1583,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) } /* Update (all) node runtimes before releasing the barrier, if needed */ - if (vm->need_vlib_worker_thread_node_runtime_update) + if (vgm->need_vlib_worker_thread_node_runtime_update) { /* * Lock stat segment here, so we's safe when @@ -1592,7 +1594,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) /* Do stats elements on main thread */ worker_thread_node_runtime_update_internal (); - vm->need_vlib_worker_thread_node_runtime_update = 0; + vgm->need_vlib_worker_thread_node_runtime_update = 0; /* Do per thread rebuilds in parallel */ refork_needed = 1; @@ -1669,6 +1671,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) void vlib_worker_wait_one_loop (void) { + vlib_global_main_t *vgm = vlib_get_global_main (); ASSERT (vlib_get_thread_index () == 0); if (vlib_get_n_threads () < 2) @@ -1683,14 +1686,14 @@ vlib_worker_wait_one_loop (void) vec_validate (counts, vlib_get_n_threads () - 1); /* record the current loop counts */ - vec_foreach_index (ii, vlib_mains) - counts[ii] = vlib_mains[ii]->main_loop_count; + vec_foreach_index (ii, vgm->vlib_mains) + counts[ii] = vgm->vlib_mains[ii]->main_loop_count; /* spin until each changes, apart from the main thread, or we'd be * a while */ for (ii = 1; ii < vec_len (counts); ii++) { - while (counts[ii] == vlib_mains[ii]->main_loop_count) + while (counts[ii] == vgm->vlib_mains[ii]->main_loop_count) CLIB_PAUSE (); } @@ -1717,7 +1720,7 @@ vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) u32 vectors = 0; ASSERT (fq); - ASSERT (vm == vlib_mains[thread_id]); + ASSERT (vm == vlib_global_main.vlib_mains[thread_id]); if (PREDICT_FALSE (fqm->node_index == ~0)) return 0; @@ -1840,6 +1843,7 @@ vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) void vlib_worker_thread_fn (void *arg) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; vlib_thread_main_t *tm = vlib_get_thread_main (); vlib_main_t *vm = vlib_get_main (); @@ -1853,8 +1857,11 @@ vlib_worker_thread_fn (void *arg) clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); - e = vlib_call_init_exit_functions_no_sort - (vm, &vm->worker_init_function_registrations, 1 /* call_once */ ); + vm->worker_init_functions_called = hash_create (0, 0); + + e = vlib_call_init_exit_functions_no_sort ( + vm, &vgm->worker_init_function_registrations, 1 /* call_once */, + 0 /* is_global */); if (e) clib_error_report (e); diff --git a/src/vlib/threads.h b/src/vlib/threads.h index b514ba415ec..d715ebfff58 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -19,8 +19,6 @@ #include #include -extern vlib_main_t **vlib_mains; - void vlib_set_thread_name (char *name); /* arg is actually a vlib__thread_t * */ @@ -239,28 +237,29 @@ typedef enum void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which); -#define foreach_vlib_main(body) \ -do { \ - vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \ - int ii; \ - \ - for (ii = 0; ii < vec_len (vlib_mains); ii++) \ - { \ - this_vlib_main = vlib_mains[ii]; \ - ASSERT (ii == 0 || \ - this_vlib_main->parked_at_barrier == 1); \ - if (this_vlib_main) \ - vec_add1 (__vlib_mains, this_vlib_main); \ - } \ - \ - for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ - { \ - this_vlib_main = __vlib_mains[ii]; \ - /* body uses this_vlib_main... */ \ - (body); \ - } \ - vec_free (__vlib_mains); \ -} while (0); +#define foreach_vlib_main(body) \ + do \ + { \ + vlib_main_t **__vlib_mains = 0, *this_vlib_main; \ + int ii; \ + \ + for (ii = 0; ii < vec_len (vlib_global_main.vlib_mains); ii++) \ + { \ + this_vlib_main = vlib_global_main.vlib_mains[ii]; \ + ASSERT (ii == 0 || this_vlib_main->parked_at_barrier == 1); \ + if (this_vlib_main) \ + vec_add1 (__vlib_mains, this_vlib_main); \ + } \ + \ + for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ + { \ + this_vlib_main = __vlib_mains[ii]; \ + /* body uses this_vlib_main... */ \ + (body); \ + } \ + vec_free (__vlib_mains); \ + } \ + while (0); #define foreach_sched_policy \ _(SCHED_OTHER, OTHER, "other") \ @@ -402,6 +401,7 @@ vlib_worker_thread_barrier_check (void) { if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier)) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_main_t *vm = vlib_get_main (); u32 thread_index = vm->thread_index; f64 t = vlib_time_now (vm); @@ -448,7 +448,7 @@ vlib_worker_thread_barrier_check (void) f64 now; vm->time_offset = 0.0; now = vlib_time_now (vm); - vm->time_offset = vlib_global_main.time_last_barrier_release - now; + vm->time_offset = vgm->vlib_mains[0]->time_last_barrier_release - now; vm->time_last_barrier_release = vlib_time_now (vm); } diff --git a/src/vlib/trace_funcs.h b/src/vlib/trace_funcs.h index d9d8d7933ee..9313d41eb7d 100644 --- a/src/vlib/trace_funcs.h +++ b/src/vlib/trace_funcs.h @@ -164,9 +164,9 @@ vlib_trace_buffer (vlib_main_t * vm, if (PREDICT_FALSE (vlib_global_main.trace_filter.trace_filter_enable)) { /* See if we're supposed to trace this packet... */ - if (vnet_is_packet_traced - (b, vlib_global_main.trace_filter.classify_table_index, - 0 /* full classify */ ) != 1) + if (vnet_is_packet_traced ( + b, vlib_global_main.trace_filter.classify_table_index, + 0 /* full classify */) != 1) return 0; } diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index a0a13f3affa..4ef96652470 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -384,6 +384,7 @@ VLIB_REGISTER_NODE (startup_config_node,static) = { static clib_error_t * unix_config (vlib_main_t * vm, unformat_input_t * input) { + vlib_global_main_t *vgm = vlib_get_global_main (); unix_main_t *um = &unix_main; clib_error_t *error = 0; gid_t gid; @@ -537,7 +538,7 @@ unix_config (vlib_main_t * vm, unformat_input_t * input) if (!(um->flags & (UNIX_FLAG_INTERACTIVE | UNIX_FLAG_NOSYSLOG))) { - openlog (vm->name, LOG_CONS | LOG_PERROR | LOG_PID, LOG_DAEMON); + openlog (vgm->name, LOG_CONS | LOG_PERROR | LOG_PID, LOG_DAEMON); clib_error_register_handler (unix_error_handler, um); if (!(um->flags & UNIX_FLAG_NODAEMON) && daemon ( /* chdir to / */ 0, @@ -692,13 +693,16 @@ vlib_thread_stack_init (uword thread_index) int vlib_unix_main (int argc, char *argv[]) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_main_t *vm = vlib_get_first_main (); /* one and only time for this! */ unformat_input_t input; clib_error_t *e; int i; + vec_validate_aligned (vgm->vlib_mains, 0, CLIB_CACHE_LINE_BYTES); + vm->argv = (u8 **) argv; - vm->name = argv[0]; + vgm->name = argv[0]; vm->heap_base = clib_mem_get_heap (); vm->heap_aligned_base = (void *) (((uword) vm->heap_base) & ~(VLIB_FRAME_ALIGN - 1)); @@ -707,8 +711,8 @@ vlib_unix_main (int argc, char *argv[]) clib_time_init (&vm->clib_time); /* Turn on the event logger at the first possible moment */ - vm->configured_elog_ring_size = 128 << 10; - elog_init (vlib_get_elog_main (), vm->configured_elog_ring_size); + vgm->configured_elog_ring_size = 128 << 10; + elog_init (vlib_get_elog_main (), vgm->configured_elog_ring_size); elog_enable_disable (vlib_get_elog_main (), 1); unformat_init_command_line (&input, (char **) vm->argv); @@ -724,8 +728,8 @@ vlib_unix_main (int argc, char *argv[]) return i; unformat_init_command_line (&input, (char **) vm->argv); - if (vm->init_functions_called == 0) - vm->init_functions_called = hash_create (0, /* value bytes */ 0); + if (vgm->init_functions_called == 0) + vgm->init_functions_called = hash_create (0, /* value bytes */ 0); e = vlib_call_all_config_functions (vm, &input, 1 /* early */ ); if (e != 0) { @@ -735,7 +739,7 @@ vlib_unix_main (int argc, char *argv[]) unformat_free (&input); /* always load symbols, for signal handler and mheap memory get/put backtrace */ - clib_elf_main_init (vm->name); + clib_elf_main_init (vgm->name); vec_validate (vlib_thread_stacks, 0); vlib_thread_stack_init (0); diff --git a/src/vlib/vlib.h b/src/vlib/vlib.h index 708732578f3..36f8a361abc 100644 --- a/src/vlib/vlib.h +++ b/src/vlib/vlib.h @@ -49,6 +49,7 @@ /* Forward declarations of structs to avoid circular dependencies. */ struct vlib_main_t; +struct vlib_global_main_t; typedef u32 vlib_log_class_t; /* All includes in alphabetical order. */ -- cgit 1.2.3-korg