From 344e3045d8346b4b204692e591e1556fc2333f97 Mon Sep 17 00:00:00 2001 From: imarom Date: Wed, 27 Jul 2016 11:08:09 +0300 Subject: support for graceful shutdown --- .../trex_control_plane/stl/console/trex_console.py | 5 + .../trex_control_plane/stl/console/trex_tui.py | 6 +- .../stl/trex_stl_lib/trex_stl_client.py | 47 +++++- src/internal_api/trex_platform_api.h | 5 + src/main_dpdk.cpp | 183 ++++++++++++++++----- src/rpc-server/commands/trex_rpc_cmd_general.cpp | 26 +++ src/rpc-server/commands/trex_rpc_cmds.h | 2 + src/rpc-server/trex_rpc_cmds_table.cpp | 2 + src/stateless/cp/trex_stateless.cpp | 32 +++- src/stateless/cp/trex_stateless.h | 6 + 10 files changed, 265 insertions(+), 49 deletions(-) diff --git a/scripts/automation/trex_control_plane/stl/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py index 7ad0cfa4..110457d6 100755 --- a/scripts/automation/trex_control_plane/stl/console/trex_console.py +++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py @@ -279,6 +279,11 @@ class TRexConsole(TRexGeneralCmd): self.stateless_client.ping_line(line) + @verify_connected + def do_shutdown (self, line): + '''Sends the server a shutdown request\n''' + self.stateless_client.shutdown_line(line) + # set verbose on / off def do_verbose(self, line): '''Shows or set verbose mode\n''' diff --git a/scripts/automation/trex_control_plane/stl/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py index d3da738b..a69c4165 100644 --- a/scripts/automation/trex_control_plane/stl/console/trex_tui.py +++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py @@ -720,6 +720,7 @@ class AsyncKeysEngineConsole: self.ac = {'start' : client.start_line, 'stop' : client.stop_line, 'pause' : client.pause_line, + 'push' : client.push_line, 'resume' : client.resume_line, 'update' : client.update_line, 'connect' : client.connect_line, @@ -847,8 +848,9 @@ class AsyncKeysEngineConsole: # handle TAB for completing filenames def handle_tab_files (self, tokens): - # we support only start command with files - if tokens[0] != 'start': + + # only commands with files + if tokens[0] not in {'start', 'push'}: return # '-f' with no paramters - no partial and use current dir diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py index a4f26f69..4e3d3092 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py @@ -1658,8 +1658,8 @@ class STLClient(object): """ - self.logger.pre_cmd( "Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'], - self.connection_info['sync_port'])) + self.logger.pre_cmd("Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'], + self.connection_info['sync_port'])) rc = self._transmit("ping", api_class = None) self.logger.post_cmd(rc) @@ -1667,6 +1667,30 @@ class STLClient(object): if not rc: raise STLError(rc) + @__api_check(True) + def server_shutdown (self, force = False): + """ + Sends the server a request for total shutdown + + :parameters: + force - shutdown server even if some ports are owned by another + user + + :raises: + + :exc:`STLError` + + """ + + self.logger.pre_cmd("Sending shutdown request for the server") + + rc = self._transmit("shutdown", params = {'force': force, 'user': self.username}) + + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + @__api_check(True) def get_active_pgids(self): """ @@ -2107,7 +2131,7 @@ class STLClient(object): ports = ports if ports is not None else self.get_acquired_ports() ports = self._validate_port_list(ports) - validate_type('pcap_filename', pcap_filename, str) + validate_type('pcap_filename', pcap_filename, basestring) validate_type('ipg_usec', ipg_usec, (float, int, type(None))) validate_type('speedup', speedup, (float, int)) validate_type('count', count, int) @@ -2174,7 +2198,7 @@ class STLClient(object): ports = ports if ports is not None else self.get_acquired_ports() ports = self._validate_port_list(ports) - validate_type('pcap_filename', pcap_filename, str) + validate_type('pcap_filename', pcap_filename, basestring) validate_type('ipg_usec', ipg_usec, (float, int, type(None))) validate_type('speedup', speedup, (float, int)) validate_type('count', count, int) @@ -2444,6 +2468,21 @@ class STLClient(object): self.ping() return RC_OK() + @__console + def shutdown_line (self, line): + '''shutdown the server''' + parser = parsing_opts.gen_parser(self, + "shutdown", + self.shutdown_line.__doc__, + parsing_opts.FORCE) + + opts = parser.parse_args(line.split()) + if not opts: + return opts + + self.server_shutdown(force = opts.force) + return RC_OK() + @__console def connect_line (self, line): '''Connects to the TRex server and acquire ports''' diff --git a/src/internal_api/trex_platform_api.h b/src/internal_api/trex_platform_api.h index 7037584b..72bab7cb 100644 --- a/src/internal_api/trex_platform_api.h +++ b/src/internal_api/trex_platform_api.h @@ -161,6 +161,8 @@ public: virtual int get_cpu_util_full(cpu_util_full_t &result) const = 0; virtual int get_mbuf_util(Json::Value &result) const = 0; virtual CFlowStatParser *get_flow_stat_parser() const = 0; + virtual void mark_for_shutdown(const char *cause) const = 0; + virtual ~TrexPlatformApi() {} }; @@ -195,6 +197,7 @@ public: int get_active_pgids(flow_stat_active_t &result) const; int get_cpu_util_full(cpu_util_full_t &result) const; int get_mbuf_util(Json::Value &result) const; + void mark_for_shutdown(const char *cause) const; CFlowStatParser *get_flow_stat_parser() const; }; @@ -264,6 +267,8 @@ public: int get_mbuf_util(Json::Value &result) const {return 0;} CFlowStatParser *get_flow_stat_parser() const {return new CFlowStatParser();} + void mark_for_shutdown(const char *cause) const {} + private: int m_dp_core_count; }; diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp index 7a6f972a..b6205f33 100644 --- a/src/main_dpdk.cpp +++ b/src/main_dpdk.cpp @@ -20,6 +20,7 @@ */ #include #include +#include #include #include #include @@ -2486,40 +2487,40 @@ public: void DumpAllPorts(FILE *fd); void dump_json(std::string & json, bool baseline); private: - std::string get_field(std::string name,float &f); - std::string get_field(std::string name,uint64_t &f); - std::string get_field_port(int port,std::string name,float &f); - std::string get_field_port(int port,std::string name,uint64_t &f); + std::string get_field(const char *name, float &f); + std::string get_field(const char *name, uint64_t &f); + std::string get_field_port(int port, const char *name, float &f); + std::string get_field_port(int port, const char *name, uint64_t &f); }; -std::string CGlobalStats::get_field(std::string name,float &f){ +std::string CGlobalStats::get_field(const char *name, float &f){ char buff[200]; if(f <= -10.0 or f >= 10.0) - snprintf(buff, sizeof(buff), "\"%s\":%.1f,",name.c_str(),f); + snprintf(buff, sizeof(buff), "\"%s\":%.1f,",name,f); else - snprintf(buff, sizeof(buff), "\"%s\":%.3e,",name.c_str(),f); + snprintf(buff, sizeof(buff), "\"%s\":%.3e,",name,f); return (std::string(buff)); } -std::string CGlobalStats::get_field(std::string name,uint64_t &f){ +std::string CGlobalStats::get_field(const char *name, uint64_t &f){ char buff[200]; - snprintf(buff, sizeof(buff), "\"%s\":%llu,",name.c_str(), (unsigned long long)f); + snprintf(buff, sizeof(buff), "\"%s\":%llu,", name, (unsigned long long)f); return (std::string(buff)); } -std::string CGlobalStats::get_field_port(int port,std::string name,float &f){ +std::string CGlobalStats::get_field_port(int port, const char *name, float &f){ char buff[200]; if(f <= -10.0 or f >= 10.0) - snprintf(buff, sizeof(buff), "\"%s-%d\":%.1f,",name.c_str(),port,f); + snprintf(buff, sizeof(buff), "\"%s-%d\":%.1f,", name, port, f); else - snprintf(buff, sizeof(buff), "\"%s-%d\":%.3e,",name.c_str(),port,f); + snprintf(buff, sizeof(buff), "\"%s-%d\":%.3e,", name, port, f); return (std::string(buff)); } -std::string CGlobalStats::get_field_port(int port,std::string name,uint64_t &f){ +std::string CGlobalStats::get_field_port(int port, const char *name, uint64_t &f){ char buff[200]; - snprintf(buff, sizeof(buff), "\"%s-%d\":%llu,",name.c_str(),port, (unsigned long long)f); + snprintf(buff, sizeof(buff), "\"%s-%d\":%llu,",name, port, (unsigned long long)f); return (std::string(buff)); } @@ -2538,8 +2539,8 @@ void CGlobalStats::dump_json(std::string & json, bool baseline){ snprintf(ts_buff , sizeof(ts_buff), "\"ts\":{\"value\":%lu, \"freq\":%lu},", os_get_hr_tick_64(), os_get_hr_freq()); json+= std::string(ts_buff); -#define GET_FIELD(f) get_field(std::string(#f),f) -#define GET_FIELD_PORT(p,f) get_field_port(p,std::string(#f),lp->f) +#define GET_FIELD(f) get_field(#f, f) +#define GET_FIELD_PORT(p,f) get_field_port(p, #f, lp->f) json+=GET_FIELD(m_cpu_util); json+=GET_FIELD(m_cpu_util_raw); @@ -2781,6 +2782,7 @@ public: m_expected_cps=0.0; m_expected_bps=0.0; m_trex_stateless = NULL; + m_mark_for_shutdown = false; } bool Create(); @@ -2795,7 +2797,19 @@ public: bool is_all_links_are_up(bool dump=false); int reset_counters(); + /** + * mark for shutdown + * on the next check - the control plane will + * call shutdown() + */ + void mark_for_shutdown(const char *cause) { + printf("\n *** TRex shutting down - cause: '%s'\n", cause); + m_mark_for_shutdown = true; + } + private: + void register_signals(); + /* try to stop all datapath cores and RX core */ void try_stop_all_cores(); /* send message to all dp cores */ @@ -2803,6 +2817,16 @@ private: int send_message_to_rx(TrexStatelessCpToRxMsgBase *msg); void check_for_dp_message_from_core(int thread_id); + bool is_marked_for_shutdown() const { + return m_mark_for_shutdown; + } + + /** + * shutdown sequence + * + */ + void shutdown(); + public: void check_for_dp_messages(); int start_master_statefull(); @@ -2898,6 +2922,7 @@ private: std::mutex m_cp_lock; TrexMonitor m_monitor; + bool m_mark_for_shutdown; public: TrexStateless *m_trex_stateless; @@ -3272,9 +3297,32 @@ int CGlobalTRex::ixgbe_start(void){ return (0); } +static void trex_termination_handler(int signum); + +void CGlobalTRex::register_signals() { + struct sigaction action; + + /* handler */ + action.sa_handler = trex_termination_handler; + + /* blocked signals during handling */ + sigemptyset(&action.sa_mask); + sigaddset(&action.sa_mask, SIGINT); + sigaddset(&action.sa_mask, SIGTERM); + + /* no flags */ + action.sa_flags = 0; + + /* register */ + sigaction(SIGINT, &action, NULL); + sigaction(SIGTERM, &action, NULL); +} + bool CGlobalTRex::Create(){ CFlowsYamlInfo pre_yaml_info; + register_signals(); + m_stats_cnt =0; if (!get_is_stateless()) { pre_yaml_info.load_from_yaml_file(CGlobalInfo::m_options.cfg_file); @@ -3908,17 +3956,16 @@ CGlobalTRex::handle_slow_path(bool &was_stopped) { if ( CGlobalInfo::m_options.preview.get_no_keyboard() ==false ) { if ( m_io_modes.handle_io_modes() ) { - printf(" CTRL -C ... \n"); - was_stopped=true; + mark_for_shutdown("CTRL + C detected"); return false; } } if ( sanity_check() ) { - printf(" Test was stopped \n"); - was_stopped=true; + mark_for_shutdown("Test was stopped"); return false; } + if (m_io_modes.m_g_mode != CTrexGlobalIoMode::gDISABLE ) { fprintf(stdout,"\033[2J"); fprintf(stdout,"\033[2H"); @@ -4032,6 +4079,41 @@ CGlobalTRex::handle_fast_path() { return true; } + +/** + * shutdown sequence + * + */ +void CGlobalTRex::shutdown() { + + /* first stop the WD */ + TrexWatchDog::getInstance().stop(); + + /* stateless shutdown */ + if (get_is_stateless()) { + m_trex_stateless->shutdown(); + } + + if (!is_all_cores_finished()) { + try_stop_all_cores(); + } + + m_mg.stop(); + + delay(1000); + + /* shutdown drivers */ + for (int i = 0; i < m_max_ports; i++) { + rte_eth_dev_stop(i); + } + + if (is_marked_for_shutdown()) { + /* we should stop latency and exit to stop agents */ + exit(-1); + } +} + + int CGlobalTRex::run_in_master() { bool was_stopped=false; @@ -4052,7 +4134,7 @@ int CGlobalTRex::run_in_master() { TrexWatchDog::getInstance().start(); - while ( true ) { + while (!is_marked_for_shutdown()) { /* fast path */ if (!handle_fast_path()) { @@ -4079,22 +4161,9 @@ int CGlobalTRex::run_in_master() { /* on exit release the lock */ cp_lock.unlock(); - /* first stop the WD */ - TrexWatchDog::getInstance().stop(); - - if (!is_all_cores_finished()) { - /* probably CLTR-C */ - try_stop_all_cores(); - } - - m_mg.stop(); - - - delay(1000); - if ( was_stopped ){ - /* we should stop latency and exit to stop agents */ - exit(-1); - } + /* shutdown everything gracefully */ + shutdown(); + return (0); } @@ -5702,6 +5771,37 @@ struct rte_mbuf * rte_mbuf_convert_to_one_seg(struct rte_mbuf *m){ return(r); } +/** + * handle a signal for termination + * + * @author imarom (7/27/2016) + * + * @param signum + */ +static void trex_termination_handler(int signum) { + std::stringstream ss; + + /* be sure that this was given on the main process */ + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); + + const char *signame = ""; + switch (signum) { + case SIGINT: + signame = "SIGINT"; + break; + + case SIGTERM: + signame = "SIGTERM"; + break; + + default: + assert(0); + } + + ss << "receieved signal '" << signame << "'"; + g_trex.mark_for_shutdown(ss.str().c_str()); +} + /*********************************************************** * platfrom API object * TODO: REMOVE THIS TO A SEPERATE FILE @@ -5871,3 +5971,12 @@ int TrexDpdkPlatformApi::get_mbuf_util(Json::Value &mbuf_pool) const { CFlowStatParser *TrexDpdkPlatformApi::get_flow_stat_parser() const { return CTRexExtendedDriverDb::Ins()->get_drv()->get_flow_stat_parser(); } + +/** + * marks the control plane for a total server shutdown + * + * @author imarom (7/27/2016) + */ +void TrexDpdkPlatformApi::mark_for_shutdown(const char *cause) const { + g_trex.mark_for_shutdown(cause); +} diff --git a/src/rpc-server/commands/trex_rpc_cmd_general.cpp b/src/rpc-server/commands/trex_rpc_cmd_general.cpp index 27010e0e..c3fba8e1 100644 --- a/src/rpc-server/commands/trex_rpc_cmd_general.cpp +++ b/src/rpc-server/commands/trex_rpc_cmd_general.cpp @@ -91,6 +91,32 @@ TrexRpcCmdPing::_run(const Json::Value ¶ms, Json::Value &result) { return (TREX_RPC_CMD_OK); } +/** + * shutdown command + */ +trex_rpc_cmd_rc_e +TrexRpcCmdShutdown::_run(const Json::Value ¶ms, Json::Value &result) { + + const string &user = parse_string(params, "user", result); + bool force = parse_bool(params, "force", result); + + /* verify every port is either free or owned by the issuer */ + for (auto port : get_stateless_obj()->get_port_list()) { + TrexPortOwner &owner = port->get_owner(); + if ( (!owner.is_free()) && (!owner.is_owned_by(user)) && !force) { + std::stringstream ss; + ss << "port " << int(port->get_port_id()) << " is owned by '" << owner.get_name() << "' - specify 'force' for override"; + generate_execute_err(result, ss.str()); + } + } + + /* signal that we got a shutdown request */ + get_stateless_obj()->get_platform_api()->mark_for_shutdown("server received RPC 'shutdown' request"); + + result["result"] = Json::objectValue; + return (TREX_RPC_CMD_OK); +} + /** * query command */ diff --git a/src/rpc-server/commands/trex_rpc_cmds.h b/src/rpc-server/commands/trex_rpc_cmds.h index 2776727d..24b95227 100644 --- a/src/rpc-server/commands/trex_rpc_cmds.h +++ b/src/rpc-server/commands/trex_rpc_cmds.h @@ -133,5 +133,7 @@ TREX_RPC_CMD_DEFINE(TrexRpcCmdValidate, "validate", 2, false, APIClass::API_CLAS TREX_RPC_CMD_DEFINE(TrexRpcCmdPushRemote, "push_remote", 6, true, APIClass::API_CLASS_TYPE_CORE); +TREX_RPC_CMD_DEFINE(TrexRpcCmdShutdown, "shutdown", 2, false, APIClass::API_CLASS_TYPE_CORE); + #endif /* __TREX_RPC_CMD_H__ */ diff --git a/src/rpc-server/trex_rpc_cmds_table.cpp b/src/rpc-server/trex_rpc_cmds_table.cpp index 6144d265..762dd614 100644 --- a/src/rpc-server/trex_rpc_cmds_table.cpp +++ b/src/rpc-server/trex_rpc_cmds_table.cpp @@ -67,6 +67,8 @@ TrexRpcCommandsTable::TrexRpcCommandsTable() { register_command(new TrexRpcCmdValidate()); register_command(new TrexRpcCmdPushRemote()); + + register_command(new TrexRpcCmdShutdown()); } diff --git a/src/stateless/cp/trex_stateless.cpp b/src/stateless/cp/trex_stateless.cpp index 8633897e..6d80539c 100644 --- a/src/stateless/cp/trex_stateless.cpp +++ b/src/stateless/cp/trex_stateless.cpp @@ -68,6 +68,8 @@ TrexStateless::TrexStateless(const TrexStatelessCfg &cfg) { */ TrexStateless::~TrexStateless() { + shutdown(); + /* release memory for ports */ for (auto port : m_ports) { delete port; @@ -75,15 +77,33 @@ TrexStateless::~TrexStateless() { m_ports.clear(); /* stops the RPC server */ - m_rpc_server->stop(); - delete m_rpc_server; - - m_rpc_server = NULL; + if (m_rpc_server) { + delete m_rpc_server; + m_rpc_server = NULL; + } - delete m_platform_api; - m_platform_api = NULL; + if (m_platform_api) { + delete m_platform_api; + m_platform_api = NULL; + } } +/** +* shutdown the server +*/ +void TrexStateless::shutdown() { + + /* stop ports */ + for (TrexStatelessPort *port : m_ports) { + /* safe to call stop even if not active */ + port->stop_traffic(); + } + + /* shutdown the RPC server */ + if (m_rpc_server) { + m_rpc_server->stop(); + } +} /** * starts the control plane side diff --git a/src/stateless/cp/trex_stateless.h b/src/stateless/cp/trex_stateless.h index 83ab6976..7ea669df 100644 --- a/src/stateless/cp/trex_stateless.h +++ b/src/stateless/cp/trex_stateless.h @@ -131,6 +131,11 @@ public: uint8_t get_dp_core_count(); + /** + * shutdown the server + */ + void shutdown(); + /** * fetch all the stats * @@ -188,6 +193,7 @@ protected: /* API */ APIClass m_api_classes[APIClass::API_CLASS_TYPE_MAX]; + }; /** -- cgit 1.2.3-korg