support for graceful shutdown

10 files changed, 265 insertions, 49 deletions

diff --git a/scripts/automation/trex_control_plane/stl/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py
index 7ad0cfa4..110457d6 100755
--- a/scripts/automation/trex_control_plane/stl/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py
@@ -279,6 +279,11 @@ class TRexConsole(TRexGeneralCmd):
         self.stateless_client.ping_line(line)
 
 
+    @verify_connected
+    def do_shutdown (self, line):
+        '''Sends the server a shutdown request\n'''
+        self.stateless_client.shutdown_line(line)
+
     # set verbose on / off
     def do_verbose(self, line):
         '''Shows or set verbose mode\n'''
diff --git a/scripts/automation/trex_control_plane/stl/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py
index d3da738b..a69c4165 100644
--- a/scripts/automation/trex_control_plane/stl/console/trex_tui.py
+++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py
@@ -720,6 +720,7 @@ class AsyncKeysEngineConsole:
         self.ac = {'start'        : client.start_line,
                    'stop'         : client.stop_line,
                    'pause'        : client.pause_line,
+                   'push'         : client.push_line,
                    'resume'       : client.resume_line,
                    'update'       : client.update_line,
                    'connect'      : client.connect_line,
@@ -847,8 +848,9 @@ class AsyncKeysEngineConsole:
 
     # handle TAB for completing filenames
     def handle_tab_files (self, tokens):
-        # we support only start command with files
-        if tokens[0] != 'start':
+
+        # only commands with files
+        if tokens[0] not in {'start', 'push'}:
             return
 
         # '-f' with no paramters - no partial and use current dir
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index a4f26f69..4e3d3092 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -1658,8 +1658,8 @@ class STLClient(object):
 
         """
 
-        self.logger.pre_cmd( "Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'],
-                                                                               self.connection_info['sync_port']))
+        self.logger.pre_cmd("Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'],
+                                                                              self.connection_info['sync_port']))
         rc = self._transmit("ping", api_class = None)
         
         self.logger.post_cmd(rc)
@@ -1668,6 +1668,30 @@ class STLClient(object):
             raise STLError(rc)
 
     @__api_check(True)
+    def server_shutdown (self, force = False):
+        """
+            Sends the server a request for total shutdown
+
+            :parameters:
+                force - shutdown server even if some ports are owned by another
+                        user
+
+            :raises:
+                + :exc:`STLError`
+
+        """
+
+        self.logger.pre_cmd("Sending shutdown request for the server")
+
+        rc = self._transmit("shutdown", params = {'force': force, 'user': self.username})
+
+        self.logger.post_cmd(rc)
+
+        if not rc:
+            raise STLError(rc)
+
+
+    @__api_check(True)
     def get_active_pgids(self):
         """
             Get active group IDs
@@ -2107,7 +2131,7 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
-        validate_type('pcap_filename', pcap_filename, str)
+        validate_type('pcap_filename', pcap_filename, basestring)
         validate_type('ipg_usec', ipg_usec, (float, int, type(None)))
         validate_type('speedup',  speedup, (float, int))
         validate_type('count',  count, int)
@@ -2174,7 +2198,7 @@ class STLClient(object):
         ports = ports if ports is not None else self.get_acquired_ports()
         ports = self._validate_port_list(ports)
 
-        validate_type('pcap_filename', pcap_filename, str)
+        validate_type('pcap_filename', pcap_filename, basestring)
         validate_type('ipg_usec', ipg_usec, (float, int, type(None)))
         validate_type('speedup',  speedup, (float, int))
         validate_type('count',  count, int)
@@ -2445,6 +2469,21 @@ class STLClient(object):
         return RC_OK()
 
     @__console
+    def shutdown_line (self, line):
+        '''shutdown the server'''
+        parser = parsing_opts.gen_parser(self,
+                                         "shutdown",
+                                         self.shutdown_line.__doc__,
+                                         parsing_opts.FORCE)
+
+        opts = parser.parse_args(line.split())
+        if not opts:
+            return opts
+
+        self.server_shutdown(force = opts.force)    
+        return RC_OK()
+
+    @__console
     def connect_line (self, line):
         '''Connects to the TRex server and acquire ports'''
         parser = parsing_opts.gen_parser(self,
diff --git a/src/internal_api/trex_platform_api.h b/src/internal_api/trex_platform_api.h
index 7037584b..72bab7cb 100644
--- a/src/internal_api/trex_platform_api.h
+++ b/src/internal_api/trex_platform_api.h
@@ -161,6 +161,8 @@ public:
     virtual int get_cpu_util_full(cpu_util_full_t &result) const = 0;
     virtual int get_mbuf_util(Json::Value &result) const = 0;
     virtual CFlowStatParser *get_flow_stat_parser() const = 0;
+    virtual void mark_for_shutdown(const char *cause) const = 0;
+
     virtual ~TrexPlatformApi() {}
 };
 
@@ -195,6 +197,7 @@ public:
     int get_active_pgids(flow_stat_active_t &result) const;
     int get_cpu_util_full(cpu_util_full_t &result) const;
     int get_mbuf_util(Json::Value &result) const;
+    void mark_for_shutdown(const char *cause) const;
     CFlowStatParser *get_flow_stat_parser() const;
 };
 
@@ -264,6 +267,8 @@ public:
     int get_mbuf_util(Json::Value &result) const {return 0;}
     CFlowStatParser *get_flow_stat_parser() const {return new CFlowStatParser();}
 
+    void mark_for_shutdown(const char *cause) const {}
+
 private:
     int m_dp_core_count;
 };
diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp
index 7a6f972a..b6205f33 100644
--- a/src/main_dpdk.cpp
+++ b/src/main_dpdk.cpp
@@ -20,6 +20,7 @@
 */
 #include <assert.h>
 #include <pthread.h>
+#include <signal.h>
 #include <pwd.h>
 #include <stdio.h>
 #include <string.h>
@@ -2486,40 +2487,40 @@ public:
     void DumpAllPorts(FILE *fd);
     void dump_json(std::string & json, bool baseline);
 private:
-    std::string get_field(std::string name,float &f);
-    std::string get_field(std::string name,uint64_t &f);
-    std::string get_field_port(int port,std::string name,float &f);
-    std::string get_field_port(int port,std::string name,uint64_t &f);
+    std::string get_field(const char *name, float &f);
+    std::string get_field(const char *name, uint64_t &f);
+    std::string get_field_port(int port, const char *name, float &f);
+    std::string get_field_port(int port, const char *name, uint64_t &f);
 
 };
 
-std::string CGlobalStats::get_field(std::string name,float &f){
+std::string CGlobalStats::get_field(const char *name, float &f){
     char buff[200];
     if(f <= -10.0 or f >= 10.0)
-        snprintf(buff, sizeof(buff), "\"%s\":%.1f,",name.c_str(),f);
+        snprintf(buff, sizeof(buff), "\"%s\":%.1f,",name,f);
     else
-        snprintf(buff, sizeof(buff), "\"%s\":%.3e,",name.c_str(),f);
+        snprintf(buff, sizeof(buff), "\"%s\":%.3e,",name,f);
     return (std::string(buff));
 }
 
-std::string CGlobalStats::get_field(std::string name,uint64_t &f){
+std::string CGlobalStats::get_field(const char *name, uint64_t &f){
     char buff[200];
-    snprintf(buff,  sizeof(buff), "\"%s\":%llu,",name.c_str(), (unsigned long long)f);
+    snprintf(buff,  sizeof(buff), "\"%s\":%llu,", name, (unsigned long long)f);
     return (std::string(buff));
 }
 
-std::string CGlobalStats::get_field_port(int port,std::string name,float &f){
+std::string CGlobalStats::get_field_port(int port, const char *name, float &f){
     char buff[200];
     if(f <= -10.0 or f >= 10.0)
-        snprintf(buff,  sizeof(buff), "\"%s-%d\":%.1f,",name.c_str(),port,f);
+        snprintf(buff,  sizeof(buff), "\"%s-%d\":%.1f,", name, port, f);
     else
-        snprintf(buff, sizeof(buff), "\"%s-%d\":%.3e,",name.c_str(),port,f);
+        snprintf(buff, sizeof(buff), "\"%s-%d\":%.3e,", name, port, f);
     return (std::string(buff));
 }
 
-std::string CGlobalStats::get_field_port(int port,std::string name,uint64_t &f){
+std::string CGlobalStats::get_field_port(int port, const char *name, uint64_t &f){
     char buff[200];
-    snprintf(buff, sizeof(buff), "\"%s-%d\":%llu,",name.c_str(),port, (unsigned long long)f);
+    snprintf(buff, sizeof(buff), "\"%s-%d\":%llu,",name, port, (unsigned long long)f);
     return (std::string(buff));
 }
 
@@ -2538,8 +2539,8 @@ void CGlobalStats::dump_json(std::string & json, bool baseline){
     snprintf(ts_buff , sizeof(ts_buff), "\"ts\":{\"value\":%lu, \"freq\":%lu},", os_get_hr_tick_64(), os_get_hr_freq());
     json+= std::string(ts_buff);
 
-#define GET_FIELD(f) get_field(std::string(#f),f)
-#define GET_FIELD_PORT(p,f) get_field_port(p,std::string(#f),lp->f)
+#define GET_FIELD(f) get_field(#f, f)
+#define GET_FIELD_PORT(p,f) get_field_port(p, #f, lp->f)
 
     json+=GET_FIELD(m_cpu_util);
     json+=GET_FIELD(m_cpu_util_raw);
@@ -2781,6 +2782,7 @@ public:
         m_expected_cps=0.0;
         m_expected_bps=0.0;
         m_trex_stateless = NULL;
+        m_mark_for_shutdown = false;
     }
 
     bool Create();
@@ -2795,7 +2797,19 @@ public:
     bool is_all_links_are_up(bool dump=false);
     int  reset_counters();
 
+    /**
+     * mark for shutdown 
+     * on the next check - the control plane will 
+     * call shutdown() 
+     */
+    void mark_for_shutdown(const char *cause) {
+        printf("\n *** TRex shutting down - cause: '%s'\n", cause);
+        m_mark_for_shutdown = true;
+    }
+
 private:
+    void register_signals();
+
     /* try to stop all datapath cores and RX core */
     void try_stop_all_cores();
     /* send message to all dp cores */
@@ -2803,6 +2817,16 @@ private:
     int  send_message_to_rx(TrexStatelessCpToRxMsgBase *msg);
     void check_for_dp_message_from_core(int thread_id);
 
+    bool is_marked_for_shutdown() const {
+        return m_mark_for_shutdown;
+    }
+
+    /**
+     * shutdown sequence
+     * 
+     */
+    void shutdown();
+
 public:
     void check_for_dp_messages();
     int start_master_statefull();
@@ -2898,6 +2922,7 @@ private:
     std::mutex          m_cp_lock;
 
     TrexMonitor         m_monitor;
+    bool                m_mark_for_shutdown;
 
 public:
     TrexStateless       *m_trex_stateless;
@@ -3272,9 +3297,32 @@ int  CGlobalTRex::ixgbe_start(void){
     return (0);
 }
 
+static void trex_termination_handler(int signum);
+
+void CGlobalTRex::register_signals() {
+    struct sigaction action;
+
+    /* handler */
+    action.sa_handler = trex_termination_handler;
+
+    /* blocked signals during handling */
+    sigemptyset(&action.sa_mask);
+    sigaddset(&action.sa_mask, SIGINT);
+    sigaddset(&action.sa_mask, SIGTERM);
+
+    /* no flags */
+    action.sa_flags = 0;
+
+    /* register */
+    sigaction(SIGINT,  &action, NULL);
+    sigaction(SIGTERM, &action, NULL);
+}
+
 bool CGlobalTRex::Create(){
     CFlowsYamlInfo     pre_yaml_info;
 
+    register_signals();
+
     m_stats_cnt =0;
     if (!get_is_stateless()) {
         pre_yaml_info.load_from_yaml_file(CGlobalInfo::m_options.cfg_file);
@@ -3908,17 +3956,16 @@ CGlobalTRex::handle_slow_path(bool &was_stopped) {
 
     if ( CGlobalInfo::m_options.preview.get_no_keyboard() ==false ) {
         if ( m_io_modes.handle_io_modes() ) {
-            printf(" CTRL -C ... \n");
-            was_stopped=true;
+            mark_for_shutdown("CTRL + C detected");
             return false;
         }
     }
 
     if ( sanity_check() ) {
-        printf(" Test was stopped \n");
-        was_stopped=true;
+        mark_for_shutdown("Test was stopped");
         return false;
     }
+
     if (m_io_modes.m_g_mode != CTrexGlobalIoMode::gDISABLE ) {
         fprintf(stdout,"\033[2J");
         fprintf(stdout,"\033[2H");
@@ -4032,6 +4079,41 @@ CGlobalTRex::handle_fast_path() {
     return true;
 }
 
+
+/**
+ * shutdown sequence
+ * 
+ */
+void CGlobalTRex::shutdown() {
+
+    /* first stop the WD */
+    TrexWatchDog::getInstance().stop();
+
+    /* stateless shutdown */
+    if (get_is_stateless()) {
+        m_trex_stateless->shutdown();
+    }
+
+    if (!is_all_cores_finished()) {
+        try_stop_all_cores();
+    }
+
+    m_mg.stop();
+
+    delay(1000);
+
+    /* shutdown drivers */
+    for (int i = 0; i < m_max_ports; i++) {
+        rte_eth_dev_stop(i);
+    }
+
+    if (is_marked_for_shutdown()) {
+        /* we should stop latency and exit to stop agents */
+        exit(-1);
+    }
+}
+
+
 int CGlobalTRex::run_in_master() {
     bool was_stopped=false;
 
@@ -4052,7 +4134,7 @@ int CGlobalTRex::run_in_master() {
 
     TrexWatchDog::getInstance().start();
 
-    while ( true ) {
+    while (!is_marked_for_shutdown()) {
 
         /* fast path */
         if (!handle_fast_path()) {
@@ -4079,22 +4161,9 @@ int CGlobalTRex::run_in_master() {
     /* on exit release the lock */
     cp_lock.unlock();
 
-    /* first stop the WD */
-    TrexWatchDog::getInstance().stop();
-
-    if (!is_all_cores_finished()) {
-        /* probably CLTR-C */
-        try_stop_all_cores();
-    }
-
-    m_mg.stop();
-
-
-    delay(1000);
-    if ( was_stopped ){
-        /* we should stop latency and exit to stop agents */
-        exit(-1);
-    }
+    /* shutdown everything gracefully */
+    shutdown();
+   
     return (0);
 }
 
@@ -5702,6 +5771,37 @@ struct rte_mbuf *  rte_mbuf_convert_to_one_seg(struct rte_mbuf *m){
     return(r);
 }
 
+/**
+ * handle a signal for termination
+ * 
+ * @author imarom (7/27/2016)
+ * 
+ * @param signum 
+ */
+static void trex_termination_handler(int signum) {
+    std::stringstream ss;
+
+    /* be sure that this was given on the main process */
+    assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+
+    const char *signame = "";
+    switch (signum) {
+    case SIGINT:
+        signame = "SIGINT";
+        break;
+
+    case SIGTERM:
+        signame = "SIGTERM";
+        break;
+
+    default:
+        assert(0);
+    }
+    
+    ss << "receieved signal '" << signame << "'";
+    g_trex.mark_for_shutdown(ss.str().c_str());
+}
+
 /***********************************************************
  * platfrom API object
  * TODO: REMOVE THIS TO A SEPERATE FILE
@@ -5871,3 +5971,12 @@ int TrexDpdkPlatformApi::get_mbuf_util(Json::Value &mbuf_pool) const {
 CFlowStatParser *TrexDpdkPlatformApi::get_flow_stat_parser() const {
     return CTRexExtendedDriverDb::Ins()->get_drv()->get_flow_stat_parser();
 }
+
+/**
+ * marks the control plane for a total server shutdown
+ * 
+ * @author imarom (7/27/2016)
+ */
+void TrexDpdkPlatformApi::mark_for_shutdown(const char *cause) const {
+    g_trex.mark_for_shutdown(cause);
+}
diff --git a/src/rpc-server/commands/trex_rpc_cmd_general.cpp b/src/rpc-server/commands/trex_rpc_cmd_general.cpp
index 27010e0e..c3fba8e1 100644
--- a/src/rpc-server/commands/trex_rpc_cmd_general.cpp
+++ b/src/rpc-server/commands/trex_rpc_cmd_general.cpp
@@ -92,6 +92,32 @@ TrexRpcCmdPing::_run(const Json::Value &params, Json::Value &result) {
 }
 
 /**
+ * shutdown command
+ */
+trex_rpc_cmd_rc_e
+TrexRpcCmdShutdown::_run(const Json::Value &params, Json::Value &result) {
+
+    const string &user = parse_string(params, "user", result);
+    bool force = parse_bool(params, "force", result);
+
+    /* verify every port is either free or owned by the issuer */
+    for (auto port : get_stateless_obj()->get_port_list()) {
+        TrexPortOwner &owner = port->get_owner();
+        if ( (!owner.is_free()) && (!owner.is_owned_by(user)) && !force) {
+            std::stringstream ss;
+            ss << "port " << int(port->get_port_id()) << " is owned by '" << owner.get_name() << "' - specify 'force' for override";
+            generate_execute_err(result, ss.str());
+        }
+    }
+
+    /* signal that we got a shutdown request */
+    get_stateless_obj()->get_platform_api()->mark_for_shutdown("server received RPC 'shutdown' request");
+
+    result["result"] = Json::objectValue;
+    return (TREX_RPC_CMD_OK);
+}
+
+/**
  * query command
  */
 trex_rpc_cmd_rc_e
diff --git a/src/rpc-server/commands/trex_rpc_cmds.h b/src/rpc-server/commands/trex_rpc_cmds.h
index 2776727d..24b95227 100644
--- a/src/rpc-server/commands/trex_rpc_cmds.h
+++ b/src/rpc-server/commands/trex_rpc_cmds.h
@@ -133,5 +133,7 @@ TREX_RPC_CMD_DEFINE(TrexRpcCmdValidate, "validate", 2, false, APIClass::API_CLAS
 
 TREX_RPC_CMD_DEFINE(TrexRpcCmdPushRemote, "push_remote", 6, true, APIClass::API_CLASS_TYPE_CORE);
 
+TREX_RPC_CMD_DEFINE(TrexRpcCmdShutdown, "shutdown", 2, false, APIClass::API_CLASS_TYPE_CORE);
+
 #endif /* __TREX_RPC_CMD_H__ */
 
diff --git a/src/rpc-server/trex_rpc_cmds_table.cpp b/src/rpc-server/trex_rpc_cmds_table.cpp
index 6144d265..762dd614 100644
--- a/src/rpc-server/trex_rpc_cmds_table.cpp
+++ b/src/rpc-server/trex_rpc_cmds_table.cpp
@@ -67,6 +67,8 @@ TrexRpcCommandsTable::TrexRpcCommandsTable() {
 
     register_command(new TrexRpcCmdValidate());
     register_command(new TrexRpcCmdPushRemote());
+
+    register_command(new TrexRpcCmdShutdown());
 }
 
 
diff --git a/src/stateless/cp/trex_stateless.cpp b/src/stateless/cp/trex_stateless.cpp
index 8633897e..6d80539c 100644
--- a/src/stateless/cp/trex_stateless.cpp
+++ b/src/stateless/cp/trex_stateless.cpp
@@ -68,6 +68,8 @@ TrexStateless::TrexStateless(const TrexStatelessCfg &cfg) {
  */
 TrexStateless::~TrexStateless() {
 
+    shutdown();
+
     /* release memory for ports */
     for (auto port : m_ports) {
         delete port;
@@ -75,15 +77,33 @@ TrexStateless::~TrexStateless() {
     m_ports.clear();
 
     /* stops the RPC server */
-    m_rpc_server->stop();
-    delete m_rpc_server;
-
-    m_rpc_server = NULL;
+    if (m_rpc_server) {
+        delete m_rpc_server;
+        m_rpc_server = NULL;
+    }
 
-    delete m_platform_api;
-    m_platform_api = NULL;
+    if (m_platform_api) {
+        delete m_platform_api;
+        m_platform_api = NULL;
+    }
 }
 
+/**
+* shutdown the server
+*/
+void TrexStateless::shutdown() {
+
+    /* stop ports */
+    for (TrexStatelessPort *port : m_ports) {
+        /* safe to call stop even if not active */
+        port->stop_traffic();
+    }
+
+    /* shutdown the RPC server */
+    if (m_rpc_server) {
+        m_rpc_server->stop();
+    }
+}
 
 /**
  * starts the control plane side
diff --git a/src/stateless/cp/trex_stateless.h b/src/stateless/cp/trex_stateless.h
index 83ab6976..7ea669df 100644
--- a/src/stateless/cp/trex_stateless.h
+++ b/src/stateless/cp/trex_stateless.h
@@ -132,6 +132,11 @@ public:
 
 
     /**
+     * shutdown the server
+     */
+    void shutdown();
+
+    /**
      * fetch all the stats
      * 
      */
@@ -188,6 +193,7 @@ protected:
 
     /* API */
     APIClass                              m_api_classes[APIClass::API_CLASS_TYPE_MAX];
+
 };
 
 /**