Merge branch 'master' into scapy_server

4 files changed, 77 insertions, 86 deletions

diff --git a/scripts/automation/trex_control_plane/server/CCustomLogger.py b/scripts/automation/trex_control_plane/server/CCustomLogger.py
index a8823cea..6d3974a6 100755
--- a/scripts/automation/trex_control_plane/server/CCustomLogger.py
+++ b/scripts/automation/trex_control_plane/server/CCustomLogger.py
@@ -3,15 +3,13 @@ import sys
 import os

 import logging

 

+def prepare_dir(log_path):

+    log_dir = os.path.dirname(log_path)

+    if not os.path.exists(log_dir):

+        os.makedirs(log_dir)

 

 def setup_custom_logger(name, log_path = None):

     # first make sure path availabe

-#   if log_path is None:

-#       log_path = os.getcwd()+'/trex_log.log'

-#   else:

-#       directory = os.path.dirname(log_path)

-#       if not os.path.exists(directory):

-#           os.makedirs(directory)

     logging.basicConfig(level   = logging.INFO, 

                         format  = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s',

                         datefmt = '%m-%d %H:%M')

@@ -31,6 +29,7 @@ def setup_custom_logger(name, log_path = None):
 

 def setup_daemon_logger (name, log_path = None):

     # first make sure path availabe

+    prepare_dir(log_path)

     try:

         os.unlink(log_path)

     except:

diff --git a/scripts/automation/trex_control_plane/server/singleton_daemon.py b/scripts/automation/trex_control_plane/server/singleton_daemon.py
index 8fdedc6e..1784cc42 100755
--- a/scripts/automation/trex_control_plane/server/singleton_daemon.py
+++ b/scripts/automation/trex_control_plane/server/singleton_daemon.py
@@ -2,6 +2,7 @@ import errno
 import os
 import shlex
 import socket
+import signal
 import tempfile
 import types
 from subprocess import Popen
@@ -73,31 +74,27 @@ class SingletonDaemon(object):
         if pid:
             return pid
 
-
-    # kill daemon
-    def kill(self, timeout = 10):
-        pid = self.get_pid()
-        if not pid:
-            return False
-        ret_code, stdout, stderr = run_command('kill %s' % pid) # usual kill
-        if ret_code:
-            raise Exception('Failed to run kill command for %s: %s' % (self.name, [ret_code, stdout, stderr]))
+    def kill_by_signal(self, pid, signal_name, timeout):
+        os.kill(pid, signal_name)
         poll_rate = 0.1
         for i in range(int(timeout / poll_rate)):
             if not self.is_running():
                 return True
             sleep(poll_rate)
-        ret_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill
-        if ret_code:
-            raise Exception('Failed to run kill -9 command for %s: %s' % (self.name, [ret_code, stdout, stderr]))
-        for i in range(int(timeout / poll_rate)):
-            if not self.is_running():
+
+    # kill daemon, with verification
+    def kill(self, timeout = 15):
+        pid = self.get_pid()
+        if not pid:
+            raise Exception('%s is not running' % self.name)
+        # try Ctrl+C, usual kill, kill -9
+        for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]:
+            if self.kill_by_signal(pid, signal_name, timeout):
                 return True
-            sleep(poll_rate)
         raise Exception('Could not kill %s, even with -9' % self.name)
 
     # try connection as RPC client, return True upon success, False if fail
-    def check_connectivity(self, timeout = 5):
+    def check_connectivity(self, timeout = 15):
         daemon = jsonrpclib.Server('http://127.0.0.1:%s/' % self.port)
         poll_rate = 0.1
         for i in range(int(timeout/poll_rate)):
@@ -140,7 +137,7 @@ class SingletonDaemon(object):
             raise Exception('%s failed to run.' % self.name)
 
     # restart the daemon
-    def restart(self, timeout = 5):
+    def restart(self, timeout = 15):
         if self.is_running():
             self.kill(timeout)
         return self.start(timeout)
diff --git a/scripts/automation/trex_control_plane/server/trex_launch_thread.py b/scripts/automation/trex_control_plane/server/trex_launch_thread.py
index 82a7f996..22606753 100755
--- a/scripts/automation/trex_control_plane/server/trex_launch_thread.py
+++ b/scripts/automation/trex_control_plane/server/trex_launch_thread.py
@@ -30,49 +30,51 @@ class AsynchronousTRexSession(threading.Thread):
         self.trexObj.zmq_dump                       = {}

 

     def run (self):

-            

-        with open(os.devnull, 'w') as DEVNULL:

-            self.time_stamps['start'] = self.time_stamps['run_time'] = time.time()

-            self.session   = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdin = DEVNULL, stderr = subprocess.PIPE, preexec_fn=os.setsid, close_fds = True)

-            logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid ))

-            while self.session.poll() is None:  # subprocess is NOT finished

-                time.sleep(0.5)

-                if self.stoprequest.is_set():

-                    logger.debug("Abort request received by handling thread. Terminating TRex session." )

-                    os.killpg(self.session.pid, signal.SIGUSR1)

-                    self.trexObj.set_status(TRexStatus.Idle)

-                    self.trexObj.set_verbose_status("TRex is Idle")

-                    break

-

-            self.time_stamps['run_time'] = time.time() - self.time_stamps['start']

-

-            try:

-                if self.time_stamps['run_time'] < 5:

-                    logger.error("TRex run failed due to wrong input parameters, or due to readability issues.")

-                    self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format(

-                        cmd = self.cmd, output = self.load_trex_output(self.export_path)))

-                    self.trexObj.errcode = -11

-                elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ):

-                    if (self.session.returncode is not None and self.session.returncode != 0):

-                        logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) )

-                    elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()):

-                        logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) )

-

-                    logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS")

-                    self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format(

-                        output = self.load_trex_output(self.export_path)))

-                    self.trexObj.errcode = -15

-                else:

-                    logger.info("TRex run session finished.")

-                    self.trexObj.set_verbose_status('TRex finished.')

-                    self.trexObj.errcode = None

-                    

-            finally:

-                self.trexObj.set_status(TRexStatus.Idle)

-                logger.info("TRex running state changed to 'Idle'.")

-                self.trexObj.expect_trex.clear()

-                logger.debug("Finished handling a single run of TRex.")

-                self.trexObj.zmq_dump   = None

+        try:

+            with open(self.export_path, 'w') as output_file:

+                self.time_stamps['start'] = self.time_stamps['run_time'] = time.time()

+                self.session   = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdout = output_file, preexec_fn=os.setsid, close_fds = True)

+                logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid ))

+                while self.session.poll() is None:  # subprocess is NOT finished

+                    time.sleep(0.5)

+                    if self.stoprequest.is_set():

+                        logger.debug("Abort request received by handling thread. Terminating TRex session." )

+                        os.killpg(self.session.pid, signal.SIGUSR1)

+                        self.trexObj.set_status(TRexStatus.Idle)

+                        self.trexObj.set_verbose_status("TRex is Idle")

+                        break

+        except Exception as e:

+            logger.error(e)

+

+        self.time_stamps['run_time'] = time.time() - self.time_stamps['start']

+

+        try:

+            if self.time_stamps['run_time'] < 5:

+                logger.error("TRex run failed due to wrong input parameters, or due to readability issues.")

+                self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format(

+                    cmd = self.cmd, output = self.load_trex_output(self.export_path)))

+                self.trexObj.errcode = -11

+            elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ):

+                if (self.session.returncode is not None and self.session.returncode != 0):

+                    logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) )

+                elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()):

+                    logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) )

+

+                logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS")

+                self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format(

+                    output = self.load_trex_output(self.export_path)))

+                self.trexObj.errcode = -15

+            else:

+                logger.info("TRex run session finished.")

+                self.trexObj.set_verbose_status('TRex finished.')

+                self.trexObj.errcode = None

+

+        finally:

+            self.trexObj.set_status(TRexStatus.Idle)

+            logger.info("TRex running state changed to 'Idle'.")

+            self.trexObj.expect_trex.clear()

+            logger.debug("Finished handling a single run of TRex.")

+            self.trexObj.zmq_dump   = None

 

     def join (self, timeout = None):

         self.stoprequest.set()

diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py
index 8f7e99f0..9fe7d70b 100755
--- a/scripts/automation/trex_control_plane/server/trex_server.py
+++ b/scripts/automation/trex_control_plane/server/trex_server.py
@@ -310,7 +310,7 @@ class CTRexServer(object):
                 return False
 
             
-    def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False):
+    def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False, debug_image = False, trex_args = ''):
         with self.start_lock:
             logger.info("Processing start_trex() command.")
             if self.is_reserved():
@@ -323,7 +323,7 @@ class CTRexServer(object):
                 return Fault(-13, '')  # raise at client TRexInUseError
             
             try:
-                server_cmd_data = self.generate_run_cmd(stateless = stateless, **trex_cmd_options)
+                server_cmd_data = self.generate_run_cmd(stateless = stateless, debug_image = debug_image, trex_args = trex_args, **trex_cmd_options)
                 self.zmq_monitor.first_dump = True
                 self.trex.start_trex(self.TREX_PATH, server_cmd_data)
                 logger.info("TRex session has been successfully initiated.")
@@ -387,23 +387,14 @@ class CTRexServer(object):
         return trex_cmds_list
 
 
-    def kill_all_trexes(self):
+    # Silently tries to kill TRexes with given signal.
+    # Responsibility of client to verify with get_trex_cmds.
+    def kill_all_trexes(self, signal_name):
         logger.info('Processing kill_all_trexes() command.')
         trex_cmds_list = self.get_trex_cmds()
-        if not trex_cmds_list:
-            return False
         for pid, cmd in trex_cmds_list:
-            logger.info('Killing process %s %s' % (pid, cmd))
-            run_command('kill %s' % pid)
-            ret_code_ps, _, _ = run_command('ps -p %s' % pid)
-            if not ret_code_ps:
-                logger.info('Killing with -9.')
-                run_command('kill -9 %s' % pid)
-                ret_code_ps, _, _ = run_command('ps -p %s' % pid)
-                if not ret_code_ps:
-                    logger.info('Could not kill process.')
-                    raise Exception('Could not kill process %s %s' % (pid, cmd))
-        return True
+            logger.info('Killing with signal %s process %s %s' % (signal_name, pid, cmd))
+            os.kill(int(pid), signal_name)
 
 
     def wait_until_kickoff_finish (self, timeout = 40):
@@ -422,7 +413,7 @@ class CTRexServer(object):
         return self.trex.get_running_info()
 
 
-    def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, **kwargs):
+    def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, debug_image = False, trex_args = '', **kwargs):
         """ generate_run_cmd(self, iom, export_path, kwargs) -> str
 
         Generates a custom running command for the kick-off of the TRex traffic generator.
@@ -459,6 +450,8 @@ class CTRexServer(object):
                 continue
             else:
                 trex_cmd_options += (dash + '{k} {val}'.format( k = tmp_key, val =  value ))
+        if trex_args:
+            trex_cmd_options += ' %s' % trex_args
 
         if not stateless:
             if 'f' not in kwargs:
@@ -466,12 +459,12 @@ class CTRexServer(object):
             if 'd' not in kwargs:
                 raise Exception('Argument -d should be specified in stateful command')
 
-        cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key > {export}".format( # -- iom 0 disables the periodic log to the screen (not needed)
+        cmd = "{nice}{run_command}{debug_image} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed)
             nice = '' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice,
             run_command = self.TREX_START_CMD,
+            debug_image = '-debug' if debug_image else '',
             cmd_options = trex_cmd_options,
-            io          = iom, 
-            export = export_path )
+            io          = iom)
 
         logger.info("TREX FULL COMMAND: {command}".format(command = cmd) )