From a78ab130e5e344989e2de9c62e061180416bae12 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Thu, 9 Jun 2016 19:42:50 +0300 Subject: fix TRex output not redirected --- .../server/trex_launch_thread.py | 88 +++++++++++----------- .../trex_control_plane/server/trex_server.py | 5 +- 2 files changed, 47 insertions(+), 46 deletions(-) (limited to 'scripts/automation/trex_control_plane/server') diff --git a/scripts/automation/trex_control_plane/server/trex_launch_thread.py b/scripts/automation/trex_control_plane/server/trex_launch_thread.py index 82a7f996..22606753 100755 --- a/scripts/automation/trex_control_plane/server/trex_launch_thread.py +++ b/scripts/automation/trex_control_plane/server/trex_launch_thread.py @@ -30,49 +30,51 @@ class AsynchronousTRexSession(threading.Thread): self.trexObj.zmq_dump = {} def run (self): - - with open(os.devnull, 'w') as DEVNULL: - self.time_stamps['start'] = self.time_stamps['run_time'] = time.time() - self.session = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdin = DEVNULL, stderr = subprocess.PIPE, preexec_fn=os.setsid, close_fds = True) - logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid )) - while self.session.poll() is None: # subprocess is NOT finished - time.sleep(0.5) - if self.stoprequest.is_set(): - logger.debug("Abort request received by handling thread. Terminating TRex session." ) - os.killpg(self.session.pid, signal.SIGUSR1) - self.trexObj.set_status(TRexStatus.Idle) - self.trexObj.set_verbose_status("TRex is Idle") - break - - self.time_stamps['run_time'] = time.time() - self.time_stamps['start'] - - try: - if self.time_stamps['run_time'] < 5: - logger.error("TRex run failed due to wrong input parameters, or due to readability issues.") - self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format( - cmd = self.cmd, output = self.load_trex_output(self.export_path))) - self.trexObj.errcode = -11 - elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ): - if (self.session.returncode is not None and self.session.returncode != 0): - logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) ) - elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()): - logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) ) - - logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS") - self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format( - output = self.load_trex_output(self.export_path))) - self.trexObj.errcode = -15 - else: - logger.info("TRex run session finished.") - self.trexObj.set_verbose_status('TRex finished.') - self.trexObj.errcode = None - - finally: - self.trexObj.set_status(TRexStatus.Idle) - logger.info("TRex running state changed to 'Idle'.") - self.trexObj.expect_trex.clear() - logger.debug("Finished handling a single run of TRex.") - self.trexObj.zmq_dump = None + try: + with open(self.export_path, 'w') as output_file: + self.time_stamps['start'] = self.time_stamps['run_time'] = time.time() + self.session = subprocess.Popen(shlex.split(self.cmd), cwd = self.launch_path, stdout = output_file, preexec_fn=os.setsid, close_fds = True) + logger.info("TRex session initialized successfully, Parent process pid is {pid}.".format( pid = self.session.pid )) + while self.session.poll() is None: # subprocess is NOT finished + time.sleep(0.5) + if self.stoprequest.is_set(): + logger.debug("Abort request received by handling thread. Terminating TRex session." ) + os.killpg(self.session.pid, signal.SIGUSR1) + self.trexObj.set_status(TRexStatus.Idle) + self.trexObj.set_verbose_status("TRex is Idle") + break + except Exception as e: + logger.error(e) + + self.time_stamps['run_time'] = time.time() - self.time_stamps['start'] + + try: + if self.time_stamps['run_time'] < 5: + logger.error("TRex run failed due to wrong input parameters, or due to readability issues.") + self.trexObj.set_verbose_status("TRex run failed due to wrong input parameters, or due to readability issues.\n\nTRex command: {cmd}\n\nRun output:\n{output}".format( + cmd = self.cmd, output = self.load_trex_output(self.export_path))) + self.trexObj.errcode = -11 + elif (self.session.returncode is not None and self.session.returncode != 0) or ( (self.time_stamps['run_time'] < self.duration) and (not self.stoprequest.is_set()) ): + if (self.session.returncode is not None and self.session.returncode != 0): + logger.debug("Failed TRex run due to session return code ({ret_code})".format( ret_code = self.session.returncode ) ) + elif ( (self.time_stamps['run_time'] < self.duration) and not self.stoprequest.is_set()): + logger.debug("Failed TRex run due to running time ({runtime}) combined with no-stopping request.".format( runtime = self.time_stamps['run_time'] ) ) + + logger.warning("TRex run was terminated unexpectedly by outer process or by the hosting OS") + self.trexObj.set_verbose_status("TRex run was terminated unexpectedly by outer process or by the hosting OS.\n\nRun output:\n{output}".format( + output = self.load_trex_output(self.export_path))) + self.trexObj.errcode = -15 + else: + logger.info("TRex run session finished.") + self.trexObj.set_verbose_status('TRex finished.') + self.trexObj.errcode = None + + finally: + self.trexObj.set_status(TRexStatus.Idle) + logger.info("TRex running state changed to 'Idle'.") + self.trexObj.expect_trex.clear() + logger.debug("Finished handling a single run of TRex.") + self.trexObj.zmq_dump = None def join (self, timeout = None): self.stoprequest.set() diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py index 8f7e99f0..5e278a74 100755 --- a/scripts/automation/trex_control_plane/server/trex_server.py +++ b/scripts/automation/trex_control_plane/server/trex_server.py @@ -466,12 +466,11 @@ class CTRexServer(object): if 'd' not in kwargs: raise Exception('Argument -d should be specified in stateful command') - cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key > {export}".format( # -- iom 0 disables the periodic log to the screen (not needed) + cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed) nice = '' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice, run_command = self.TREX_START_CMD, cmd_options = trex_cmd_options, - io = iom, - export = export_path ) + io = iom) logger.info("TREX FULL COMMAND: {command}".format(command = cmd) ) -- cgit From 3207f14a91cc01b44b28ba541e5968f58a7e5ec2 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Fri, 17 Jun 2016 15:39:00 +0300 Subject: add ctrl+c as first try to kill daemons/TRex. master daemon change directory to "/" to deal with case it's directory was deleted and update_trex() was used. remove check connection to daemons as part of init of client (can be done manually per needed daemon) --- .../trex_control_plane/server/singleton_daemon.py | 31 ++++++++++------------ .../trex_control_plane/server/trex_server.py | 19 ++++--------- 2 files changed, 19 insertions(+), 31 deletions(-) (limited to 'scripts/automation/trex_control_plane/server') diff --git a/scripts/automation/trex_control_plane/server/singleton_daemon.py b/scripts/automation/trex_control_plane/server/singleton_daemon.py index 8fdedc6e..79deace1 100755 --- a/scripts/automation/trex_control_plane/server/singleton_daemon.py +++ b/scripts/automation/trex_control_plane/server/singleton_daemon.py @@ -2,6 +2,7 @@ import errno import os import shlex import socket +import signal import tempfile import types from subprocess import Popen @@ -73,31 +74,27 @@ class SingletonDaemon(object): if pid: return pid - - # kill daemon - def kill(self, timeout = 10): - pid = self.get_pid() - if not pid: - return False - ret_code, stdout, stderr = run_command('kill %s' % pid) # usual kill - if ret_code: - raise Exception('Failed to run kill command for %s: %s' % (self.name, [ret_code, stdout, stderr])) + def kill_by_signal(self, pid, signal_name, timeout): + os.kill(pid, signal_name) poll_rate = 0.1 for i in range(int(timeout / poll_rate)): if not self.is_running(): return True sleep(poll_rate) - ret_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill - if ret_code: - raise Exception('Failed to run kill -9 command for %s: %s' % (self.name, [ret_code, stdout, stderr])) - for i in range(int(timeout / poll_rate)): - if not self.is_running(): + + # kill daemon, with verification + def kill(self, timeout = 15): + pid = self.get_pid() + if not pid: + return False + # try Ctrl+C, usual kill, kill -9 + for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]: + if self.kill_by_signal(pid, signal_name, timeout): return True - sleep(poll_rate) raise Exception('Could not kill %s, even with -9' % self.name) # try connection as RPC client, return True upon success, False if fail - def check_connectivity(self, timeout = 5): + def check_connectivity(self, timeout = 15): daemon = jsonrpclib.Server('http://127.0.0.1:%s/' % self.port) poll_rate = 0.1 for i in range(int(timeout/poll_rate)): @@ -140,7 +137,7 @@ class SingletonDaemon(object): raise Exception('%s failed to run.' % self.name) # restart the daemon - def restart(self, timeout = 5): + def restart(self, timeout = 15): if self.is_running(): self.kill(timeout) return self.start(timeout) diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py index 5e278a74..3da629ec 100755 --- a/scripts/automation/trex_control_plane/server/trex_server.py +++ b/scripts/automation/trex_control_plane/server/trex_server.py @@ -387,23 +387,14 @@ class CTRexServer(object): return trex_cmds_list - def kill_all_trexes(self): + # Silently tries to kill TRexes with given signal. + # Responsibility of client to verify with get_trex_cmds. + def kill_all_trexes(self, signal_name): logger.info('Processing kill_all_trexes() command.') trex_cmds_list = self.get_trex_cmds() - if not trex_cmds_list: - return False for pid, cmd in trex_cmds_list: - logger.info('Killing process %s %s' % (pid, cmd)) - run_command('kill %s' % pid) - ret_code_ps, _, _ = run_command('ps -p %s' % pid) - if not ret_code_ps: - logger.info('Killing with -9.') - run_command('kill -9 %s' % pid) - ret_code_ps, _, _ = run_command('ps -p %s' % pid) - if not ret_code_ps: - logger.info('Could not kill process.') - raise Exception('Could not kill process %s %s' % (pid, cmd)) - return True + logger.info('Killing with signal %s process %s %s' % (signal_name, pid, cmd)) + os.kill(int(pid), signal_name) def wait_until_kickoff_finish (self, timeout = 40): -- cgit From d93a7ff83d3d29fc6c254dd56af235dbe7abacb3 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Fri, 17 Jun 2016 18:47:56 +0300 Subject: master_daemon: show better status of executed command --- scripts/automation/trex_control_plane/server/singleton_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/automation/trex_control_plane/server') diff --git a/scripts/automation/trex_control_plane/server/singleton_daemon.py b/scripts/automation/trex_control_plane/server/singleton_daemon.py index 79deace1..1784cc42 100755 --- a/scripts/automation/trex_control_plane/server/singleton_daemon.py +++ b/scripts/automation/trex_control_plane/server/singleton_daemon.py @@ -86,7 +86,7 @@ class SingletonDaemon(object): def kill(self, timeout = 15): pid = self.get_pid() if not pid: - return False + raise Exception('%s is not running' % self.name) # try Ctrl+C, usual kill, kill -9 for signal_name in [signal.SIGINT, signal.SIGTERM, signal.SIGKILL]: if self.kill_by_signal(pid, signal_name, timeout): -- cgit From e058f6e034fbf51bc5e29d449ea806e7a8633b70 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Mon, 20 Jun 2016 13:47:18 +0300 Subject: regression: add flag to run with debug image --- scripts/automation/trex_control_plane/server/trex_server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'scripts/automation/trex_control_plane/server') diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py index 3da629ec..091b729b 100755 --- a/scripts/automation/trex_control_plane/server/trex_server.py +++ b/scripts/automation/trex_control_plane/server/trex_server.py @@ -310,7 +310,7 @@ class CTRexServer(object): return False - def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False): + def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False, debug_image = False): with self.start_lock: logger.info("Processing start_trex() command.") if self.is_reserved(): @@ -323,7 +323,7 @@ class CTRexServer(object): return Fault(-13, '') # raise at client TRexInUseError try: - server_cmd_data = self.generate_run_cmd(stateless = stateless, **trex_cmd_options) + server_cmd_data = self.generate_run_cmd(stateless = stateless, debug_image = debug_image, **trex_cmd_options) self.zmq_monitor.first_dump = True self.trex.start_trex(self.TREX_PATH, server_cmd_data) logger.info("TRex session has been successfully initiated.") @@ -413,7 +413,7 @@ class CTRexServer(object): return self.trex.get_running_info() - def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, **kwargs): + def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, debug_image = False, **kwargs): """ generate_run_cmd(self, iom, export_path, kwargs) -> str Generates a custom running command for the kick-off of the TRex traffic generator. @@ -457,9 +457,10 @@ class CTRexServer(object): if 'd' not in kwargs: raise Exception('Argument -d should be specified in stateful command') - cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed) + cmd = "{nice}{run_command}{debug_image} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed) nice = '' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice, run_command = self.TREX_START_CMD, + debug_image = '-debug' if debug_image else '', cmd_options = trex_cmd_options, io = iom) -- cgit From ddbed4330eee8b4c40d8ca83535dbf5ac69bb81f Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Tue, 21 Jun 2016 18:17:55 +0300 Subject: api: add passing of additional arguments to TRex --- scripts/automation/trex_control_plane/server/trex_server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'scripts/automation/trex_control_plane/server') diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py index 091b729b..9fe7d70b 100755 --- a/scripts/automation/trex_control_plane/server/trex_server.py +++ b/scripts/automation/trex_control_plane/server/trex_server.py @@ -310,7 +310,7 @@ class CTRexServer(object): return False - def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False, debug_image = False): + def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False, debug_image = False, trex_args = ''): with self.start_lock: logger.info("Processing start_trex() command.") if self.is_reserved(): @@ -323,7 +323,7 @@ class CTRexServer(object): return Fault(-13, '') # raise at client TRexInUseError try: - server_cmd_data = self.generate_run_cmd(stateless = stateless, debug_image = debug_image, **trex_cmd_options) + server_cmd_data = self.generate_run_cmd(stateless = stateless, debug_image = debug_image, trex_args = trex_args, **trex_cmd_options) self.zmq_monitor.first_dump = True self.trex.start_trex(self.TREX_PATH, server_cmd_data) logger.info("TRex session has been successfully initiated.") @@ -413,7 +413,7 @@ class CTRexServer(object): return self.trex.get_running_info() - def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, debug_image = False, **kwargs): + def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, debug_image = False, trex_args = '', **kwargs): """ generate_run_cmd(self, iom, export_path, kwargs) -> str Generates a custom running command for the kick-off of the TRex traffic generator. @@ -450,6 +450,8 @@ class CTRexServer(object): continue else: trex_cmd_options += (dash + '{k} {val}'.format( k = tmp_key, val = value )) + if trex_args: + trex_cmd_options += ' %s' % trex_args if not stateless: if 'f' not in kwargs: -- cgit From d04442ab671f768a1b645fb887d4a9cd575c7852 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Thu, 23 Jun 2016 09:56:30 +0300 Subject: daemons logger: create log dir if does not exist --- scripts/automation/trex_control_plane/server/CCustomLogger.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'scripts/automation/trex_control_plane/server') diff --git a/scripts/automation/trex_control_plane/server/CCustomLogger.py b/scripts/automation/trex_control_plane/server/CCustomLogger.py index a8823cea..6d3974a6 100755 --- a/scripts/automation/trex_control_plane/server/CCustomLogger.py +++ b/scripts/automation/trex_control_plane/server/CCustomLogger.py @@ -3,15 +3,13 @@ import sys import os import logging +def prepare_dir(log_path): + log_dir = os.path.dirname(log_path) + if not os.path.exists(log_dir): + os.makedirs(log_dir) def setup_custom_logger(name, log_path = None): # first make sure path availabe -# if log_path is None: -# log_path = os.getcwd()+'/trex_log.log' -# else: -# directory = os.path.dirname(log_path) -# if not os.path.exists(directory): -# os.makedirs(directory) logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', datefmt = '%m-%d %H:%M') @@ -31,6 +29,7 @@ def setup_custom_logger(name, log_path = None): def setup_daemon_logger (name, log_path = None): # first make sure path availabe + prepare_dir(log_path) try: os.unlink(log_path) except: -- cgit