diff options
| -rwxr-xr-x | scripts/automation/regression/stateful_tests/trex_general_test.py | 11 | ||||
| -rwxr-xr-x | scripts/automation/regression/trex_unit_test.py | 11 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py | 12 | ||||
| -rwxr-xr-x | scripts/master_daemon.py | 9 | ||||
| -rwxr-xr-x | scripts/trex_daemon_server | 56 |
5 files changed, 69 insertions, 30 deletions
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py index 38e97597..33b954fc 100755 --- a/scripts/automation/regression/stateful_tests/trex_general_test.py +++ b/scripts/automation/regression/stateful_tests/trex_general_test.py @@ -274,9 +274,16 @@ class CTRexGeneral_Test(unittest.TestCase): # raise def unzip_client_package(self): + client_pkg_files = glob('%s/trex_client*.tar.gz' % CTRexScenario.scripts_path) + if not len(client_pkg_files): + raise Exception('Could not find client package') + if len(client_pkg_files) > 1: + raise Exception('Found more than one client packages') if not os.path.exists('%s/trex_client' % CTRexScenario.scripts_path): print('\nUnzipping package') - CTRexScenario.trex.master_daemon.unpack_client() + return_code, _, stderr = misc_methods.run_command("tar -xzf %s -C %s" % (client_pkg_files[0], CTRexScenario.scripts_path)) + if return_code: + raise Exception('Could not untar the client package: %s' % stderr) else: print('\nClient package is untarred') @@ -304,7 +311,7 @@ class CTRexGeneral_Test(unittest.TestCase): self.trex.force_kill(confirm = False) if not self.is_loopback: print('') - if self.trex: # stateful + if not self.stl_trex: # stateful self.router.load_clean_config() self.router.clear_counters() self.router.clear_packet_drop_stats() diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py index d15762bf..f2cf38cb 100755 --- a/scripts/automation/regression/trex_unit_test.py +++ b/scripts/automation/regression/trex_unit_test.py @@ -166,6 +166,10 @@ class CTRexTestConfiguringPlugin(Plugin): CTRexScenario.server_logs = self.server_logs CTRexScenario.trex = CTRexClient(trex_host = self.configuration.trex['trex_name'], verbose = self.json_verbose) + if not CTRexScenario.trex.check_master_connectivity(): + print('Could not connect to master daemon') + sys.exit(-1) + CTRexScenario.scripts_path = CTRexScenario.trex.get_trex_path() if options.ga and CTRexScenario.setup_name: CTRexScenario.GAManager = GAmanager(GoogleID = 'UA-75220362-4', UserID = CTRexScenario.setup_name, @@ -179,24 +183,21 @@ class CTRexTestConfiguringPlugin(Plugin): def begin (self): if self.pkg and self.kill_running and not CTRexScenario.is_copied: - if not CTRexScenario.trex.check_master_connectivity(): - print('Could not connect to master daemon') - sys.exit(-1) print('Updating TRex to %s' % self.pkg) if not CTRexScenario.trex.master_daemon.update_trex(self.pkg): print('Failed updating TRex') sys.exit(-1) else: print('Updated') - CTRexScenario.scripts_path = '/tmp/trex-scripts' CTRexScenario.is_copied = True if self.functional or self.collect_only: return + print('Restarting TRex daemon server') res = CTRexScenario.trex.restart_trex_daemon() if not res: print('Could not restart TRex daemon server') sys.exit(-1) - # launch TRex daemon on relevant setup + trex_cmds = CTRexScenario.trex.get_trex_cmds() if trex_cmds: if self.kill_running: diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py index 09ee52ef..0aac043e 100755 --- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py @@ -354,6 +354,18 @@ class CTRexClient(object): self.prompt_verbose_data() + def get_trex_path(self): + ''' + Returns TRex path on server + ''' + try: + return self.master_daemon.get_trex_path() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + def wait_until_kickoff_finish(self, timeout = 40): """ Block the client application until TRex changes state from 'Starting' to either 'Idle' or 'Running' diff --git a/scripts/master_daemon.py b/scripts/master_daemon.py index b2e61f02..9214cfc8 100755 --- a/scripts/master_daemon.py +++ b/scripts/master_daemon.py @@ -24,6 +24,9 @@ def check_connectivity(): def add(a, b): # for sanity checks return a + b +def get_trex_path(): + return args.trex_dir + def is_trex_daemon_running(): ret_code, stdout, stderr = run_command('ps -u root --format comm') if ret_code: @@ -43,7 +46,7 @@ def stop_trex_daemon(): return False return_code, stdout, stderr = run_command('%s stop' % trex_daemon_path) if return_code: - raise Exception('Could not stop trex_daemon_server, err: %s' % stderr) + raise Exception('Could not stop trex_daemon_server, %s' % [return_code, stdout, stderr]) for i in range(50): if not is_trex_daemon_running(): return True @@ -134,7 +137,7 @@ def run_command(command, timeout = 15): command = 'timeout %s %s' % (timeout, command) # pipes might stuck, even with timeout with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: - proc = subprocess.Popen(shlex.split(command), stdout=stdout_file, stderr=stderr_file) + proc = subprocess.Popen(shlex.split(command), stdout = stdout_file, stderr = stderr_file, cwd = args.trex_dir) proc.wait() stdout_file.seek(0) stderr_file.seek(0) @@ -170,6 +173,7 @@ def start_master_daemon_func(): print('Started master daemon (port %s)' % args.daemon_port) server.register_function(add) server.register_function(check_connectivity) + server.register_function(get_trex_path) server.register_function(is_trex_daemon_running) server.register_function(restart_trex_daemon) server.register_function(start_trex_daemon) @@ -264,6 +268,7 @@ if os.path.isfile(args.trex_dir): raise Exception('Given path is a file') if not os.path.exists(args.trex_dir): os.makedirs(args.trex_dir) +os.chmod(args.trex_dir, 0o777) if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) diff --git a/scripts/trex_daemon_server b/scripts/trex_daemon_server index 6d08e458..b21a2626 100755 --- a/scripts/trex_daemon_server +++ b/scripts/trex_daemon_server @@ -1,6 +1,7 @@ #!/usr/bin/python import os, sys, getpass +import tempfile from time import time, sleep import subprocess, shlex, multiprocessing from argparse import ArgumentParser @@ -27,25 +28,36 @@ except ImportError: def run_command(command, timeout = 10): commmand = 'timeout %s %s' % (timeout, command) - proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd = os.getcwd()) - stdout, stderr = proc.communicate() - return (proc.returncode, stdout.decode(errors = 'replace'), stderr.decode(errors = 'replace')) + # pipes might stuck, even with timeout + with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: + proc = subprocess.Popen(shlex.split(command), stdout = stdout_file, stderr = stderr_file, cwd = daemon_dir) + proc.wait() + stdout_file.seek(0) + stderr_file.seek(0) + return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')) def get_daemon_pid(): - return_code, stdout, stderr = run_command('netstat -tlnp') - if return_code: - fail('Failed to determine which program holds port %s, netstat error: %s' % (args.daemon_port, stderr)) - for line in stdout.splitlines(): - if '0.0.0.0:%s' % args.daemon_port in line: - line_arr = line.split() - if '/' not in line_arr[-1]: - fail('Expecting pid/program name in netstat line of using port %s, got: %s' % (args.daemon_port, line_arr[-1])) - pid, program = line_arr[-1].split('/') - if 'python' not in program and 'trex_server' not in program and 'trex_daemon_server' not in program: - fail('Some other program holds port %s, not our daemon: %s. Please verify.' % (args.daemon_port, program)) - return int(pid) - return None + err = None + for i in range(5): + try: + return_code, stdout, stderr = run_command('netstat -tlnp') + if return_code: + raise Exception('Failed to run netstat.\nStdout: %s\nStderr: %s' % (stdout, stderr)) + for line in stdout.splitlines(): + if '0.0.0.0:%s' % args.daemon_port in line: + line_arr = line.split() + if '/' not in line_arr[-1]: + raise Exception('Expecting pid/program name in netstat line of using port %s, got: %s' % (args.daemon_port, line)) + pid, program = line_arr[-1].split('/') + if 'python' not in program and 'trex_server' not in program and 'trex_daemon_server' not in program: + raise Exception('Some other program holds port %s, not our daemon: %s. Please verify.' % (args.daemon_port, program)) + return int(pid) + return None + except Exception as e: + err = e + sleep(0.1) + fail('Could not determine daemon pid, err: %s' % err) def show_daemon_status(): @@ -92,22 +104,23 @@ def kill_daemon(): print(termstyle.red('TRex server daemon is NOT running')) return True return_code, stdout, stderr = run_command('kill %s' % pid) # usual kill - if return_code: - fail('Failed to kill trex_daemon, error: %s' % stderr) + #if return_code: + # fail('Failed to kill trex_daemon, error: %s' % stderr) for i in range(50): if not get_daemon_pid(): print(termstyle.green('TRex server daemon is killed')) return True sleep(0.1) return_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill - if return_code: - fail('Failed to kill trex_daemon, error: %s' % stderr) + #if return_code: + # fail('Failed to kill trex_daemon, error: %s' % stderr) for i in range(50): if not get_daemon_pid(): print(termstyle.green('TRex server daemon is killed')) return True sleep(0.1) - fail('Failed to kill trex_daemon, even with -9. Please review manually.') # should not happen + fail('Failed to kill trex_daemon, even with -9. Please review manually.\n' \ + 'Return code: %s\nStdout: %s\nStderr: %s' % return_code, stdout, stderr) # should not happen ### Main ### @@ -129,6 +142,7 @@ trex_server.trex_parser.add_argument('action', choices=action_funcs.keys(), trex_server.trex_parser.usage = None args = trex_server.trex_parser.parse_args() +daemon_dir = os.path.dirname(os.path.realpath(__file__)) mkpath('/var/log/trex') mkpath('/var/run/trex') |