diff options
| -rwxr-xr-x | scripts/automation/regression/stateful_tests/trex_general_test.py | 9 | ||||
| -rwxr-xr-x | scripts/automation/regression/trex_unit_test.py | 110 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/server/CCustomLogger.py | 4 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/server/extended_daemon_runner.py | 146 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/server/trex_daemon_server | 25 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/server/trex_daemon_server.py | 79 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/server/trex_server.py | 137 | ||||
| -rwxr-xr-x | scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py | 224 | ||||
| l---------[-rwxr-xr-x] | scripts/daemon_server | 28 | ||||
| -rwxr-xr-x | scripts/master_daemon.py | 281 | ||||
| -rwxr-xr-x | scripts/trex_daemon_server | 158 |
11 files changed, 774 insertions, 427 deletions
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py index 4b03321c..ac2d7b89 100755 --- a/scripts/automation/regression/stateful_tests/trex_general_test.py +++ b/scripts/automation/regression/stateful_tests/trex_general_test.py @@ -56,11 +56,10 @@ class CTRexGeneral_Test(unittest.TestCase): self.configuration = CTRexScenario.configuration self.benchmark = CTRexScenario.benchmark self.trex = CTRexScenario.trex + self.stl_trex = CTRexScenario.stl_trex self.trex_crashed = CTRexScenario.trex_crashed self.modes = CTRexScenario.modes - #self.GAManager = CTRexScenario.GAManager - self.GAManager = None # remove GA for now - #CTRexScenario.GAManager + self.GAManager = CTRexScenario.GAManager self.skipping = False self.fail_reasons = [] if not hasattr(self, 'unsupported_modes'): @@ -308,7 +307,7 @@ class CTRexGeneral_Test(unittest.TestCase): test_setup_modes_conflict = self.modes & set(self.unsupported_modes) if test_setup_modes_conflict: self.skip("The test can't run with following modes of given setup: %s " % test_setup_modes_conflict) - if self.trex and not self.trex.is_idle(): + if not self.stl_trex and not self.trex.is_idle(): print('Warning: TRex is not idle at setUp, trying to stop it.') self.trex.force_kill(confirm = False) if not self.is_loopback: @@ -327,7 +326,7 @@ class CTRexGeneral_Test(unittest.TestCase): # def test_isInitialized(self): # assert CTRexScenario.is_init == True def tearDown(self): - if self.trex and not self.trex.is_idle(): + if not self.stl_trex and not self.trex.is_idle(): print('Warning: TRex is not idle at tearDown, trying to stop it.') self.trex.force_kill(confirm = False) if not self.skipping: diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py index b9f0c05c..d15762bf 100755 --- a/scripts/automation/regression/trex_unit_test.py +++ b/scripts/automation/regression/trex_unit_test.py @@ -46,6 +46,7 @@ from pprint import pprint import subprocess import re import time +from distutils.dir_util import mkpath def check_trex_path(trex_path): if os.path.isfile('%s/trex_daemon_server' % trex_path): @@ -64,35 +65,6 @@ def get_trex_path(): raise Exception('Could not determine trex_under_test folder, try setting env.var. TREX_UNDER_TEST') return latest_build_path -STATEFUL_STOP_COMMAND = './trex_daemon_server stop; sleep 1; ./trex_daemon_server stop; sleep 1' -STATEFUL_RUN_COMMAND = 'rm /var/log/trex/trex_daemon_server.log; ./trex_daemon_server start; sleep 2; ./trex_daemon_server show' -TREX_FILES = ('_t-rex-64', '_t-rex-64-o', '_t-rex-64-debug', '_t-rex-64-debug-o') - -def trex_remote_command(trex_data, command, background = False, from_scripts = True, timeout = 20): - if from_scripts: - return misc_methods.run_remote_command(trex_data['trex_name'], ('cd %s; ' % CTRexScenario.scripts_path)+ command, background, timeout) - return misc_methods.run_remote_command(trex_data['trex_name'], command, background, timeout) - -# 1 = running, 0 - not running -def check_trex_running(trex_data): - commands = [] - for filename in TREX_FILES: - commands.append('ps -C %s > /dev/null' % filename) - return_code, _, _ = trex_remote_command(trex_data, ' || '.join(commands), from_scripts = False) - return not return_code - -def kill_trex_process(trex_data): - return_code, stdout, _ = trex_remote_command(trex_data, 'ps -u root --format comm,pid,cmd | grep _t-rex-64 | grep -v grep || true', from_scripts = False) - assert return_code == 0, 'last remote command failed' - if stdout: - for process in stdout.split('\n'): - try: - proc_name, pid, full_cmd = re.split('\s+', process, maxsplit=2) - if proc_name.find('t-rex-64') >= 0: - print('Killing remote process: %s' % full_cmd) - trex_remote_command(trex_data, 'kill %s' % pid, from_scripts = False) - except: - continue def address_to_ip(address): for i in range(10): @@ -142,9 +114,6 @@ class CTRexTestConfiguringPlugin(Plugin): parser.add_option('--pkg', action="store", dest="pkg", help="Run with given TRex package. Make sure the path available at server machine.") - parser.add_option('--no-ssh', '--no_ssh', action="store_true", default = False, - dest="no_ssh", - help="Flag to disable any ssh to server machine.") parser.add_option('--collect', action="store_true", default = False, dest="collect", help="Alias to --collect-only.") @@ -157,6 +126,9 @@ class CTRexTestConfiguringPlugin(Plugin): parser.add_option('--long', action="store_true", default = False, dest="long", help="Flag of long tests (stability).") + parser.add_option('--ga', action="store_true", default = False, + dest="ga", + help="Flag to send benchmarks to GA.") def configure(self, options, conf): self.collect_only = options.collect_only @@ -166,7 +138,6 @@ class CTRexTestConfiguringPlugin(Plugin): self.stateless = options.stateless self.stateful = options.stateful self.pkg = options.pkg - self.no_ssh = options.no_ssh self.json_verbose = options.json_verbose self.telnet_verbose = options.telnet_verbose if self.functional and (not self.pkg or self.no_ssh): @@ -193,43 +164,52 @@ class CTRexTestConfiguringPlugin(Plugin): CTRexScenario.benchmark = self.benchmark CTRexScenario.modes = set(self.modes) CTRexScenario.server_logs = self.server_logs + CTRexScenario.trex = CTRexClient(trex_host = self.configuration.trex['trex_name'], + verbose = self.json_verbose) + if options.ga and CTRexScenario.setup_name: + CTRexScenario.GAManager = GAmanager(GoogleID = 'UA-75220362-4', + UserID = CTRexScenario.setup_name, + QueueSize = 100, + Timeout = 5, # seconds + UserPermission = 1, + BlockingMode = 1, + appName = 'TRex', + appVer = '1.11.232') def begin (self): - if self.pkg and not CTRexScenario.is_copied and not self.no_ssh: - new_path = '/tmp/trex-scripts' - rsync_template = 'rm -rf /tmp/trex-scripts; mkdir -p %s; rsync -Lc %s /tmp; tar -mxzf /tmp/%s -C %s; mv %s/v*.*/* %s' - rsync_command = rsync_template % (new_path, self.pkg, os.path.basename(self.pkg), new_path, new_path, new_path) - return_code, stdout, stderr = trex_remote_command(self.configuration.trex, rsync_command, from_scripts = False, timeout = 300) - if return_code: - print('Failed copying') + if self.pkg and self.kill_running and not CTRexScenario.is_copied: + if not CTRexScenario.trex.check_master_connectivity(): + print('Could not connect to master daemon') sys.exit(-1) - CTRexScenario.scripts_path = new_path + print('Updating TRex to %s' % self.pkg) + if not CTRexScenario.trex.master_daemon.update_trex(self.pkg): + print('Failed updating TRex') + sys.exit(-1) + else: + print('Updated') + CTRexScenario.scripts_path = '/tmp/trex-scripts' CTRexScenario.is_copied = True if self.functional or self.collect_only: return + res = CTRexScenario.trex.restart_trex_daemon() + if not res: + print('Could not restart TRex daemon server') + sys.exit(-1) # launch TRex daemon on relevant setup - if not self.no_ssh: + trex_cmds = CTRexScenario.trex.get_trex_cmds() + if trex_cmds: if self.kill_running: - if self.stateful: - trex_remote_command(self.configuration.trex, STATEFUL_STOP_COMMAND) - kill_trex_process(self.configuration.trex) - time.sleep(1) - elif check_trex_running(self.configuration.trex): + CTRexScenario.trex.kill_all_trexes() + else: print('TRex is already running') sys.exit(-1) - - if self.stateful: - if not self.no_ssh: - trex_remote_command(self.configuration.trex, STATEFUL_RUN_COMMAND) - CTRexScenario.trex = CTRexClient(trex_host = self.configuration.trex['trex_name'], verbose = self.json_verbose) - elif self.stateless: - if not self.no_ssh: - cores = self.configuration.trex.get('trex_cores', 1) - if 'virt_nics' in self.modes and cores > 1: - raise Exception('Number of cores should be 1 with virtual NICs') - trex_remote_command(self.configuration.trex, './t-rex-64 -i -c %s' % cores, background = True) + if self.stateless: + cores = self.configuration.trex.get('trex_cores', 1) + if 'virt_nics' in self.modes and cores > 1: + raise Exception('Number of cores should be 1 with virtual NICs') + CTRexScenario.trex.start_stateless(c = cores) CTRexScenario.stl_trex = STLClient(username = 'TRexRegression', server = self.configuration.trex['trex_name'], verbose_level = self.json_verbose) @@ -251,11 +231,8 @@ class CTRexTestConfiguringPlugin(Plugin): if self.stateful: CTRexScenario.trex = None if self.stateless: - CTRexScenario.trex_stl = None - if not self.no_ssh: - if self.stateful: - trex_remote_command(self.configuration.trex, STATEFUL_STOP_COMMAND) - kill_trex_process(self.configuration.trex) + CTRexScenario.trex.force_kill(False) + CTRexScenario.stl_trex = None def save_setup_info(): @@ -274,10 +251,6 @@ def save_setup_info(): print('Error saving setup info: %s ' % err) -def set_report_dir (report_dir): - if not os.path.exists(report_dir): - os.mkdir(report_dir) - if __name__ == "__main__": # setting defaults. By default we run all the test suite @@ -305,10 +278,9 @@ if __name__ == "__main__": xml_name = 'unit_test.xml' if CTRexScenario.setup_dir: CTRexScenario.setup_name = os.path.basename(CTRexScenario.setup_dir) - CTRexScenario.GAManager = GAmanager(GoogleID='UA-75220362-4', UserID=CTRexScenario.setup_name, QueueSize=100, Timeout=5, UserPermission=1, BlockingMode=1, appName='TRex', appVer='1.11.232') #timeout in seconds xml_name = 'report_%s.xml' % CTRexScenario.setup_name xml_arg= '--xunit-file=%s/%s' % (CTRexScenario.report_dir, xml_name) - set_report_dir(CTRexScenario.report_dir) + mkpath(CTRexScenario.report_dir) sys_args = sys.argv[:] for i, arg in enumerate(sys.argv): diff --git a/scripts/automation/trex_control_plane/server/CCustomLogger.py b/scripts/automation/trex_control_plane/server/CCustomLogger.py index ecf7d519..a8823cea 100755 --- a/scripts/automation/trex_control_plane/server/CCustomLogger.py +++ b/scripts/automation/trex_control_plane/server/CCustomLogger.py @@ -31,6 +31,10 @@ def setup_custom_logger(name, log_path = None): def setup_daemon_logger (name, log_path = None):
# first make sure path availabe
+ try:
+ os.unlink(log_path)
+ except:
+ pass
logging.basicConfig(level = logging.INFO,
format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
diff --git a/scripts/automation/trex_control_plane/server/extended_daemon_runner.py b/scripts/automation/trex_control_plane/server/extended_daemon_runner.py deleted file mode 100755 index 7bc25aac..00000000 --- a/scripts/automation/trex_control_plane/server/extended_daemon_runner.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/python
-
-import outer_packages
-import lockfile
-from daemon import runner,daemon
-from daemon.runner import *
-import os, sys
-from argparse import ArgumentParser
-from trex_server import trex_parser
-try:
- from termstyle import termstyle
-except ImportError:
- import termstyle
-
-
-def daemonize_parser(parser_obj, action_funcs, help_menu):
- """Update the regular process parser to deal with daemon process options"""
- parser_obj.description += " (as a daemon process)"
- parser_obj.usage = None
- parser_obj.add_argument("action", choices=action_funcs,
- action="store", help=help_menu)
-
-
-class ExtendedDaemonRunner(runner.DaemonRunner):
- """ Controller for a callable running in a separate background process.
-
- The first command-line argument is the action to take:
-
- * 'start': Become a daemon and call `app.run()`.
- * 'stop': Exit the daemon process specified in the PID file.
- * 'restart': Stop, then start.
-
- """
-
- help_menu = """Specify action command to be applied on server.
- (*) start : start the application in as a daemon process.
- (*) show : prompt an updated status of daemon process (running/ not running).
- (*) stop : exit the daemon process.
- (*) restart : stop, then start again the application as daemon process
- (*) start-live : start the application in live mode (no daemon process).
- """
-
- def __init__(self, app, parser_obj):
- """ Set up the parameters of a new runner.
- THIS METHOD INTENTIONALLY DO NOT INVOKE SUPER __init__() METHOD
-
- :param app: The application instance; see below.
- :return: ``None``.
-
- The `app` argument must have the following attributes:
-
- * `stdin_path`, `stdout_path`, `stderr_path`: Filesystem paths
- to open and replace the existing `sys.stdin`, `sys.stdout`,
- `sys.stderr`.
-
- * `pidfile_path`: Absolute filesystem path to a file that will
- be used as the PID file for the daemon. If ``None``, no PID
- file will be used.
-
- * `pidfile_timeout`: Used as the default acquisition timeout
- value supplied to the runner's PID lock file.
-
- * `run`: Callable that will be invoked when the daemon is
- started.
-
- """
- super(runner.DaemonRunner, self).__init__()
- # update action_funcs to support more operations
- self.update_action_funcs()
-
- daemonize_parser(parser_obj, self.action_funcs, ExtendedDaemonRunner.help_menu)
- args = parser_obj.parse_args()
- self.action = unicode(args.action)
-
- self.app = app
- self.daemon_context = daemon.DaemonContext()
- self.daemon_context.stdin = open(app.stdin_path, 'rt')
- try:
- self.daemon_context.stdout = open(app.stdout_path, 'w+t')
- except IOError as err:
- # catch 'tty' error when launching server from remote location
- app.stdout_path = "/dev/null"
- self.daemon_context.stdout = open(app.stdout_path, 'w+t')
- self.daemon_context.stderr = open(app.stderr_path,
- 'a+t', buffering=0)
-
- self.pidfile = None
- if app.pidfile_path is not None:
- self.pidfile = make_pidlockfile(app.pidfile_path, app.pidfile_timeout)
- self.daemon_context.pidfile = self.pidfile
-
- # mask out all arguments that aren't relevant to main app script
-
- def update_action_funcs(self):
- self.action_funcs.update({u'start-live': self._start_live, u'show': self._show}) # add key (=action), value (=desired func)
-
- @staticmethod
- def _start_live(self):
- self.app.run()
-
- @staticmethod
- def _show(self):
- if self.pidfile.is_locked():
- print termstyle.red("TRex server daemon is running")
- else:
- print termstyle.red("TRex server daemon is NOT running")
-
- def do_action(self):
- self.__prevent_duplicate_runs()
- self.__prompt_init_msg()
- try:
- super(ExtendedDaemonRunner, self).do_action()
- if self.action == 'stop':
- self.__verify_termination()
- except runner.DaemonRunnerStopFailureError:
- if self.action == 'restart':
- # error means server wasn't running in the first place- so start it!
- self.action = 'start'
- self.do_action()
-
-
- def __prevent_duplicate_runs(self):
- if self.action == 'start' and self.pidfile.is_locked():
- print termstyle.green("Server daemon is already running")
- exit(1)
- elif self.action == 'stop' and not self.pidfile.is_locked():
- print termstyle.green("Server daemon is not running")
- exit(1)
-
- def __prompt_init_msg(self):
- if self.action == 'start':
- print termstyle.green("Starting daemon server...")
- elif self.action == 'stop':
- print termstyle.green("Stopping daemon server...")
-
- def __verify_termination(self):
- pass
-# import time
-# while self.pidfile.is_locked():
-# time.sleep(2)
-# self._stop()
-#
-
-
-if __name__ == "__main__":
- pass
diff --git a/scripts/automation/trex_control_plane/server/trex_daemon_server b/scripts/automation/trex_control_plane/server/trex_daemon_server deleted file mode 100755 index 3494e303..00000000 --- a/scripts/automation/trex_control_plane/server/trex_daemon_server +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python - -import os -import sys - -core = 0 - -if '--core' in sys.argv: - try: - idx = sys.argv.index('--core') - core = int(sys.argv[idx + 1]) - if core > 31 or core < 0: - print "Error: please provide core argument between 0 to 31" - exit(-1) - del sys.argv[idx:idx+2] - except IndexError: - print "Error: please make sure core option provided with argument" - exit(-1) - except ValueError: - print "Error: please make sure core option provided with integer argument" - exit(-1) - -str_argv = ' '.join(sys.argv[1:]) -cmd = "taskset -c {core} python automation/trex_control_plane/server/trex_daemon_server.py {argv}".format(core = core, argv = str_argv) -os.system(cmd) diff --git a/scripts/automation/trex_control_plane/server/trex_daemon_server.py b/scripts/automation/trex_control_plane/server/trex_daemon_server.py deleted file mode 100755 index 9784d42a..00000000 --- a/scripts/automation/trex_control_plane/server/trex_daemon_server.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python - -import outer_packages -import daemon -from trex_server import do_main_program, trex_parser -import CCustomLogger - -import logging -import time -import sys -import os, errno -import grp -import signal -from daemon import runner -from extended_daemon_runner import ExtendedDaemonRunner -import lockfile -import errno - -class TRexServerApp(object): - def __init__(self): - TRexServerApp.create_working_dirs() - self.stdin_path = '/dev/null' - self.stdout_path = '/dev/tty' # All standard prints will come up from this source. - self.stderr_path = "/var/log/trex/trex_daemon_server.log" # All log messages will come up from this source - self.pidfile_path = '/var/run/trex/trex_daemon_server.pid' - self.pidfile_timeout = 5 # timeout in seconds - - def run(self): - do_main_program() - - - @staticmethod - def create_working_dirs(): - if not os.path.exists('/var/log/trex'): - os.mkdir('/var/log/trex') - if not os.path.exists('/var/run/trex'): - os.mkdir('/var/run/trex') - - - -def main (): - - trex_app = TRexServerApp() - - # setup the logger - default_log_path = '/var/log/trex/trex_daemon_server.log' - - try: - CCustomLogger.setup_daemon_logger('TRexServer', default_log_path) - logger = logging.getLogger('TRexServer') - logger.setLevel(logging.INFO) - formatter = logging.Formatter("%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s") - handler = logging.FileHandler("/var/log/trex/trex_daemon_server.log") - logger.addHandler(handler) - except EnvironmentError, e: - if e.errno == errno.EACCES: # catching permission denied error - print "Launching user must have sudo privileges in order to run TRex daemon.\nTerminating daemon process." - exit(-1) - - daemon_runner = ExtendedDaemonRunner(trex_app, trex_parser) - - #This ensures that the logger file handle does not get closed during daemonization - daemon_runner.daemon_context.files_preserve=[handler.stream] - - try: - if not set(['start', 'stop']).isdisjoint(set(sys.argv)): - print "Logs are saved at: {log_path}".format( log_path = default_log_path ) - daemon_runner.do_action() - - except lockfile.LockTimeout as inst: - logger.error(inst) - print inst - print """ - Please try again once the timeout has been reached. - If this error continues, consider killing the process manually and restart the daemon.""" - - -if __name__ == "__main__": - main() diff --git a/scripts/automation/trex_control_plane/server/trex_server.py b/scripts/automation/trex_control_plane/server/trex_server.py index e32fc9d1..3dcb3e97 100755 --- a/scripts/automation/trex_control_plane/server/trex_server.py +++ b/scripts/automation/trex_control_plane/server/trex_server.py @@ -27,6 +27,8 @@ from zmq_monitor_thread import ZmqMonitorSession from argparse import ArgumentParser, RawTextHelpFormatter from json import JSONEncoder import re +import shlex +import tempfile # setup the logger @@ -52,6 +54,8 @@ class CTRexServer(object): trex_zmq_port : int the port number on which trex's zmq module will interact with daemon server default value: 4500 + nice: int + priority of the TRex process Instantiate a TRex client object, and connecting it to listening daemon-server """ @@ -76,7 +80,6 @@ class CTRexServer(object): raise Exception(err) def add(self, x, y): - print "server function add ",x,y logger.info("Processing add function. Parameters are: {0}, {1} ".format( x, y )) return x + y # return Fault(-10, "") @@ -123,38 +126,49 @@ class CTRexServer(object): raise # set further functionality and peripherals to server instance + self.server.register_function(self.add) + self.server.register_function(self.cancel_reservation) + self.server.register_function(self.connectivity_check) + self.server.register_function(self.force_trex_kill) + self.server.register_function(self.get_file) + self.server.register_function(self.get_files_list) + self.server.register_function(self.get_files_path) + self.server.register_function(self.get_running_info) + self.server.register_function(self.get_running_status) + self.server.register_function(self.get_trex_cmds) + self.server.register_function(self.get_trex_daemon_log) + self.server.register_function(self.get_trex_log) + self.server.register_function(self.get_trex_version) + self.server.register_function(self.is_reserved) + self.server.register_function(self.is_running) + self.server.register_function(self.kill_all_trexes) + self.server.register_function(self.push_file) + self.server.register_function(self.reserve_trex) + self.server.register_function(self.start_trex) + self.server.register_function(self.stop_trex) + self.server.register_function(self.wait_until_kickoff_finish) + signal.signal(signal.SIGTSTP, self.stop_handler) + signal.signal(signal.SIGTERM, self.stop_handler) try: - self.server.register_function(self.add) - self.server.register_function(self.cancel_reservation) - self.server.register_function(self.connectivity_check) - self.server.register_function(self.force_trex_kill) - self.server.register_function(self.get_file) - self.server.register_function(self.get_files_list) - self.server.register_function(self.get_files_path) - self.server.register_function(self.get_running_info) - self.server.register_function(self.get_running_status) - self.server.register_function(self.get_trex_daemon_log) - self.server.register_function(self.get_trex_log) - self.server.register_function(self.get_trex_version) - self.server.register_function(self.is_reserved) - self.server.register_function(self.is_running) - self.server.register_function(self.push_file) - self.server.register_function(self.reserve_trex) - self.server.register_function(self.start_trex) - self.server.register_function(self.stop_trex) - self.server.register_function(self.wait_until_kickoff_finish) - signal.signal(signal.SIGTSTP, self.stop_handler) - signal.signal(signal.SIGTERM, self.stop_handler) self.zmq_monitor.start() self.server.serve_forever() except KeyboardInterrupt: logger.info("Daemon shutdown request detected." ) - except Exception as e: - logger.error(e) finally: self.zmq_monitor.join() # close ZMQ monitor thread resources self.server.shutdown() - pass + #self.server.server_close() + + def _run_command(self, command, timeout = 15, cwd = None): + if timeout: + command = 'timeout %s %s' % (timeout, command) + # pipes might stuck, even with timeout + with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: + proc = subprocess.Popen(shlex.split(command), stdout=stdout_file, stderr=stderr_file, cwd = cwd) + proc.wait() + stdout_file.seek(0) + stderr_file.seek(0) + return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')) # get files from Trex server and return their content (mainly for logs) @staticmethod @@ -215,8 +229,7 @@ class CTRexServer(object): try: logger.info("Processing get_trex_version() command.") if not self.trex_version: - help_print = subprocess.Popen(['./t-rex-64', '--help'], cwd = self.TREX_PATH, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = help_print.communicate() + ret_code, stdout, stderr = self._run_command('./t-rex-64 --help', cwd = self.TREX_PATH, timeout = 0) search_result = re.search('\n\s*(Version\s*:.+)', stdout, re.DOTALL) if not search_result: raise Exception('Could not determine version from ./t-rex-64 --help') @@ -226,7 +239,7 @@ class CTRexServer(object): else: return binascii.a2b_base64(self.trex_version) except Exception as e: - err_str = "Can't get trex version, error: {0}".format(e) + err_str = "Can't get trex version, error: %s" % e logger.error(err_str) return Fault(-33, err_str) @@ -301,7 +314,7 @@ class CTRexServer(object): return False - def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40): + def start_trex(self, trex_cmd_options, user, block_to_success = True, timeout = 40, stateless = False): with self.start_lock: logger.info("Processing start_trex() command.") if self.is_reserved(): @@ -313,8 +326,8 @@ class CTRexServer(object): logger.info("TRex is already taken, cannot create another run until done.") return Fault(-13, '') # raise at client TRexInUseError - try: - server_cmd_data = self.generate_run_cmd(**trex_cmd_options) + try: + server_cmd_data = self.generate_run_cmd(stateless = stateless, **trex_cmd_options) self.zmq_monitor.first_dump = True self.trex.start_trex(self.TREX_PATH, server_cmd_data) logger.info("TRex session has been successfully initiated.") @@ -342,7 +355,7 @@ class CTRexServer(object): except TypeError as e: logger.error("TRex command generation failed, probably because either -f (traffic generation .yaml file) and -c (num of cores) was not specified correctly.\nReceived params: {params}".format( params = trex_cmd_options) ) - raise TypeError('TRex -f (traffic generation .yaml file) and -c (num of cores) must be specified.') + raise TypeError('TRex -f (traffic generation .yaml file) and -c (num of cores) must be specified. %s' % e) def stop_trex(self, seq): @@ -362,6 +375,41 @@ class CTRexServer(object): logger.info("Processing force_trex_kill() command. --> Killing TRex session indiscriminately.") return self.trex.stop_trex() + # returns list of tuples (pid, command line) of running TRex(es) + def get_trex_cmds(self): + logger.info('Processing get_trex_cmds() command.') + ret_code, stdout, stderr = self._run_command('ps -u root --format pid,comm,cmd') + if ret_code: + raise Exception('Failed to determine running processes, stderr: %s' % stderr) + trex_cmds_list = [] + for line in stdout.splitlines(): + pid, proc_name, full_cmd = line.strip().split(' ', 2) + pid = pid.strip() + full_cmd = full_cmd.strip() + if proc_name.find('t-rex-64') >= 0: + trex_cmds_list.append((pid, full_cmd)) + return trex_cmds_list + + + def kill_all_trexes(self): + logger.info('Processing kill_all_trexes() command.') + trex_cmds_list = self.get_trex_cmds() + if not trex_cmds_list: + return False + for pid, cmd in trex_cmds_list: + logger.info('Killing process %s %s' % (pid, cmd)) + self._run_command('kill %s' % pid) + ret_code_ps, _, _ = self._run_command('ps -p %s' % pid) + if not ret_code_ps: + logger.info('Killing with -9.') + self._run_command('kill -9 %s' % pid) + ret_code_ps, _, _ = self._run_command('ps -p %s' % pid) + if not ret_code_ps: + logger.info('Could not kill process.') + raise Exception('Could not kill process %s %s' % (pid, cmd)) + return True + + def wait_until_kickoff_finish (self, timeout = 40): # block until TRex exits Starting state logger.info("Processing wait_until_kickoff_finish() command.") @@ -377,14 +425,19 @@ class CTRexServer(object): logger.info("Processing get_running_info() command.") return self.trex.get_running_info() - def generate_run_cmd (self, f, d, iom = 0, export_path="/tmp/trex.txt", **kwargs): - """ generate_run_cmd(self, trex_cmd_options, export_path) -> str + + def generate_run_cmd (self, iom = 0, export_path="/tmp/trex.txt", stateless = False, **kwargs): + """ generate_run_cmd(self, iom, export_path, kwargs) -> str Generates a custom running command for the kick-off of the TRex traffic generator. Returns a tuple of command (string) and export path (string) to be issued on the trex server Parameters ---------- + iom: int + 0 = don't print stats screen to log, 1 = print stats (can generate huge logs) + stateless: boolean + True = run as stateless, False = require -f and -d arguments kwargs: dictionary Dictionary of parameters for trex. For example: (c=1, nc=True, l_pkt_mode=3). Notice that when sending command line parameters that has -, you need to replace it with _. @@ -396,7 +449,8 @@ class CTRexServer(object): if 'results_file_path' in kwargs: export_path = kwargs['results_file_path'] del kwargs['results_file_path'] - + if stateless: + kwargs['i'] = True # adding additional options to the command trex_cmd_options = '' @@ -408,18 +462,23 @@ class CTRexServer(object): else: trex_cmd_options += (dash + '{k} {val}'.format( k = tmp_key, val = value )) - cmd = "{nice}{run_command} -f {gen_file} -d {duration} --iom {io} {cmd_options} --no-key > {export}".format( # -- iom 0 disables the periodic log to the screen (not needed) + if not stateless: + if 'f' not in kwargs: + raise Exception('Argument -f should be specified in stateful command') + if 'd' not in kwargs: + raise Exception('Argument -d should be specified in stateful command') + + cmd = "{nice}{run_command} --iom {io} {cmd_options} --no-key > {export}".format( # -- iom 0 disables the periodic log to the screen (not needed) nice = '' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice, run_command = self.TREX_START_CMD, - gen_file = f, - duration = d, cmd_options = trex_cmd_options, io = iom, export = export_path ) logger.info("TREX FULL COMMAND: {command}".format(command = cmd) ) - return (cmd, export_path, long(d)) + return (cmd, export_path, kwargs.get('d', 0)) + def __check_trex_path_validity(self): # check for executable existance diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py index fd409b16..342c75de 100755 --- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py @@ -39,7 +39,7 @@ class CTRexClient(object): This class defines the client side of the RESTfull interaction with TRex """ - def __init__(self, trex_host, max_history_size = 100, filtered_latency_amount = 0.001, trex_daemon_port = 8090, trex_zmq_port = 4500, verbose = False): + def __init__(self, trex_host, max_history_size = 100, filtered_latency_amount = 0.001, trex_daemon_port = 8090, master_daemon_port = 8091, trex_zmq_port = 4500, verbose = False): """ Instantiate a TRex client object, and connecting it to listening daemon-server @@ -60,6 +60,10 @@ class CTRexClient(object): the port number on which the trex-daemon server can be reached default value: **8090** + master_daemon_port : int + the port number on which the master-daemon server can be reached + + default value: **8091** trex_zmq_port : int the port number on which trex's zmq module will interact with daemon server @@ -78,19 +82,50 @@ class CTRexClient(object): except: # give it another try self.trex_host = socket.gethostbyname(trex_host) self.trex_daemon_port = trex_daemon_port + self.master_daemon_port = master_daemon_port self.trex_zmq_port = trex_zmq_port self.seq = None + self._last_sample = time.time() + self.__default_user = get_current_user() self.verbose = verbose self.result_obj = CTRexResult(max_history_size, filtered_latency_amount) self.decoder = JSONDecoder() - self.trex_server_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port ) - self.__verbose_print("Connecting to TRex @ {trex_path} ...".format( trex_path = self.trex_server_path ) ) self.history = jsonrpclib.history.History() - self.server = jsonrpclib.Server(self.trex_server_path, history = self.history) - self.check_server_connectivity() - self.__verbose_print("Connection established successfully!") - self._last_sample = time.time() - self.__default_user = get_current_user() + self.master_daemon_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = master_daemon_port ) + self.trex_server_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port ) + self.connect_master() + self.connect_server() + + + def connect_master(self): + ''' + Connects to Master daemon via JsonRPC. + This daemon controls TRex daemon server. + ''' + try: + print('Connecting to Master daemon @ %s ...' % self.master_daemon_path) + self.master_daemon = jsonrpclib.Server(self.master_daemon_path, history = self.history) + print self.check_master_connectivity() + print('Connected to Master daemon.') + return True + except Exception as e: + print(e) + return False + + def connect_server(self): + ''' + Connects to TRex daemon server via JsonRPC. + This daemon controls TRex. (start/stop + ''' + try: + print('Connecting to TRex daemon server @ %s ...' % self.trex_server_path) + self.server = jsonrpclib.Server(self.trex_server_path, history = self.history) + self.check_server_connectivity() + print('Connected TRex server daemon.') + return True + except Exception as e: + print(e) + return False def add (self, x, y): @@ -105,7 +140,7 @@ class CTRexClient(object): def start_trex (self, f, d, block_to_success = True, timeout = 40, user = None, trex_development = False, **trex_cmd_options): """ - Request to start a TRex run on server. + Request to start a TRex run on server in stateful mode. :parameters: f : str @@ -168,6 +203,53 @@ class CTRexClient(object): else: # TRex is has been started by another user raise TRexInUseError('TRex is already being used by another user or process. Try again once TRex is back in IDLE state.') + + def start_stateless(self, block_to_success = True, timeout = 40, user = None, **trex_cmd_options): + """ + Request to start a TRex run on server in stateless mode. + + :parameters: + block_to_success : bool + determine if this method blocks until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + default value : **True** + timeout : int + maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + default value: **40** + user : str + the identity of the the run issuer. + trex_cmd_options : key, val + sets desired TRex options using key=val syntax, separated by comma. + for keys with no value, state key=True + + :return: + **True** on success + + :raises: + + :exc:`trex_exceptions.TRexError`, in case one of the trex_cmd_options raised an exception at server. + + :exc:`trex_exceptions.TRexInUseError`, in case TRex is already taken. + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying start TRex. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + user = user or self.__default_user + retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout, True) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + if retval!=0: + self.seq = retval # update seq num only on successful submission + return True + else: # TRex is has been started by another user + raise TRexInUseError('TRex is already being used by another user or process. Try again once TRex is back in IDLE state.') + + def stop_trex (self): """ Request to stop a TRex run on server. @@ -238,6 +320,40 @@ class CTRexClient(object): finally: self.prompt_verbose_data() + def kill_all_trexes(self): + """ + Kills running TRex processes (if exists) on the server, not only owned by current daemon. + Raises exception upon error killing. + + :return: + + **True** if any process killed + + **False** otherwise. + + """ + try: + return self.server.kill_all_trexes() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + + def get_trex_cmds(self): + """ + Gets list of running TRex pids and command lines. + Can be used to verify if any TRex is running. + + :return: + List of tuples (pid, command) of running TRexes + """ + try: + return self.server.get_trex_cmds() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + def wait_until_kickoff_finish(self, timeout = 40): """ Block the client application until TRex changes state from 'Starting' to either 'Idle' or 'Running' @@ -306,8 +422,9 @@ class CTRexClient(object): raise except ProtocolError as err: raise - finally: - self.prompt_verbose_data() + #is printed by self.get_running_info() + #finally: + # self.prompt_verbose_data() def is_idle (self): """ @@ -844,11 +961,11 @@ class CTRexClient(object): return method_to_call() except socket.error as e: if e.errno == errno.ECONNREFUSED: - raise SocketError(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") + raise SocketError(errno.ECONNREFUSED, "Connection to TRex daemon server was refused. Please make sure the server is up.") def check_server_connectivity (self): """ - Checks for server valid connectivity. + Checks TRex daemon server for connectivity. """ try: socket.gethostbyname(self.trex_host) @@ -857,7 +974,86 @@ class CTRexClient(object): raise socket.gaierror(e.errno, "Could not resolve server hostname. Please make sure hostname entered correctly.") except socket.error as e: if e.errno == errno.ECONNREFUSED: - raise socket.error(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") + raise socket.error(errno.ECONNREFUSED, "Connection to TRex daemon server was refused. Please make sure the server is up.") + finally: + self.prompt_verbose_data() + + + def master_add(self, x, y): + ''' Sanity check for Master daemon ''' + try: + return self.master_daemon.add(x,y) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + + def check_master_connectivity (self): + ''' + Check Master daemon for connectivity. + Return True upon success + ''' + try: + socket.gethostbyname(self.trex_host) + return self.master_daemon.check_connectivity() + except socket.gaierror as e: + raise socket.gaierror(e.errno, "Could not resolve server hostname. Please make sure hostname entered correctly.") + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + raise socket.error(errno.ECONNREFUSED, "Connection to Master daemon was refused. Please make sure the server is up.") + finally: + self.prompt_verbose_data() + + def is_trex_daemon_running(self): + ''' + Check if TRex server daemon is running. + Returns True/False + ''' + try: + return self.master_daemon.is_trex_daemon_running() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + def restart_trex_daemon(self): + ''' + Restart TRex server daemon. Useful after update. + Will not fail if daemon is initially stopped. + ''' + try: + return self.master_daemon.restart_trex_daemon() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + def start_trex_daemon(self): + ''' + Start TRex server daemon. + :return: + + **True** if success. + + **False** if TRex server daemon already running. + ''' + try: + return self.master_daemon.start_trex_daemon() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + finally: + self.prompt_verbose_data() + + def stop_trex_daemon(self): + ''' + Stop TRex server daemon. + :return: + + **True** if success. + + **False** if TRex server daemon already running. + ''' + try: + return self.master_daemon.stop_trex_daemon() + except AppError as err: + self._handle_AppError_exception(err.args[0]) finally: self.prompt_verbose_data() diff --git a/scripts/daemon_server b/scripts/daemon_server index 90fc614d..4df5cfda 100755..120000 --- a/scripts/daemon_server +++ b/scripts/daemon_server @@ -1,27 +1 @@ -#!/usr/bin/python - -import os -import sys - -core = 0 - -if '--core' in sys.argv: - try: - idx = sys.argv.index('--core') - core = int(sys.argv[idx + 1]) - if core > 31 or core < 0: - print "Error: please provide core argument between 0 to 31" - exit(-1) - del sys.argv[idx:idx+2] - except IndexError: - print "Error: please make sure core option provided with argument" - exit(-1) - except ValueError: - print "Error: please make sure core option provided with integer argument" - exit(-1) - -str_argv = ' '.join(sys.argv[1:]) -cmd = "taskset -c {core} python automation/trex_control_plane/server/trex_daemon_server.py {argv}".format(core = core, argv = str_argv) -os.system(cmd) - - +trex_daemon_server
\ No newline at end of file diff --git a/scripts/master_daemon.py b/scripts/master_daemon.py new file mode 100755 index 00000000..caaaf5fc --- /dev/null +++ b/scripts/master_daemon.py @@ -0,0 +1,281 @@ +#!/usr/bin/python +import os, sys, getpass +import tempfile +import argparse +import socket +from time import time, sleep +import subprocess, shlex, shutil, multiprocessing +from distutils.dir_util import mkpath +from glob import glob +import logging +logging.basicConfig(level = logging.FATAL) # keep quiet + +sys.path.append(os.path.join('external_libs', 'jsonrpclib-pelix-0.2.5')) +from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + +sys.path.append(os.path.join('external_libs', 'termstyle')) +import termstyle + + +### Server functions ### + +def check_connectivity(): + return True + +def add(a, b): # for sanity checks + return a + b + +def is_trex_daemon_running(): + ret_code, stdout, stderr = run_command('ps -u root --format comm') + if ret_code: + raise Exception('Failed to list running processes, stderr: %s' % stderr) + if 'trex_daemon_ser' in stdout: # name is cut + return True + return False + +def restart_trex_daemon(): + if is_trex_daemon_running: + stop_trex_daemon() + start_trex_daemon() + return True + +def stop_trex_daemon(): + if not is_trex_daemon_running(): + return False + return_code, stdout, stderr = run_command('%s stop' % trex_daemon_path) + if return_code: + raise Exception('Could not stop trex_daemon_server, err: %s' % stderr) + for i in range(20): + if not is_trex_daemon_running(): + return True + sleep(0.1) + raise Exception('Could not stop trex_daemon_server') + +def start_trex_daemon(): + if is_trex_daemon_running(): + return False + return_code, stdout, stderr = run_command('%s start' % trex_daemon_path) + if return_code: + raise Exception('Could not run trex_daemon_server, err: %s' % stderr) + for i in range(20): + if is_trex_daemon_running(): + return True + sleep(0.1) + raise Exception('Could not run trex_daemon_server') + +def update_trex(package_path = 'http://trex-tgn.cisco.com/trex/release/latest'): + if not args.allow_update: + raise Exception('Updading server not allowed') + try: + cur_dir = args.trex_dir + bu_dir = '%s_BU%i' % (cur_dir, int(time())) + # rename current dir for backup + try: + if os.path.exists(bu_dir): + shutil.rmtree(bu_dir) + shutil.move(cur_dir, bu_dir) + mkpath(cur_dir) + except Exception as e: + raise Exception('Could not make backup of previous directory. Err: %s' % e) + # getting new package + if package_path.startswith('http://'): + file_name = package_path.split('/')[-1] + ret_code, stdout, stderr = run_command('wget %s -O %s' % (package_path, cur_dir), timeout = 600) + elif os.path.normpath(package_path).startswith('/auto/') and package_path.endswith('.tar.gz'): + file_name = os.path.basename(package_path) + ret_code, stdout, stderr = run_command('rsync -cL %s %s' % (package_path, cur_dir), timeout = 300) + else: + raise Exception('package_path should be http address or path starting with /auto') + if ret_code: + raise Exception('Could not get requested package: %s' % stderr) + # unpacking + ret_code, stdout, stderr = run_command('tar -mxzf %s -C %s' % (os.path.join(cur_dir, file_name), cur_dir)) + if ret_code: + raise Exception('Could not untar the package: %s' % stderr) + # version is the name of dir + dir_cont = glob(os.path.join(cur_dir, 'v[0-9].[0-9][0-9]')) + if not len(dir_cont): + raise Exception('Did not found directory with version name after unpacking.') + if len(dir_cont) > 1: + raise Exception('Found more than one directory with version name after unpacking.') + version = os.path.basename(dir_cont[0]) + # moving files from dir with version one level up + for root, dirs, files in os.walk(os.path.join(cur_dir, version)): + for f in files: + shutil.move(os.path.join(root, f), os.path.join(cur_dir, f)) + for d in dirs: + shutil.move(os.path.join(root, d), os.path.join(cur_dir, d)) + # no errors, remove BU dir + if os.path.exists(bu_dir): + shutil.rmtree(bu_dir) + return True + except: # something went wrong, return backup dir + if os.path.exists(cur_dir): + shutil.rmtree(cur_dir) + shutil.move(bu_dir, cur_dir) + raise + + +def unpack_client(): + if not args.allow_update: + raise Exception('Unpacking of client not allowed') + # determining client archive + client_pkg_files = glob(os.path.join(args.trex_dir, 'trex_client*.tar.gz')) + if not len(client_pkg_files): + raise Exception('Could not find client package') + if len(client_pkg_files) > 1: + raise Exception('Found more than one client packages') + client_pkg_name = os.path.basename(client_pkg_files[0]) + client_new_path = os.path.join(args.trex_dir, 'trex_client') + if os.path.exists(client_new_path): + if os.path.islink(client_new_path) or os.path.isfile(client_new_path): + os.unlink(client_new_path) + else: + shutil.rmtree(client_new_path) + # unpacking + ret_code, stdout, stderr = run_command('tar -mxzf %s -C %s' % (client_pkg_files[0], args.trex_dir)) + if ret_code: + raise Exception('Could not untar the client package: %s' % stderr) + return True + +### /Server functions ### + +def fail(msg): + print(msg) + sys.exit(-1) + +def run_command(command, timeout = 15): + command = 'timeout %s %s' % (timeout, command) + # pipes might stuck, even with timeout + with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: + proc = subprocess.Popen(shlex.split(command), stdout=stdout_file, stderr=stderr_file) + proc.wait() + stdout_file.seek(0) + stderr_file.seek(0) + return (proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')) + +def show_master_daemon_status(): + if get_master_daemon_pid(): + print(termstyle.red('Master daemon is running')) + else: + print(termstyle.red('Master daemon is NOT running')) + +def start_master_daemon(): + if get_master_daemon_pid(): + print(termstyle.red('Master daemon is already running')) + return + server = multiprocessing.Process(target = start_master_daemon_func) + server.daemon = True + server.start() + for i in range(10): + if get_master_daemon_pid(): + print(termstyle.green('Master daemon is started')) + os._exit(0) + sleep(0.1) + fail(termstyle.red('Master daemon failed to run')) + +def restart_master_daemon(): + if get_master_daemon_pid(): + kill_master_daemon() + start_master_daemon() + +def start_master_daemon_func(): + server = SimpleJSONRPCServer(('0.0.0.0', args.daemon_port)) + print('Started master daemon (port %s)' % args.daemon_port) + server.register_function(add) + server.register_function(check_connectivity) + server.register_function(is_trex_daemon_running) + server.register_function(restart_trex_daemon) + server.register_function(start_trex_daemon) + server.register_function(stop_trex_daemon) + server.register_function(unpack_client) + server.register_function(update_trex) + server.serve_forever() + + +def get_master_daemon_pid(): + return_code, stdout, stderr = run_command('netstat -tlnp') + if return_code: + fail('Failed to determine which program holds port %s, netstat error: %s' % (args.daemon_port, stderr)) + for line in stdout.splitlines(): + if '0.0.0.0:%s' % args.daemon_port in line: + line_arr = line.split() + if len(line_arr) != 7: + fail('Could not parse netstat line to determine which process holds port %s: %s'(args.daemon_port, line)) + if '/' not in line_arr[6]: + fail('Expecting pid/program name in netstat line of using port %s, got: %s'(args.daemon_port, line_arr[6])) + pid, program = line_arr[6].split('/') + if 'python' not in program and 'master_daemon' not in program: + fail('Some other program holds port %s, not our daemon: %s. Please verify.' % (args.daemon_port, program)) + return pid + return None + +def kill_master_daemon(): + pid = get_master_daemon_pid() + if not pid: + print(termstyle.red('Master daemon is NOT running')) + return True + return_code, stdout, stderr = run_command('kill %s' % pid) # usual kill + if return_code: + fail('Failed to kill master daemon, error: %s' % stderr) + for i in range(10): + if not get_master_daemon_pid(): + print(termstyle.green('Master daemon is killed')) + return True + sleep(0.1) + return_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill + if return_code: + fail('Failed to kill trex_daemon, error: %s' % stderr) + for i in range(10): + if not get_master_daemon_pid(): + print(termstyle.green('Master daemon is killed')) + return True + sleep(0.1) + fail('Failed to kill master daemon, even with -9. Please review manually.') # should not happen + +# returns True if given path is under current dir or /tmp +def _check_path_under_current_or_temp(path): + if not os.path.relpath(path, '/tmp').startswith(os.pardir): + return True + if not os.path.relpath(path, os.getcwd()).startswith(os.pardir): + return True + return False + +### Main ### + +if getpass.getuser() != 'root': + fail('Please run this program as root/with sudo') + +actions_help = '''Specify action command to be applied on master daemon. + (*) start : start the master daemon. + (*) show : prompt the status of master daemon process (running / not running). + (*) stop : exit the master daemon process. + (*) restart : stop, then start again the master daemon process + ''' +action_funcs = {'start': start_master_daemon, + 'show': show_master_daemon_status, + 'stop': kill_master_daemon, + 'restart': restart_master_daemon, + } + +parser = argparse.ArgumentParser(description = 'Runs master daemon that can start/stop TRex daemon or update TRex version.') +parser.add_argument('-p', '--daemon-port', type=int, default = 8091, dest='daemon_port', + help = 'Select port on which the master_daemon runs.\nDefault is 8091.', action = 'store') +parser.add_argument('-d', '--trex-dir', type=str, default = os.getcwd(), dest='trex_dir', + help = 'Path of TRex, default is current dir', action = 'store') +parser.add_argument('--allow-update', default = False, dest='allow_update', action = 'store_true', + help = "Allow update of TRex via RPC command. WARNING: It's security hole! Use on your risk!") +parser.add_argument('action', choices=action_funcs.keys(), + action='store', help=actions_help) +parser.usage = None +args = parser.parse_args() + +if not os.path.exists(args.trex_dir): + raise Exception("Given path '%s' does not exist" % args.trex_dir) +if not _check_path_under_current_or_temp(args.trex_dir): + raise Exception('Only allowed to use path under /tmp or current directory') + +trex_daemon_path = os.path.join(args.trex_dir, 'trex_daemon_server') +action_funcs[args.action]() + + diff --git a/scripts/trex_daemon_server b/scripts/trex_daemon_server index 3494e303..be958976 100755 --- a/scripts/trex_daemon_server +++ b/scripts/trex_daemon_server @@ -1,25 +1,137 @@ #!/usr/bin/python -import os -import sys - -core = 0 - -if '--core' in sys.argv: - try: - idx = sys.argv.index('--core') - core = int(sys.argv[idx + 1]) - if core > 31 or core < 0: - print "Error: please provide core argument between 0 to 31" - exit(-1) - del sys.argv[idx:idx+2] - except IndexError: - print "Error: please make sure core option provided with argument" - exit(-1) - except ValueError: - print "Error: please make sure core option provided with integer argument" - exit(-1) - -str_argv = ' '.join(sys.argv[1:]) -cmd = "taskset -c {core} python automation/trex_control_plane/server/trex_daemon_server.py {argv}".format(core = core, argv = str_argv) -os.system(cmd) +import os, sys, getpass +from time import time, sleep +import subprocess, shlex, multiprocessing +from argparse import ArgumentParser +from distutils.dir_util import mkpath + +def fail(msg): + print(msg) + sys.exit(-1) + +if getpass.getuser() != 'root': + fail('Please run this program as root/with sudo') + +sys.path.append(os.path.join('automation', 'trex_control_plane', 'server')) +if 'start-live' not in sys.argv: + import CCustomLogger + CCustomLogger.setup_daemon_logger('TRexServer', '/var/log/trex/trex_daemon_server.log') +import trex_server + +try: + from termstyle import termstyle +except ImportError: + import termstyle + + +def run_command(command, timeout = 10): + commmand = 'timeout %s %s' % (timeout, command) + proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd = os.getcwd()) + stdout, stderr = proc.communicate() + return (proc.returncode, stdout.decode(errors = 'replace'), stderr.decode(errors = 'replace')) + + +def get_daemon_pid(): + return_code, stdout, stderr = run_command('netstat -tlnp') + if return_code: + fail('Failed to determine which program holds port %s, netstat error: %s' % (args.daemon_port, stderr)) + for line in stdout.splitlines(): + if '0.0.0.0:%s' % args.daemon_port in line: + line_arr = line.split() + if '/' not in line_arr[-1]: + fail('Expecting pid/program name in netstat line of using port %s, got: %s'(args.daemon_port, line_arr[-1])) + pid, program = line_arr[-1].split('/') + if 'python' not in program and 'trex_server' not in program and 'trex_daemon_server' not in program: + fail('Some other program holds port %s, not our daemon: %s. Please verify.' % (args.daemon_port, program)) + return int(pid) + return None + + +def show_daemon_status(): + if get_daemon_pid(): + print(termstyle.red('TRex server daemon is running')) + else: + print(termstyle.red('TRex server daemon is NOT running')) + + +def start_daemon(): + if get_daemon_pid(): + print(termstyle.red('TRex server daemon is already running')) + return + # Usual daemon will die with current process, detach it with double fork + # https://www.safaribooksonline.com/library/view/python-cookbook/0596001673/ch06s08.html + pid = os.fork() + if pid > 0: + for i in range(10): + if get_daemon_pid(): + print(termstyle.green('TRex server daemon is started')) + os._exit(0) + sleep(0.1) + fail(termstyle.red('TRex server daemon failed to run')) + os.setsid() + pid = os.fork() + if pid > 0: + os._exit(0) + trex_server.do_main_program() + + +def start_live(): + if get_daemon_pid(): + fail(termstyle.red('TRex server daemon is already running')) + trex_server.do_main_program() + +def restart_daemon(): + if get_daemon_pid(): + kill_daemon() + start_daemon() + +def kill_daemon(): + pid = get_daemon_pid() + if not pid: + print(termstyle.red('TRex server daemon is NOT running')) + return True + return_code, stdout, stderr = run_command('kill %s' % pid) # usual kill + if return_code: + fail('Failed to kill trex_daemon, error: %s' % stderr) + for i in range(10): + if not get_daemon_pid(): + print(termstyle.green('TRex server daemon is killed')) + return True + sleep(0.1) + return_code, stdout, stderr = run_command('kill -9 %s' % pid) # unconditional kill + if return_code: + fail('Failed to kill trex_daemon, error: %s' % stderr) + for i in range(10): + if not get_daemon_pid(): + print(termstyle.green('TRex server daemon is killed')) + return True + sleep(0.1) + fail('Failed to kill trex_daemon, even with -9. Please review manually.') # should not happen + +### Main ### + +actions_help = '''Specify action command to be applied on server. + (*) start : start the application in as a daemon process. + (*) show : prompt an updated status of daemon process (running/ not running). + (*) stop : exit the daemon process. + (*) restart : stop, then start again the application as daemon process + (*) start-live : start the application in live mode (no daemon process). + ''' +action_funcs = {'start': start_daemon, + 'show': show_daemon_status, + 'stop': kill_daemon, + 'restart': restart_daemon, + 'start-live': start_live, + } +trex_server.trex_parser.add_argument('action', choices=action_funcs.keys(), + action='store', help=actions_help) +trex_server.trex_parser.usage = None +args = trex_server.trex_parser.parse_args() + +mkpath('/var/log/trex') +mkpath('/var/run/trex') + +action_funcs[args.action]() + + |