From b91c216db1aa10ca7cc81b8c74b04ab79df251fe Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Wed, 30 Mar 2016 11:29:11 +0300 Subject: add to run_functional_tests --python2 and --python3 flags make stateful tree similar to stl : stf/trex_stf_lib and stf/examples change trex_client package: add profiles, use full stl dir. stl examples: now use stl_path.STL_PROFILES_PATH variable regression: add higher timeout to rsync. --- .../trex_control_plane/stf/CCustomLogger.py | 100 -- .../automation/trex_control_plane/stf/__init__.py | 1 - .../trex_control_plane/stf/examples/stf_example.py | 53 + .../trex_control_plane/stf/examples/stf_path.py | 4 + .../trex_control_plane/stf/external_packages.py | 28 - .../trex_control_plane/stf/general_utils.py | 95 -- .../trex_control_plane/stf/outer_packages.py | 30 - .../automation/trex_control_plane/stf/text_opts.py | 192 ---- .../trex_control_plane/stf/trex_client.py | 1216 -------------------- .../trex_control_plane/stf/trex_daemon_server.py | 79 -- .../trex_control_plane/stf/trex_exceptions.py | 140 --- .../trex_control_plane/stf/trex_status.py | 8 - .../trex_control_plane/stf/trex_status_e.py | 11 - .../stf/trex_stf_lib/CCustomLogger.py | 100 ++ .../stf/trex_stf_lib/__init__.py | 1 + .../stf/trex_stf_lib/external_packages.py | 28 + .../stf/trex_stf_lib/general_utils.py | 95 ++ .../stf/trex_stf_lib/outer_packages.py | 30 + .../stf/trex_stf_lib/text_opts.py | 192 ++++ .../stf/trex_stf_lib/trex_client.py | 1216 ++++++++++++++++++++ .../stf/trex_stf_lib/trex_daemon_server.py | 79 ++ .../stf/trex_stf_lib/trex_exceptions.py | 140 +++ .../stf/trex_stf_lib/trex_status.py | 8 + .../stf/trex_stf_lib/trex_status_e.py | 11 + 24 files changed, 1957 insertions(+), 1900 deletions(-) delete mode 100755 scripts/automation/trex_control_plane/stf/CCustomLogger.py delete mode 100755 scripts/automation/trex_control_plane/stf/__init__.py create mode 100755 scripts/automation/trex_control_plane/stf/examples/stf_example.py create mode 100755 scripts/automation/trex_control_plane/stf/examples/stf_path.py delete mode 100755 scripts/automation/trex_control_plane/stf/external_packages.py delete mode 100755 scripts/automation/trex_control_plane/stf/general_utils.py delete mode 100755 scripts/automation/trex_control_plane/stf/outer_packages.py delete mode 100755 scripts/automation/trex_control_plane/stf/text_opts.py delete mode 100755 scripts/automation/trex_control_plane/stf/trex_client.py delete mode 100755 scripts/automation/trex_control_plane/stf/trex_daemon_server.py delete mode 100755 scripts/automation/trex_control_plane/stf/trex_exceptions.py delete mode 100644 scripts/automation/trex_control_plane/stf/trex_status.py delete mode 100755 scripts/automation/trex_control_plane/stf/trex_status_e.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/CCustomLogger.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/__init__.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/external_packages.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/general_utils.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/outer_packages.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/text_opts.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_daemon_server.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_exceptions.py create mode 100644 scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status_e.py (limited to 'scripts/automation/trex_control_plane/stf') diff --git a/scripts/automation/trex_control_plane/stf/CCustomLogger.py b/scripts/automation/trex_control_plane/stf/CCustomLogger.py deleted file mode 100755 index ecf7d519..00000000 --- a/scripts/automation/trex_control_plane/stf/CCustomLogger.py +++ /dev/null @@ -1,100 +0,0 @@ - -import sys -import os -import logging - - -def setup_custom_logger(name, log_path = None): - # first make sure path availabe -# if log_path is None: -# log_path = os.getcwd()+'/trex_log.log' -# else: -# directory = os.path.dirname(log_path) -# if not os.path.exists(directory): -# os.makedirs(directory) - logging.basicConfig(level = logging.INFO, - format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', - datefmt = '%m-%d %H:%M') -# filename= log_path, -# filemode= 'w') -# -# # define a Handler which writes INFO messages or higher to the sys.stderr -# consoleLogger = logging.StreamHandler() -# consoleLogger.setLevel(logging.ERROR) -# # set a format which is simpler for console use -# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') -# # tell the handler to use this format -# consoleLogger.setFormatter(formatter) -# -# # add the handler to the logger -# logging.getLogger(name).addHandler(consoleLogger) - -def setup_daemon_logger (name, log_path = None): - # first make sure path availabe - logging.basicConfig(level = logging.INFO, - format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', - datefmt = '%m-%d %H:%M', - filename= log_path, - filemode= 'w') - -class CustomLogger(object): - - def __init__(self, log_filename): - # Store the original stdout and stderr - sys.stdout.flush() - sys.stderr.flush() - - self.stdout_fd = os.dup(sys.stdout.fileno()) - self.devnull = os.open('/dev/null', os.O_WRONLY) - self.log_file = open(log_filename, 'w') - self.silenced = False - self.pending_log_file_prints = 0 - - # silence all prints from stdout - def silence(self): - os.dup2(self.devnull, sys.stdout.fileno()) - self.silenced = True - - # restore stdout status - def restore(self): - sys.stdout.flush() - sys.stderr.flush() - # Restore normal stdout - os.dup2(self.stdout_fd, sys.stdout.fileno()) - self.silenced = False - - #print a message to the log (both stdout / log file) - def log(self, text, force = False, newline = True): - self.log_file.write((text + "\n") if newline else text) - self.pending_log_file_prints += 1 - - if (self.pending_log_file_prints >= 10): - self.log_file.flush() - self.pending_log_file_prints = 0 - - self.console(text, force, newline) - - # print a message to the console alone - def console(self, text, force = False, newline = True): - _text = (text + "\n") if newline else text - # if we are silenced and not forced - go home - if self.silenced and not force: - return - - if self.silenced: - os.write(self.stdout_fd, _text) - else: - sys.stdout.write(_text) - - sys.stdout.flush() - - # flush - def flush(self): - sys.stdout.flush() - self.log_file.flush() - - def __exit__(self, type, value, traceback): - sys.stdout.flush() - self.log_file.flush() - os.close(self.devnull) - os.close(self.log_file) diff --git a/scripts/automation/trex_control_plane/stf/__init__.py b/scripts/automation/trex_control_plane/stf/__init__.py deleted file mode 100755 index 5a1da046..00000000 --- a/scripts/automation/trex_control_plane/stf/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ["trex_status_e", "trex_exceptions"] diff --git a/scripts/automation/trex_control_plane/stf/examples/stf_example.py b/scripts/automation/trex_control_plane/stf/examples/stf_example.py new file mode 100755 index 00000000..f6ebffe7 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/examples/stf_example.py @@ -0,0 +1,53 @@ +import argparse +import stf_path +from trex_stf_lib.trex_client import CTRexClient + +# sample TRex stateless run +# assuming server daemon is running. + +def minimal_stateful_test(server): + print('Connecting to %s' % server) + trex_client = CTRexClient(server) + + print('Connected, start TRex') + trex_client.start_trex( + c = 1, + m = 700, + f = 'cap2/http_simple.yaml', + d = 5, + l = 1000, + trex_development = True, + ) + + print('Sample until end') + result = trex_client.sample_to_run_finish() + + print('Test results:') + print(result) + + print('TX by ports:') + tx_ptks_dict = result.get_last_value('trex-global.data', 'opackets-*') + print(' | '.join(['%s: %s' % (k.split('-')[-1], tx_ptks_dict[k]) for k in sorted(tx_ptks_dict.keys())])) + + print('RX by ports:') + rx_ptks_dict = result.get_last_value('trex-global.data', 'ipackets-*') + print(' | '.join(['%s: %s' % (k.split('-')[-1], rx_ptks_dict[k]) for k in sorted(rx_ptks_dict.keys())])) + + print('CPU utilization:') + print(result.get_value_list('trex-global.data.m_cpu_util')) + + #print('Dump of *latest* result sample, uncomment to see it all') + #print(result.get_latest_dump()) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Example for TRex Stateful, assuming server daemon is running.") + parser.add_argument('-s', '--server', + dest='server', + help='Remote trex address', + default='127.0.0.1', + type = str) + args = parser.parse_args() + + minimal_stateful_test(args.server) + diff --git a/scripts/automation/trex_control_plane/stf/examples/stf_path.py b/scripts/automation/trex_control_plane/stf/examples/stf_path.py new file mode 100755 index 00000000..bb401148 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/examples/stf_path.py @@ -0,0 +1,4 @@ +import sys + +# FIXME to the write path for trex_stf_lib +sys.path.insert(0, "../") diff --git a/scripts/automation/trex_control_plane/stf/external_packages.py b/scripts/automation/trex_control_plane/stf/external_packages.py deleted file mode 100755 index 7353c397..00000000 --- a/scripts/automation/trex_control_plane/stf/external_packages.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/router/bin/python - -import sys -import os - -CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) -ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory -PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) - -CLIENT_UTILS_MODULES = ['yaml-3.11' - ] - -def import_common_modules(): - # must be in a higher priority - sys.path.insert(0, PATH_TO_PYTHON_LIB) - sys.path.append(ROOT_PATH) - import_module_list(CLIENT_UTILS_MODULES) - - -def import_module_list(modules_list): - assert(isinstance(modules_list, list)) - for p in modules_list: - full_path = os.path.join(PATH_TO_PYTHON_LIB, p) - fix_path = os.path.normcase(full_path) - sys.path.insert(1, full_path) - -import_common_modules() - diff --git a/scripts/automation/trex_control_plane/stf/general_utils.py b/scripts/automation/trex_control_plane/stf/general_utils.py deleted file mode 100755 index d2521f02..00000000 --- a/scripts/automation/trex_control_plane/stf/general_utils.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/router/bin/python - -import sys -import site -import string -import random -import os - -try: - import pwd -except ImportError: - import getpass - pwd = None - -using_python_3 = True if sys.version_info.major == 3 else False - - -def user_input(): - if using_python_3: - return input() - else: - # using python version 2 - return raw_input() - -def get_current_user(): - if pwd: - return pwd.getpwuid(os.geteuid()).pw_name - else: - return getpass.getuser() - -def import_module_list_by_path (modules_list): - assert(isinstance(modules_list, list)) - for full_path in modules_list: - site.addsitedir(full_path) - -def find_path_to_pardir (pardir, base_path = os.getcwd() ): - """ - Finds the absolute path for some parent dir `pardir`, starting from base_path - - The request is only valid if the stop initiator is the same client as the TRex run initiator. - - :parameters: - pardir : str - name of an upper-level directory to which we want to find an absolute path for - base_path : str - a full (usually nested) path from which we want to find a parent folder. - - default value : **current working dir** - - :return: - string representation of the full path to - - """ - components = base_path.split(os.sep) - return str.join(os.sep, components[:components.index(pardir)+1]) - - -def random_id_gen(length=8): - """ - A generator for creating a random chars id of specific length - - :parameters: - length : int - the desired length of the generated id - - default: 8 - - :return: - a random id with each next() request. - """ - id_chars = string.ascii_lowercase + string.digits - while True: - return_id = '' - for i in range(length): - return_id += random.choice(id_chars) - yield return_id - -def id_count_gen(): - """ - A generator for creating an increasing id for objects, starting from 0 - - :parameters: - None - - :return: - an id (unsigned int) with each next() request. - """ - return_id = 0 - while True: - yield return_id - return_id += 1 - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/stf/outer_packages.py b/scripts/automation/trex_control_plane/stf/outer_packages.py deleted file mode 100755 index 5e29f8d6..00000000 --- a/scripts/automation/trex_control_plane/stf/outer_packages.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/router/bin/python - -import sys -import os - - -CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) -PARENT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, 'external_libs')) -SCRIPTS_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, 'external_libs')) - -CLIENT_MODULES = ['enum34-1.0.4', - 'jsonrpclib-pelix-0.2.5', -# 'termstyle', -# 'yaml-3.11' - ] - - -def import_module_list(ext_libs_path): - for p in CLIENT_MODULES: - full_path = os.path.join(ext_libs_path, p) - if not os.path.exists(full_path): - raise Exception('Library %s is absent in path %s' % (p, ext_libs_path)) - sys.path.insert(1, full_path) - -if os.path.exists(PARENT_PATH): - import_module_list(PARENT_PATH) -elif os.path.exists(SCRIPTS_PATH): - import_module_list(SCRIPTS_PATH) -else: - raise Exception('Could not find external libs in path: %s' % [PARENT_PATH, SCRIPTS_PATH]) diff --git a/scripts/automation/trex_control_plane/stf/text_opts.py b/scripts/automation/trex_control_plane/stf/text_opts.py deleted file mode 100755 index 78a0ab1f..00000000 --- a/scripts/automation/trex_control_plane/stf/text_opts.py +++ /dev/null @@ -1,192 +0,0 @@ -import json -import re - -TEXT_CODES = {'bold': {'start': '\x1b[1m', - 'end': '\x1b[22m'}, - 'cyan': {'start': '\x1b[36m', - 'end': '\x1b[39m'}, - 'blue': {'start': '\x1b[34m', - 'end': '\x1b[39m'}, - 'red': {'start': '\x1b[31m', - 'end': '\x1b[39m'}, - 'magenta': {'start': '\x1b[35m', - 'end': '\x1b[39m'}, - 'green': {'start': '\x1b[32m', - 'end': '\x1b[39m'}, - 'yellow': {'start': '\x1b[33m', - 'end': '\x1b[39m'}, - 'underline': {'start': '\x1b[4m', - 'end': '\x1b[24m'}} - -class TextCodesStripper: - keys = [re.escape(v['start']) for k,v in TEXT_CODES.iteritems()] - keys += [re.escape(v['end']) for k,v in TEXT_CODES.iteritems()] - pattern = re.compile("|".join(keys)) - - @staticmethod - def strip (s): - return re.sub(TextCodesStripper.pattern, '', s) - -def format_num (size, suffix = "", compact = True, opts = ()): - txt = "NaN" - - if type(size) == str: - return "N/A" - - u = '' - - if compact: - for unit in ['','K','M','G','T','P']: - if abs(size) < 1000.0: - u = unit - break - size /= 1000.0 - - if isinstance(size, float): - txt = "%3.2f" % (size) - else: - txt = "{:,}".format(size) - - if u or suffix: - txt += " {:}{:}".format(u, suffix) - - if isinstance(opts, tuple): - return format_text(txt, *opts) - else: - return format_text(txt, (opts)) - - - -def format_time (t_sec): - if t_sec < 0: - return "infinite" - - if t_sec < 1: - # low numbers - for unit in ['ms', 'usec', 'ns']: - t_sec *= 1000.0 - if t_sec >= 1.0: - return '{:,.2f} [{:}]'.format(t_sec, unit) - - return "NaN" - - else: - # seconds - if t_sec < 60.0: - return '{:,.2f} [{:}]'.format(t_sec, 'sec') - - # minutes - t_sec /= 60.0 - if t_sec < 60.0: - return '{:,.2f} [{:}]'.format(t_sec, 'minutes') - - # hours - t_sec /= 60.0 - if t_sec < 24.0: - return '{:,.2f} [{:}]'.format(t_sec, 'hours') - - # days - t_sec /= 24.0 - return '{:,.2f} [{:}]'.format(t_sec, 'days') - - -def format_percentage (size): - return "%0.2f %%" % (size) - -def bold(text): - return text_attribute(text, 'bold') - - -def cyan(text): - return text_attribute(text, 'cyan') - - -def blue(text): - return text_attribute(text, 'blue') - - -def red(text): - return text_attribute(text, 'red') - - -def magenta(text): - return text_attribute(text, 'magenta') - - -def green(text): - return text_attribute(text, 'green') - -def yellow(text): - return text_attribute(text, 'yellow') - -def underline(text): - return text_attribute(text, 'underline') - - -def text_attribute(text, attribute): - if isinstance(text, str): - return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], - txt=text, - stop=TEXT_CODES[attribute]['end']) - elif isinstance(text, unicode): - return u"{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], - txt=text, - stop=TEXT_CODES[attribute]['end']) - else: - raise Exception("not a string") - - -FUNC_DICT = {'blue': blue, - 'bold': bold, - 'green': green, - 'yellow': yellow, - 'cyan': cyan, - 'magenta': magenta, - 'underline': underline, - 'red': red} - - -def format_text(text, *args): - return_string = text - for i in args: - func = FUNC_DICT.get(i) - if func: - return_string = func(return_string) - - return return_string - - -def format_threshold (value, red_zone, green_zone): - if value >= red_zone[0] and value <= red_zone[1]: - return format_text("{0}".format(value), 'red') - - if value >= green_zone[0] and value <= green_zone[1]: - return format_text("{0}".format(value), 'green') - - return "{0}".format(value) - -# pretty print for JSON -def pretty_json (json_str, use_colors = True): - pretty_str = json.dumps(json.loads(json_str), indent = 4, separators=(',', ': '), sort_keys = True) - - if not use_colors: - return pretty_str - - try: - # int numbers - pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*[^.])',r'\1{0}'.format(blue(r'\2')), pretty_str) - # float - pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*\.[0-9]+)',r'\1{0}'.format(magenta(r'\2')), pretty_str) - # # strings - # - pretty_str = re.sub(r'([ ]*:[ ]+)("[^"]*")',r'\1{0}'.format(red(r'\2')), pretty_str) - pretty_str = re.sub(r"('[^']*')", r'{0}\1{1}'.format(TEXT_CODES['magenta']['start'], - TEXT_CODES['red']['start']), pretty_str) - except : - pass - - return pretty_str - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/stf/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_client.py deleted file mode 100755 index 919253d1..00000000 --- a/scripts/automation/trex_control_plane/stf/trex_client.py +++ /dev/null @@ -1,1216 +0,0 @@ -#!/router/bin/python - -# internal libs -import sys -import os -import socket -import errno -import time -import re -import copy -import binascii -from distutils.util import strtobool -from collections import deque, OrderedDict -from json import JSONDecoder -import traceback - -try: - from . import outer_packages - from .trex_status_e import TRexStatus - from .trex_exceptions import * - from .trex_exceptions import exception_handler - from .general_utils import * -except Exception as e: # is __main__ - import outer_packages - from trex_status_e import TRexStatus - from trex_exceptions import * - from trex_exceptions import exception_handler - from general_utils import * - -# external libs -import jsonrpclib -from jsonrpclib import ProtocolError, AppError -from enum import Enum - - - -class CTRexClient(object): - """ - This class defines the client side of the RESTfull interaction with TRex - """ - - def __init__(self, trex_host, max_history_size = 100, trex_daemon_port = 8090, trex_zmq_port = 4500, verbose = False): - """ - Instantiate a TRex client object, and connecting it to listening daemon-server - - :parameters: - trex_host : str - a string of the TRex ip address or hostname. - max_history_size : int - a number to set the maximum history size of a single TRex run. Each sampling adds a new item to history. - - default value : **100** - trex_daemon_port : int - the port number on which the trex-daemon server can be reached - - default value: **8090** - trex_zmq_port : int - the port number on which trex's zmq module will interact with daemon server - - default value: **4500** - verbose : bool - sets a verbose output on supported class method. - - default value : **False** - - :raises: - socket errors, in case server could not be reached. - - """ - try: - self.trex_host = socket.gethostbyname(trex_host) - except: # give it another try - self.trex_host = socket.gethostbyname(trex_host) - self.trex_daemon_port = trex_daemon_port - self.trex_zmq_port = trex_zmq_port - self.seq = None - self.verbose = verbose - self.result_obj = CTRexResult(max_history_size) - self.decoder = JSONDecoder() - self.trex_server_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port ) - self.__verbose_print("Connecting to TRex @ {trex_path} ...".format( trex_path = self.trex_server_path ) ) - self.history = jsonrpclib.history.History() - self.server = jsonrpclib.Server(self.trex_server_path, history = self.history) - self.check_server_connectivity() - self.__verbose_print("Connection established successfully!") - self._last_sample = time.time() - self.__default_user = get_current_user() - - - def add (self, x, y): - try: - return self.server.add(x,y) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def start_trex (self, f, d, block_to_success = True, timeout = 40, user = None, trex_development = False, **trex_cmd_options): - """ - Request to start a TRex run on server. - - :parameters: - f : str - a path (on server) for the injected traffic data (.yaml file) - d : int - the desired duration of the test. must be at least 30 seconds long. - block_to_success : bool - determine if this method blocks until TRex changes state from 'Starting' to either 'Idle' or 'Running' - - default value : **True** - timeout : int - maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' - - default value: **40** - user : str - the identity of the the run issuer. - trex_cmd_options : key, val - sets desired TRex options using key=val syntax, separated by comma. - for keys with no value, state key=True - - :return: - **True** on success - - :raises: - + :exc:`ValueError`, in case 'd' parameter inserted with wrong value. - + :exc:`trex_exceptions.TRexError`, in case one of the trex_cmd_options raised an exception at server. - + :exc:`trex_exceptions.TRexInUseError`, in case TRex is already taken. - + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying start TRex. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - user = user or self.__default_user - try: - d = int(d) - if d < 30 and not trex_development: # test duration should be at least 30 seconds, unless trex_development flag is specified. - raise ValueError - except ValueError: - raise ValueError('d parameter must be integer, specifying how long TRex run, and must be larger than 30 secs.') - - trex_cmd_options.update( {'f' : f, 'd' : d} ) - if not trex_cmd_options.get('l'): - self.result_obj.latency_checked = False - if 'k' in trex_cmd_options: - timeout += int(trex_cmd_options['k']) # during 'k' seconds TRex stays in 'Starting' state - - self.result_obj.clear_results() - try: - issue_time = time.time() - retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - if retval!=0: - self.seq = retval # update seq num only on successful submission - return True - else: # TRex is has been started by another user - raise TRexInUseError('TRex is already being used by another user or process. Try again once TRex is back in IDLE state.') - - def stop_trex (self): - """ - Request to stop a TRex run on server. - - The request is only valid if the stop initiator is the same client as the TRex run initiator. - - :parameters: - None - - :return: - + **True** on successful termination - + **False** if request issued but TRex wasn't running. - - :raises: - + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex ir running but started by another user. - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - return self.server.stop_trex(self.seq) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def force_kill (self, confirm = True): - """ - Force killing of running TRex process (if exists) on the server. - - .. tip:: This method is a safety method and **overrides any running or reserved resources**, and as such isn't designed to be used on a regular basis. - Always consider using :func:`trex_client.CTRexClient.stop_trex` instead. - - In the end of this method, TRex will return to IDLE state with no reservation. - - :parameters: - confirm : bool - Prompt a user confirmation before continue terminating TRex session - - :return: - + **True** on successful termination - + **False** otherwise. - - :raises: - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - if confirm: - prompt = "WARNING: This will terminate active TRex session indiscriminately.\nAre you sure? " - sys.stdout.write('%s [y/n]\n' % prompt) - while True: - try: - if strtobool(user_input().lower()): - break - else: - return - except ValueError: - sys.stdout.write('Please respond with \'y\' or \'n\'.\n') - try: - return self.server.force_trex_kill() - except AppError as err: - # Silence any kind of application errors- by design - return False - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def wait_until_kickoff_finish(self, timeout = 40): - """ - Block the client application until TRex changes state from 'Starting' to either 'Idle' or 'Running' - - The request is only valid if the stop initiator is the same client as the TRex run initiator. - - :parameters: - timeout : int - maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' - - :return: - + **True** on successful termination - + **False** if request issued but TRex wasn't running. - - :raises: - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + ProtocolError, in case of error in JSON-RPC protocol. - - .. note:: Exceptions are throws only when start_trex did not block in the first place, i.e. `block_to_success` parameter was set to `False` - - """ - - try: - return self.server.wait_until_kickoff_finish(timeout) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def is_running (self, dump_out = False): - """ - Poll for TRex running status. - - If TRex is running, a history item will be added into result_obj and processed. - - .. tip:: This method is especially useful for iterating until TRex run is finished. - - :parameters: - dump_out : dict - if passed, the pointer object is cleared and the latest dump stored in it. - - :return: - + **True** if TRex is running. - + **False** if TRex is not running. - - :raises: - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + :exc:`TypeError`, in case JSON stream decoding error. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - res = self.get_running_info() - if res == {}: - return False - if (dump_out != False) and (isinstance(dump_out, dict)): # save received dump to given 'dump_out' pointer - dump_out.clear() - dump_out.update(res) - return True - except TRexWarning as err: - if err.code == -12: # TRex is either still at 'Starting' state or in Idle state, however NO error occured - return False - except TRexException: - raise - except ProtocolError as err: - raise - finally: - self.prompt_verbose_data() - - def is_idle (self): - """ - Poll for TRex running status, check if TRex is in Idle state. - - :parameters: - None - - :return: - + **True** if TRex is idle. - + **False** if TRex is starting or running. - - :raises: - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + :exc:`TypeError`, in case JSON stream decoding error. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - if self.get_running_status()['state'] == TRexStatus.Idle: - return True - return False - except TRexException: - raise - except ProtocolError as err: - raise - finally: - self.prompt_verbose_data() - - def get_trex_files_path (self): - """ - Fetches the local path in which files are stored when pushed to TRex server from client. - - :parameters: - None - - :return: - string representation of the desired path - - .. note:: The returned path represents a path on the TRex server **local machine** - - :raises: - ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - return (self.server.get_files_path() + '/') - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def get_running_status (self): - """ - Fetches the current TRex status. - - If available, a verbose data will accompany the state itself. - - :parameters: - None - - :return: - dictionary with 'state' and 'verbose' keys. - - :raises: - ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - res = self.server.get_running_status() - res['state'] = TRexStatus(res['state']) - return res - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def get_running_info (self): - """ - Performs single poll of TRex running data and process it into the result object (named `result_obj`). - - .. tip:: This method will throw an exception if TRex isn't running. Always consider using :func:`trex_client.CTRexClient.is_running` which handles a single poll operation in safer manner. - - :parameters: - None - - :return: - dictionary containing the most updated data dump from TRex. - - :raises: - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + :exc:`TypeError`, in case JSON stream decoding error. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - if not self.is_query_relevance(): - # if requested in timeframe smaller than the original sample rate, return the last known data without interacting with server - return self.result_obj.get_latest_dump() - else: - try: - latest_dump = self.decoder.decode( self.server.get_running_info() ) # latest dump is not a dict, but json string. decode it. - self.result_obj.update_result_data(latest_dump) - return latest_dump - except TypeError as inst: - raise TypeError('JSON-RPC data decoding failed. Check out incoming JSON stream.') - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def sample_until_condition (self, condition_func, time_between_samples = 5): - """ - Automatically sets ongoing sampling of TRex data, with sampling rate described by time_between_samples. - - On each fetched dump, the condition_func is applied on the result objects, and if returns True, the sampling will stop. - - :parameters: - condition_func : function - function that operates on result_obj and checks if a condition has been met - - .. note:: `condition_finc` is applied on `CTRexResult` object. Make sure to design a relevant method. - time_between_samples : int - determines the time between each sample of the server - - default value : **5** - - :return: - the first result object (see :class:`CTRexResult` for further details) of the TRex run on which the condition has been met. - - :raises: - + :exc:`UserWarning`, in case the condition_func method condition hasn't been met - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + :exc:`TypeError`, in case JSON stream decoding error. - + ProtocolError, in case of error in JSON-RPC protocol. - + :exc:`Exception`, in case the condition_func suffered from any kind of exception - - """ - # make sure TRex is running. raise exceptions here if any - self.wait_until_kickoff_finish() - try: - while self.is_running(): - results = self.get_result_obj() - if condition_func(results): - # if condition satisfied, stop TRex and return result object - self.stop_trex() - return results - time.sleep(time_between_samples) - except TRexWarning: - # means we're back to Idle state, and didn't meet our condition - raise UserWarning("TRex results condition wasn't met during TRex run.") - except Exception: - # this could come from provided method 'condition_func' - raise - - def sample_to_run_finish (self, time_between_samples = 5): - """ - Automatically sets automatically sampling of TRex data with sampling rate described by time_between_samples until TRex run finished. - - :parameters: - time_between_samples : int - determines the time between each sample of the server - - default value : **5** - - :return: - the latest result object (see :class:`CTRexResult` for further details) with sampled data. - - :raises: - + :exc:`UserWarning`, in case the condition_func method condition hasn't been met - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + :exc:`TypeError`, in case JSON stream decoding error. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - self.wait_until_kickoff_finish() - - try: - while self.is_running(): - time.sleep(time_between_samples) - except TRexWarning: - pass - results = self.get_result_obj() - return results - - def sample_x_seconds (self, sample_time, time_between_samples = 5): - """ - Automatically sets ongoing sampling of TRex data for sample_time seconds, with sampling rate described by time_between_samples. - Does not stop the TRex afterwards! - - .. tip:: Useful for changing the device (Router, ASA etc.) configuration after given time. - - :parameters: - sample_time : int - sample the TRex this number of seconds - - time_between_samples : int - determines the time between each sample of the server - - default value : **5** - - :return: - the first result object (see :class:`CTRexResult` for further details) of the TRex run after given sample_time. - - :raises: - + :exc:`UserWarning`, in case the TRex run ended before sample_time duration - + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). - + :exc:`TypeError`, in case JSON stream decoding error. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - # make sure TRex is running. raise exceptions here if any - self.wait_until_kickoff_finish() - elapsed_time = 0 - while self.is_running(): - if elapsed_time >= sample_time: - return self.get_result_obj() - time.sleep(time_between_samples) - elapsed_time += time_between_samples - raise UserWarning("TRex has stopped at %s seconds (before expected %s seconds)\nTry increasing test duration or decreasing sample_time" % (elapsed_time, sample_time)) - - def get_result_obj (self, copy_obj = True): - """ - Returns the result object of the trex_client's instance. - - By default, returns a **copy** of the objects (so that changes to the original object are masked). - - :parameters: - copy_obj : bool - False means that a reference to the original (possibly changing) object are passed - - defaul value : **True** - - :return: - the latest result object (see :class:`CTRexResult` for further details) with sampled data. - - """ - if copy_obj: - return copy.deepcopy(self.result_obj) - else: - return self.result_obj - - def is_reserved (self): - """ - Checks if TRex is currently reserved to any user or not. - - :parameters: - None - - :return: - + **True** if TRex is reserved. - + **False** otherwise. - - :raises: - ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - return self.server.is_reserved() - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def get_trex_daemon_log (self): - """ - Get Trex daemon log. - - :return: - String representation of TRex daemon log - - :raises: - + :exc:`trex_exceptions.TRexRequestDenied`, in case file could not be read. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - return binascii.a2b_base64(self.server.get_trex_daemon_log()) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def get_trex_log (self): - """ - Get TRex CLI output log - - :return: - String representation of TRex log - - :raises: - + :exc:`trex_exceptions.TRexRequestDenied`, in case file could not be fetched at server side. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - try: - return binascii.a2b_base64(self.server.get_trex_log()) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def get_trex_version (self): - """ - Get TRex version details. - - :return: - Trex details (Version, User, Date, Uuid, Git SHA) as ordered dictionary - - :raises: - + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex version could not be determined. - + ProtocolError, in case of error in JSON-RPC protocol. - + General Exception is case one of the keys is missing in response - """ - - try: - version_dict = OrderedDict() - result_lines = binascii.a2b_base64(self.server.get_trex_version()).split('\n') - for line in result_lines: - if not line: - continue - key, value = line.strip().split(':', 1) - version_dict[key.strip()] = value.strip() - for key in ('Version', 'User', 'Date', 'Uuid', 'Git SHA'): - if key not in version_dict: - raise Exception('get_trex_version: got server response without key: {0}'.format(key)) - return version_dict - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def reserve_trex (self, user = None): - """ - Reserves the usage of TRex to a certain user. - - When TRex is reserved, it can't be reserved. - - :parameters: - user : str - a username of the desired owner of TRex - - default: current logged user - - :return: - **True** if reservation made successfully - - :raises: - + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying to make the reservation. - + :exc:`trex_exceptions.TRexInUseError`, in case TRex is currently running. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - username = user or self.__default_user - try: - return self.server.reserve_trex(user = username) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def cancel_reservation (self, user = None): - """ - Cancels a current reservation of TRex to a certain user. - - When TRex is reserved, no other user can start new TRex runs. - - - :parameters: - user : str - a username of the desired owner of TRex - - default: current logged user - - :return: - + **True** if reservation canceled successfully, - + **False** if there was no reservation at all. - - :raises: - + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying to cancel the reservation. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - - username = user or self.__default_user - try: - return self.server.cancel_reservation(user = username) - except AppError as err: - self._handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def push_files (self, filepaths): - """ - Pushes a file (or a list of files) to store locally on server. - - :parameters: - filepaths : str or list - a path to a file to be pushed to server. - if a list of paths is passed, all of those will be pushed to server - - :return: - + **True** if file(s) copied successfully. - + **False** otherwise. - - :raises: - + :exc:`IOError`, in case specified file wasn't found or could not be accessed. - + ProtocolError, in case of error in JSON-RPC protocol. - - """ - paths_list = None - if isinstance(filepaths, str): - paths_list = [filepaths] - elif isinstance(filepaths, list): - paths_list = filepaths - else: - raise TypeError("filepaths argument must be of type str or list") - - for filepath in paths_list: - try: - if not os.path.exists(filepath): - raise IOError(errno.ENOENT, "The requested `{fname}` file wasn't found. Operation aborted.".format( - fname = filepath) ) - else: - filename = os.path.basename(filepath) - with open(filepath, 'rb') as f: - file_content = f.read() - self.server.push_file(filename, binascii.b2a_base64(file_content)) - finally: - self.prompt_verbose_data() - return True - - def is_query_relevance(self): - """ - Checks if time between any two consecutive server queries (asking for live running data) passed. - - .. note:: The allowed minimum time between each two consecutive samples is 0.5 seconds. - - :parameters: - None - - :return: - + **True** if more than 0.5 seconds has been past from last server query. - + **False** otherwise. - - """ - cur_time = time.time() - if cur_time-self._last_sample < 0.5: - return False - else: - self._last_sample = cur_time - return True - - def call_server_mathod_safely (self, method_to_call): - try: - return method_to_call() - except socket.error as e: - if e.errno == errno.ECONNREFUSED: - raise SocketError(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") - - def check_server_connectivity (self): - """ - Checks for server valid connectivity. - """ - try: - socket.gethostbyname(self.trex_host) - return self.server.connectivity_check() - except socket.gaierror as e: - raise socket.gaierror(e.errno, "Could not resolve server hostname. Please make sure hostname entered correctly.") - except socket.error as e: - if e.errno == errno.ECONNREFUSED: - raise socket.error(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") - finally: - self.prompt_verbose_data() - - def prompt_verbose_data(self): - """ - This method prompts any verbose data available, only if `verbose` option has been turned on. - """ - if self.verbose: - print ('\n') - print ("(*) JSON-RPC request:", self.history.request) - print ("(*) JSON-RPC response:", self.history.response) - - def __verbose_print(self, print_str): - """ - This private method prints the `print_str` string only in case self.verbose flag is turned on. - - :parameters: - print_str : str - a string to be printed - - :returns: - None - """ - if self.verbose: - print (print_str) - - - - def _handle_AppError_exception(self, err): - """ - This private method triggres the TRex dedicated exception generation in case a general ProtocolError has been raised. - """ - # handle known exceptions based on known error codes. - # if error code is not known, raise ProtocolError - raise exception_handler.gen_exception(err) - - -class CTRexResult(object): - """ - A class containing all results received from TRex. - - Ontop to containing the results, this class offers easier data access and extended results processing options - """ - def __init__(self, max_history_size): - """ - Instatiate a TRex result object - - :parameters: - max_history_size : int - a number to set the maximum history size of a single TRex run. Each sampling adds a new item to history. - - """ - self._history = deque(maxlen = max_history_size) - self.clear_results() - self.latency_checked = True - - def __repr__(self): - return ("Is valid history? {arg}\n".format( arg = self.is_valid_hist() ) + - "Done warmup? {arg}\n".format( arg = self.is_done_warmup() ) + - "Expected tx rate: {arg}\n".format( arg = self.get_expected_tx_rate() ) + - "Current tx rate: {arg}\n".format( arg = self.get_current_tx_rate() ) + - "Maximum latency: {arg}\n".format( arg = self.get_max_latency() ) + - "Average latency: {arg}\n".format( arg = self.get_avg_latency() ) + - "Average window latency: {arg}\n".format( arg = self.get_avg_window_latency() ) + - "Total drops: {arg}\n".format( arg = self.get_total_drops() ) + - "Drop rate: {arg}\n".format( arg = self.get_drop_rate() ) + - "History size so far: {arg}\n".format( arg = len(self._history) ) ) - - def get_expected_tx_rate (self): - """ - Fetches the expected TX rate in various units representation - - :parameters: - None - - :return: - dictionary containing the expected TX rate, where the key is the measurement units, and the value is the measurement value. - - """ - return self._expected_tx_rate - - def get_current_tx_rate (self): - """ - Fetches the current TX rate in various units representation - - :parameters: - None - - :return: - dictionary containing the current TX rate, where the key is the measurement units, and the value is the measurement value. - - """ - return self._current_tx_rate - - def get_max_latency (self): - """ - Fetches the maximum latency measured on each of the interfaces - - :parameters: - None - - :return: - dictionary containing the maximum latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. - - """ - return self._max_latency - - def get_avg_latency (self): - """ - Fetches the average latency measured on each of the interfaces from the start of TRex run - - :parameters: - None - - :return: - dictionary containing the average latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. - - The `all` key represents the average of all interfaces' average - - """ - return self._avg_latency - - def get_avg_window_latency (self): - """ - Fetches the average latency measured on each of the interfaces from all the sampled currently stored in window. - - :parameters: - None - - :return: - dictionary containing the average latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. - - The `all` key represents the average of all interfaces' average - - """ - return self._avg_window_latency - - def get_total_drops (self): - """ - Fetches the total number of drops identified from the moment TRex run began. - - :parameters: - None - - :return: - total drops count (as int) - - """ - return self._total_drops - - def get_drop_rate (self): - """ - Fetches the most recent drop rate in pkts/sec units. - - :parameters: - None - - :return: - current drop rate (as float) - - """ - return self._drop_rate - - def is_valid_hist (self): - """ - Checks if result obejct contains valid data. - - :parameters: - None - - :return: - + **True** if history is valid. - + **False** otherwise. - - """ - return self.valid - - def set_valid_hist (self, valid_stat = True): - """ - Sets result obejct validity status. - - :parameters: - valid_stat : bool - defines the validity status - - dafault value : **True** - - :return: - None - - """ - self.valid = valid_stat - - def is_done_warmup (self): - """ - Checks if TRex latest results TX-rate indicates that TRex has reached its expected TX-rate. - - :parameters: - None - - :return: - + **True** if expected TX-rate has been reached. - + **False** otherwise. - - """ - return self._done_warmup - - def get_last_value (self, tree_path_to_key, regex = None): - """ - A dynamic getter from the latest sampled data item stored in the result object. - - :parameters: - tree_path_to_key : str - defines a path to desired data. - - .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. - | Use '[i]' to access the i'th indexed object of an array. - - tree_path_to_key : regex - apply a regex to filter results out from a multiple results set. - - Filter applies only on keys of dictionary type. - - dafault value : **None** - - :return: - + a list of values relevant to the specified path - + None if no results were fetched or the history isn't valid. - - """ - if not self.is_valid_hist(): - return None - else: - return CTRexResult.__get_value_by_path(self._history[len(self._history)-1], tree_path_to_key, regex) - - def get_value_list (self, tree_path_to_key, regex = None, filter_none = True): - """ - A dynamic getter from all sampled data items stored in the result object. - - :parameters: - tree_path_to_key : str - defines a path to desired data. - - .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. - | Use '[i]' to access the i'th indexed object of an array. - - tree_path_to_key : regex - apply a regex to filter results out from a multiple results set. - - Filter applies only on keys of dictionary type. - - dafault value : **None** - - filter_none : bool - specify if None results should be filtered out or not. - - dafault value : **True** - - :return: - + a list of values relevant to the specified path. Each item on the list refers to a single server sample. - + None if no results were fetched or the history isn't valid. - """ - - if not self.is_valid_hist(): - return None - else: - raw_list = list( map(lambda x: CTRexResult.__get_value_by_path(x, tree_path_to_key, regex), self._history) ) - if filter_none: - return list (filter(lambda x: x!=None, raw_list) ) - else: - return raw_list - - def get_latest_dump(self): - """ - A getter to the latest sampled data item stored in the result object. - - :parameters: - None - - :return: - + a dictionary of the latest data item - + an empty dictionary if history is empty. - - """ - history_size = len(self._history) - if history_size != 0: - return self._history[len(self._history) - 1] - else: - return {} - - def update_result_data (self, latest_dump): - """ - Integrates a `latest_dump` dictionary into the CTRexResult object. - - :parameters: - latest_dump : dict - a dictionary with the items desired to be integrated into the object history and stats - - :return: - None - - """ - # add latest dump to history - if latest_dump != {}: - self._history.append(latest_dump) - if not self.valid: - self.valid = True - - # parse important fields and calculate averages and others - if self._expected_tx_rate is None: - # get the expected data only once since it doesn't change - self._expected_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_expected_\w+") - - self._current_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_(?!expected_)\w+") - if not self._done_warmup and self._expected_tx_rate is not None: - # check for up to 2% change between expected and actual - if (self._current_tx_rate['m_tx_bps']/self._expected_tx_rate['m_tx_expected_bps'] > 0.98): - self._done_warmup = True - - # handle latency data - if self.latency_checked: - latency_pre = "trex-latency" - self._max_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "max-")#None # TBC - # support old typo - if self._max_latency is None: - latency_pre = "trex-latecny" - self._max_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "max-") - - self._avg_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "avg-")#None # TBC - self._avg_latency = CTRexResult.__avg_all_and_rename_keys(self._avg_latency) - - avg_win_latency_list = self.get_value_list("{latency}.data".format(latency = latency_pre), "avg-") - self._avg_window_latency = CTRexResult.__calc_latency_win_stats(avg_win_latency_list) - - tx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_tx_pkts") - rx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_rx_pkts") - if tx_pkts is not None and rx_pkts is not None: - self._total_drops = tx_pkts - rx_pkts - self._drop_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_rx_drop_bps") - - def clear_results (self): - """ - Clears all results and sets the history's validity to `False` - - :parameters: - None - - :return: - None - - """ - self.valid = False - self._done_warmup = False - self._expected_tx_rate = None - self._current_tx_rate = None - self._max_latency = None - self._avg_latency = None - self._avg_window_latency = None - self._total_drops = None - self._drop_rate = None - self._history.clear() - - @staticmethod - def __get_value_by_path (dct, tree_path, regex = None): - try: - for i, p in re.findall(r'(\d+)|([\w|-]+)', tree_path): - dct = dct[p or int(i)] - if regex is not None and isinstance(dct, dict): - res = {} - for key,val in dct.items(): - match = re.match(regex, key) - if match: - res[key]=val - return res - else: - return dct - except (KeyError, TypeError): - return None - - @staticmethod - def __calc_latency_win_stats (latency_win_list): - res = {'all' : None } - port_dict = {'all' : []} - list( map(lambda x: CTRexResult.__update_port_dict(x, port_dict), latency_win_list) ) - - # finally, calculate everages for each list - res['all'] = float("%.3f" % (sum(port_dict['all'])/float(len(port_dict['all']))) ) - port_dict.pop('all') - for port, avg_list in port_dict.items(): - res[port] = float("%.3f" % (sum(avg_list)/float(len(avg_list))) ) - - return res - - @staticmethod - def __update_port_dict (src_avg_dict, dest_port_dict): - all_list = src_avg_dict.values() - dest_port_dict['all'].extend(all_list) - for key, val in src_avg_dict.items(): - reg_res = re.match("avg-(\d+)", key) - if reg_res: - tmp_key = "port"+reg_res.group(1) - if tmp_key in dest_port_dict: - dest_port_dict[tmp_key].append(val) - else: - dest_port_dict[tmp_key] = [val] - - @staticmethod - def __avg_all_and_rename_keys (src_dict): - res = {} - all_list = src_dict.values() - res['all'] = float("%.3f" % (sum(all_list)/float(len(all_list))) ) - for key, val in src_dict.items(): - reg_res = re.match("avg-(\d+)", key) - if reg_res: - tmp_key = "port"+reg_res.group(1) - res[tmp_key] = val # don't touch original fields values - return res - - - -if __name__ == "__main__": - pass - diff --git a/scripts/automation/trex_control_plane/stf/trex_daemon_server.py b/scripts/automation/trex_control_plane/stf/trex_daemon_server.py deleted file mode 100755 index 9784d42a..00000000 --- a/scripts/automation/trex_control_plane/stf/trex_daemon_server.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python - -import outer_packages -import daemon -from trex_server import do_main_program, trex_parser -import CCustomLogger - -import logging -import time -import sys -import os, errno -import grp -import signal -from daemon import runner -from extended_daemon_runner import ExtendedDaemonRunner -import lockfile -import errno - -class TRexServerApp(object): - def __init__(self): - TRexServerApp.create_working_dirs() - self.stdin_path = '/dev/null' - self.stdout_path = '/dev/tty' # All standard prints will come up from this source. - self.stderr_path = "/var/log/trex/trex_daemon_server.log" # All log messages will come up from this source - self.pidfile_path = '/var/run/trex/trex_daemon_server.pid' - self.pidfile_timeout = 5 # timeout in seconds - - def run(self): - do_main_program() - - - @staticmethod - def create_working_dirs(): - if not os.path.exists('/var/log/trex'): - os.mkdir('/var/log/trex') - if not os.path.exists('/var/run/trex'): - os.mkdir('/var/run/trex') - - - -def main (): - - trex_app = TRexServerApp() - - # setup the logger - default_log_path = '/var/log/trex/trex_daemon_server.log' - - try: - CCustomLogger.setup_daemon_logger('TRexServer', default_log_path) - logger = logging.getLogger('TRexServer') - logger.setLevel(logging.INFO) - formatter = logging.Formatter("%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s") - handler = logging.FileHandler("/var/log/trex/trex_daemon_server.log") - logger.addHandler(handler) - except EnvironmentError, e: - if e.errno == errno.EACCES: # catching permission denied error - print "Launching user must have sudo privileges in order to run TRex daemon.\nTerminating daemon process." - exit(-1) - - daemon_runner = ExtendedDaemonRunner(trex_app, trex_parser) - - #This ensures that the logger file handle does not get closed during daemonization - daemon_runner.daemon_context.files_preserve=[handler.stream] - - try: - if not set(['start', 'stop']).isdisjoint(set(sys.argv)): - print "Logs are saved at: {log_path}".format( log_path = default_log_path ) - daemon_runner.do_action() - - except lockfile.LockTimeout as inst: - logger.error(inst) - print inst - print """ - Please try again once the timeout has been reached. - If this error continues, consider killing the process manually and restart the daemon.""" - - -if __name__ == "__main__": - main() diff --git a/scripts/automation/trex_control_plane/stf/trex_exceptions.py b/scripts/automation/trex_control_plane/stf/trex_exceptions.py deleted file mode 100755 index 0de38411..00000000 --- a/scripts/automation/trex_control_plane/stf/trex_exceptions.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/router/bin/python - -#from rpc_exceptions import RPCExceptionHandler, WrappedRPCError - -from jsonrpclib import Fault, ProtocolError, AppError - -class RPCError(Exception): - """ - This is the general RPC error exception class from which :exc:`trex_exceptions.TRexException` inherits. - - Every exception in this class has as error format according to JSON-RPC convention convention: code, message and data. - - """ - def __init__(self, code, message, remote_data = None): - self.code = code - self.msg = message or self._default_message - self.data = remote_data - self.args = (code, self.msg, remote_data) - - def __str__(self): - return self.__repr__() - def __repr__(self): - if self.args[2] is not None: - return u"[errcode:%r] %r. Extended data: %r" % (self.args[0], self.args[1], self.args[2]) - else: - return u"[errcode:%r] %r" % (self.args[0], self.args[1]) - -class TRexException(RPCError): - """ - This is the most general TRex exception. - - All exceptions inherits from this class has an error code and a default message which describes the most common use case of the error. - - This exception isn't used by default and will only when an unrelated to ProtocolError will occur, and it can't be resolved to any of the deriviate exceptions. - - """ - code = -10 - _default_message = 'TRex encountered an unexpected error. please contact TRex dev team.' - # api_name = 'TRex' - -class TRexError(TRexException): - """ - This is the most general TRex exception. - - This exception isn't used by default and will only when an unrelated to ProtocolError will occur, and it can't be resolved to any of the deriviate exceptions. - """ - code = -11 - _default_message = 'TRex run failed due to wrong input parameters, or due to reachability issues.' - -class TRexWarning(TRexException): - """ Indicates a warning from TRex server. When this exception raises it normally used to indicate required data isn't ready yet """ - code = -12 - _default_message = 'TRex is starting (data is not available yet).' - -class TRexRequestDenied(TRexException): - """ Indicates the desired reques was denied by the server """ - code = -33 - _default_message = 'TRex desired request denied because the requested resource is already taken. Try again once TRex is back in IDLE state.' - -class TRexInUseError(TRexException): - """ - Indicates that TRex is currently in use - - """ - code = -13 - _default_message = 'TRex is already being used by another user or process. Try again once TRex is back in IDLE state.' - -class TRexRunFailedError(TRexException): - """ Indicates that TRex has failed due to some reason. This Exception is used when TRex process itself terminates due to unknown reason """ - code = -14 - _default_message = '' - -class TRexIncompleteRunError(TRexException): - """ - Indicates that TRex has failed due to some reason. - This Exception is used when TRex process itself terminated with error fault or it has been terminated by an external intervention in the OS. - - """ - code = -15 - _default_message = 'TRex run was terminated unexpectedly by outer process or by the hosting OS' - -EXCEPTIONS = [TRexException, TRexError, TRexWarning, TRexInUseError, TRexRequestDenied, TRexRunFailedError, TRexIncompleteRunError] - -class CExceptionHandler(object): - """ - CExceptionHandler is responsible for generating TRex API related exceptions in client side. - """ - def __init__(self, exceptions): - """ - Instatiate a CExceptionHandler object - - :parameters: - - exceptions : list - a list of all TRex acceptable exception objects. - - default list: - - :exc:`trex_exceptions.TRexException` - - :exc:`trex_exceptions.TRexError` - - :exc:`trex_exceptions.TRexWarning` - - :exc:`trex_exceptions.TRexInUseError` - - :exc:`trex_exceptions.TRexRequestDenied` - - :exc:`trex_exceptions.TRexRunFailedError` - - :exc:`trex_exceptions.TRexIncompleteRunError` - - """ - if isinstance(exceptions, type): - exceptions = [ exceptions, ] - self.exceptions = exceptions - self.exceptions_dict = dict((e.code, e) for e in self.exceptions) - - def gen_exception (self, err): - """ - Generates an exception based on a general ProtocolError exception object `err`. - - When TRex is reserved, no other user can start new TRex runs. - - - :parameters: - - err : exception - a ProtocolError exception raised by :class:`trex_client.CTRexClient` class - - :return: - A TRex exception from the exception list defined in class creation. - - If such exception wasn't found, returns a TRexException exception - - """ - code, message, data = err - try: - exp = self.exceptions_dict[code] - return exp(exp.code, message, data) - except KeyError: - # revert to TRexException when unknown error application raised - return TRexException(err) - - -exception_handler = CExceptionHandler( EXCEPTIONS ) - diff --git a/scripts/automation/trex_control_plane/stf/trex_status.py b/scripts/automation/trex_control_plane/stf/trex_status.py deleted file mode 100644 index f132720c..00000000 --- a/scripts/automation/trex_control_plane/stf/trex_status.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/router/bin/python - -# define the states in which a T-Rex can hold during its lifetime -# TRexStatus = Enum('TRexStatus', 'Idle Starting Running') - -IDLE = 1 -STARTING = 2 -RUNNING = 3 diff --git a/scripts/automation/trex_control_plane/stf/trex_status_e.py b/scripts/automation/trex_control_plane/stf/trex_status_e.py deleted file mode 100755 index 79a25acc..00000000 --- a/scripts/automation/trex_control_plane/stf/trex_status_e.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/router/bin/python - -try: - from . import outer_packages -except: - import outer_packages -from enum import Enum - - -# define the states in which a TRex can hold during its lifetime -TRexStatus = Enum('TRexStatus', 'Idle Starting Running') diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/CCustomLogger.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/CCustomLogger.py new file mode 100755 index 00000000..ecf7d519 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/CCustomLogger.py @@ -0,0 +1,100 @@ + +import sys +import os +import logging + + +def setup_custom_logger(name, log_path = None): + # first make sure path availabe +# if log_path is None: +# log_path = os.getcwd()+'/trex_log.log' +# else: +# directory = os.path.dirname(log_path) +# if not os.path.exists(directory): +# os.makedirs(directory) + logging.basicConfig(level = logging.INFO, + format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', + datefmt = '%m-%d %H:%M') +# filename= log_path, +# filemode= 'w') +# +# # define a Handler which writes INFO messages or higher to the sys.stderr +# consoleLogger = logging.StreamHandler() +# consoleLogger.setLevel(logging.ERROR) +# # set a format which is simpler for console use +# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') +# # tell the handler to use this format +# consoleLogger.setFormatter(formatter) +# +# # add the handler to the logger +# logging.getLogger(name).addHandler(consoleLogger) + +def setup_daemon_logger (name, log_path = None): + # first make sure path availabe + logging.basicConfig(level = logging.INFO, + format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', + datefmt = '%m-%d %H:%M', + filename= log_path, + filemode= 'w') + +class CustomLogger(object): + + def __init__(self, log_filename): + # Store the original stdout and stderr + sys.stdout.flush() + sys.stderr.flush() + + self.stdout_fd = os.dup(sys.stdout.fileno()) + self.devnull = os.open('/dev/null', os.O_WRONLY) + self.log_file = open(log_filename, 'w') + self.silenced = False + self.pending_log_file_prints = 0 + + # silence all prints from stdout + def silence(self): + os.dup2(self.devnull, sys.stdout.fileno()) + self.silenced = True + + # restore stdout status + def restore(self): + sys.stdout.flush() + sys.stderr.flush() + # Restore normal stdout + os.dup2(self.stdout_fd, sys.stdout.fileno()) + self.silenced = False + + #print a message to the log (both stdout / log file) + def log(self, text, force = False, newline = True): + self.log_file.write((text + "\n") if newline else text) + self.pending_log_file_prints += 1 + + if (self.pending_log_file_prints >= 10): + self.log_file.flush() + self.pending_log_file_prints = 0 + + self.console(text, force, newline) + + # print a message to the console alone + def console(self, text, force = False, newline = True): + _text = (text + "\n") if newline else text + # if we are silenced and not forced - go home + if self.silenced and not force: + return + + if self.silenced: + os.write(self.stdout_fd, _text) + else: + sys.stdout.write(_text) + + sys.stdout.flush() + + # flush + def flush(self): + sys.stdout.flush() + self.log_file.flush() + + def __exit__(self, type, value, traceback): + sys.stdout.flush() + self.log_file.flush() + os.close(self.devnull) + os.close(self.log_file) diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/__init__.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/__init__.py new file mode 100755 index 00000000..5a1da046 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/__init__.py @@ -0,0 +1 @@ +__all__ = ["trex_status_e", "trex_exceptions"] diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/external_packages.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/external_packages.py new file mode 100755 index 00000000..7353c397 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/external_packages.py @@ -0,0 +1,28 @@ +#!/router/bin/python + +import sys +import os + +CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) +ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory +PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) + +CLIENT_UTILS_MODULES = ['yaml-3.11' + ] + +def import_common_modules(): + # must be in a higher priority + sys.path.insert(0, PATH_TO_PYTHON_LIB) + sys.path.append(ROOT_PATH) + import_module_list(CLIENT_UTILS_MODULES) + + +def import_module_list(modules_list): + assert(isinstance(modules_list, list)) + for p in modules_list: + full_path = os.path.join(PATH_TO_PYTHON_LIB, p) + fix_path = os.path.normcase(full_path) + sys.path.insert(1, full_path) + +import_common_modules() + diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/general_utils.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/general_utils.py new file mode 100755 index 00000000..d2521f02 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/general_utils.py @@ -0,0 +1,95 @@ +#!/router/bin/python + +import sys +import site +import string +import random +import os + +try: + import pwd +except ImportError: + import getpass + pwd = None + +using_python_3 = True if sys.version_info.major == 3 else False + + +def user_input(): + if using_python_3: + return input() + else: + # using python version 2 + return raw_input() + +def get_current_user(): + if pwd: + return pwd.getpwuid(os.geteuid()).pw_name + else: + return getpass.getuser() + +def import_module_list_by_path (modules_list): + assert(isinstance(modules_list, list)) + for full_path in modules_list: + site.addsitedir(full_path) + +def find_path_to_pardir (pardir, base_path = os.getcwd() ): + """ + Finds the absolute path for some parent dir `pardir`, starting from base_path + + The request is only valid if the stop initiator is the same client as the TRex run initiator. + + :parameters: + pardir : str + name of an upper-level directory to which we want to find an absolute path for + base_path : str + a full (usually nested) path from which we want to find a parent folder. + + default value : **current working dir** + + :return: + string representation of the full path to + + """ + components = base_path.split(os.sep) + return str.join(os.sep, components[:components.index(pardir)+1]) + + +def random_id_gen(length=8): + """ + A generator for creating a random chars id of specific length + + :parameters: + length : int + the desired length of the generated id + + default: 8 + + :return: + a random id with each next() request. + """ + id_chars = string.ascii_lowercase + string.digits + while True: + return_id = '' + for i in range(length): + return_id += random.choice(id_chars) + yield return_id + +def id_count_gen(): + """ + A generator for creating an increasing id for objects, starting from 0 + + :parameters: + None + + :return: + an id (unsigned int) with each next() request. + """ + return_id = 0 + while True: + yield return_id + return_id += 1 + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/outer_packages.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/outer_packages.py new file mode 100755 index 00000000..f8d50ce6 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/outer_packages.py @@ -0,0 +1,30 @@ +#!/router/bin/python + +import sys +import os + + +CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) +PACKAGE_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, 'external_libs')) +SCRIPTS_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs')) + +CLIENT_MODULES = ['enum34-1.0.4', + 'jsonrpclib-pelix-0.2.5', +# 'termstyle', +# 'yaml-3.11' + ] + + +def import_module_list(ext_libs_path): + for p in CLIENT_MODULES: + full_path = os.path.join(ext_libs_path, p) + if not os.path.exists(full_path): + raise Exception('Library %s is absent in path %s' % (p, ext_libs_path)) + sys.path.insert(1, full_path) + +if os.path.exists(PACKAGE_PATH): + import_module_list(PACKAGE_PATH) +elif os.path.exists(SCRIPTS_PATH): + import_module_list(SCRIPTS_PATH) +else: + raise Exception('Could not find external libs in path: %s' % [PACKAGE_PATH, SCRIPTS_PATH]) diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/text_opts.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/text_opts.py new file mode 100755 index 00000000..78a0ab1f --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/text_opts.py @@ -0,0 +1,192 @@ +import json +import re + +TEXT_CODES = {'bold': {'start': '\x1b[1m', + 'end': '\x1b[22m'}, + 'cyan': {'start': '\x1b[36m', + 'end': '\x1b[39m'}, + 'blue': {'start': '\x1b[34m', + 'end': '\x1b[39m'}, + 'red': {'start': '\x1b[31m', + 'end': '\x1b[39m'}, + 'magenta': {'start': '\x1b[35m', + 'end': '\x1b[39m'}, + 'green': {'start': '\x1b[32m', + 'end': '\x1b[39m'}, + 'yellow': {'start': '\x1b[33m', + 'end': '\x1b[39m'}, + 'underline': {'start': '\x1b[4m', + 'end': '\x1b[24m'}} + +class TextCodesStripper: + keys = [re.escape(v['start']) for k,v in TEXT_CODES.iteritems()] + keys += [re.escape(v['end']) for k,v in TEXT_CODES.iteritems()] + pattern = re.compile("|".join(keys)) + + @staticmethod + def strip (s): + return re.sub(TextCodesStripper.pattern, '', s) + +def format_num (size, suffix = "", compact = True, opts = ()): + txt = "NaN" + + if type(size) == str: + return "N/A" + + u = '' + + if compact: + for unit in ['','K','M','G','T','P']: + if abs(size) < 1000.0: + u = unit + break + size /= 1000.0 + + if isinstance(size, float): + txt = "%3.2f" % (size) + else: + txt = "{:,}".format(size) + + if u or suffix: + txt += " {:}{:}".format(u, suffix) + + if isinstance(opts, tuple): + return format_text(txt, *opts) + else: + return format_text(txt, (opts)) + + + +def format_time (t_sec): + if t_sec < 0: + return "infinite" + + if t_sec < 1: + # low numbers + for unit in ['ms', 'usec', 'ns']: + t_sec *= 1000.0 + if t_sec >= 1.0: + return '{:,.2f} [{:}]'.format(t_sec, unit) + + return "NaN" + + else: + # seconds + if t_sec < 60.0: + return '{:,.2f} [{:}]'.format(t_sec, 'sec') + + # minutes + t_sec /= 60.0 + if t_sec < 60.0: + return '{:,.2f} [{:}]'.format(t_sec, 'minutes') + + # hours + t_sec /= 60.0 + if t_sec < 24.0: + return '{:,.2f} [{:}]'.format(t_sec, 'hours') + + # days + t_sec /= 24.0 + return '{:,.2f} [{:}]'.format(t_sec, 'days') + + +def format_percentage (size): + return "%0.2f %%" % (size) + +def bold(text): + return text_attribute(text, 'bold') + + +def cyan(text): + return text_attribute(text, 'cyan') + + +def blue(text): + return text_attribute(text, 'blue') + + +def red(text): + return text_attribute(text, 'red') + + +def magenta(text): + return text_attribute(text, 'magenta') + + +def green(text): + return text_attribute(text, 'green') + +def yellow(text): + return text_attribute(text, 'yellow') + +def underline(text): + return text_attribute(text, 'underline') + + +def text_attribute(text, attribute): + if isinstance(text, str): + return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], + txt=text, + stop=TEXT_CODES[attribute]['end']) + elif isinstance(text, unicode): + return u"{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], + txt=text, + stop=TEXT_CODES[attribute]['end']) + else: + raise Exception("not a string") + + +FUNC_DICT = {'blue': blue, + 'bold': bold, + 'green': green, + 'yellow': yellow, + 'cyan': cyan, + 'magenta': magenta, + 'underline': underline, + 'red': red} + + +def format_text(text, *args): + return_string = text + for i in args: + func = FUNC_DICT.get(i) + if func: + return_string = func(return_string) + + return return_string + + +def format_threshold (value, red_zone, green_zone): + if value >= red_zone[0] and value <= red_zone[1]: + return format_text("{0}".format(value), 'red') + + if value >= green_zone[0] and value <= green_zone[1]: + return format_text("{0}".format(value), 'green') + + return "{0}".format(value) + +# pretty print for JSON +def pretty_json (json_str, use_colors = True): + pretty_str = json.dumps(json.loads(json_str), indent = 4, separators=(',', ': '), sort_keys = True) + + if not use_colors: + return pretty_str + + try: + # int numbers + pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*[^.])',r'\1{0}'.format(blue(r'\2')), pretty_str) + # float + pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*\.[0-9]+)',r'\1{0}'.format(magenta(r'\2')), pretty_str) + # # strings + # + pretty_str = re.sub(r'([ ]*:[ ]+)("[^"]*")',r'\1{0}'.format(red(r'\2')), pretty_str) + pretty_str = re.sub(r"('[^']*')", r'{0}\1{1}'.format(TEXT_CODES['magenta']['start'], + TEXT_CODES['red']['start']), pretty_str) + except : + pass + + return pretty_str + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py new file mode 100755 index 00000000..919253d1 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py @@ -0,0 +1,1216 @@ +#!/router/bin/python + +# internal libs +import sys +import os +import socket +import errno +import time +import re +import copy +import binascii +from distutils.util import strtobool +from collections import deque, OrderedDict +from json import JSONDecoder +import traceback + +try: + from . import outer_packages + from .trex_status_e import TRexStatus + from .trex_exceptions import * + from .trex_exceptions import exception_handler + from .general_utils import * +except Exception as e: # is __main__ + import outer_packages + from trex_status_e import TRexStatus + from trex_exceptions import * + from trex_exceptions import exception_handler + from general_utils import * + +# external libs +import jsonrpclib +from jsonrpclib import ProtocolError, AppError +from enum import Enum + + + +class CTRexClient(object): + """ + This class defines the client side of the RESTfull interaction with TRex + """ + + def __init__(self, trex_host, max_history_size = 100, trex_daemon_port = 8090, trex_zmq_port = 4500, verbose = False): + """ + Instantiate a TRex client object, and connecting it to listening daemon-server + + :parameters: + trex_host : str + a string of the TRex ip address or hostname. + max_history_size : int + a number to set the maximum history size of a single TRex run. Each sampling adds a new item to history. + + default value : **100** + trex_daemon_port : int + the port number on which the trex-daemon server can be reached + + default value: **8090** + trex_zmq_port : int + the port number on which trex's zmq module will interact with daemon server + + default value: **4500** + verbose : bool + sets a verbose output on supported class method. + + default value : **False** + + :raises: + socket errors, in case server could not be reached. + + """ + try: + self.trex_host = socket.gethostbyname(trex_host) + except: # give it another try + self.trex_host = socket.gethostbyname(trex_host) + self.trex_daemon_port = trex_daemon_port + self.trex_zmq_port = trex_zmq_port + self.seq = None + self.verbose = verbose + self.result_obj = CTRexResult(max_history_size) + self.decoder = JSONDecoder() + self.trex_server_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port ) + self.__verbose_print("Connecting to TRex @ {trex_path} ...".format( trex_path = self.trex_server_path ) ) + self.history = jsonrpclib.history.History() + self.server = jsonrpclib.Server(self.trex_server_path, history = self.history) + self.check_server_connectivity() + self.__verbose_print("Connection established successfully!") + self._last_sample = time.time() + self.__default_user = get_current_user() + + + def add (self, x, y): + try: + return self.server.add(x,y) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def start_trex (self, f, d, block_to_success = True, timeout = 40, user = None, trex_development = False, **trex_cmd_options): + """ + Request to start a TRex run on server. + + :parameters: + f : str + a path (on server) for the injected traffic data (.yaml file) + d : int + the desired duration of the test. must be at least 30 seconds long. + block_to_success : bool + determine if this method blocks until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + default value : **True** + timeout : int + maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + default value: **40** + user : str + the identity of the the run issuer. + trex_cmd_options : key, val + sets desired TRex options using key=val syntax, separated by comma. + for keys with no value, state key=True + + :return: + **True** on success + + :raises: + + :exc:`ValueError`, in case 'd' parameter inserted with wrong value. + + :exc:`trex_exceptions.TRexError`, in case one of the trex_cmd_options raised an exception at server. + + :exc:`trex_exceptions.TRexInUseError`, in case TRex is already taken. + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying start TRex. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + user = user or self.__default_user + try: + d = int(d) + if d < 30 and not trex_development: # test duration should be at least 30 seconds, unless trex_development flag is specified. + raise ValueError + except ValueError: + raise ValueError('d parameter must be integer, specifying how long TRex run, and must be larger than 30 secs.') + + trex_cmd_options.update( {'f' : f, 'd' : d} ) + if not trex_cmd_options.get('l'): + self.result_obj.latency_checked = False + if 'k' in trex_cmd_options: + timeout += int(trex_cmd_options['k']) # during 'k' seconds TRex stays in 'Starting' state + + self.result_obj.clear_results() + try: + issue_time = time.time() + retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + if retval!=0: + self.seq = retval # update seq num only on successful submission + return True + else: # TRex is has been started by another user + raise TRexInUseError('TRex is already being used by another user or process. Try again once TRex is back in IDLE state.') + + def stop_trex (self): + """ + Request to stop a TRex run on server. + + The request is only valid if the stop initiator is the same client as the TRex run initiator. + + :parameters: + None + + :return: + + **True** on successful termination + + **False** if request issued but TRex wasn't running. + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex ir running but started by another user. + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return self.server.stop_trex(self.seq) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def force_kill (self, confirm = True): + """ + Force killing of running TRex process (if exists) on the server. + + .. tip:: This method is a safety method and **overrides any running or reserved resources**, and as such isn't designed to be used on a regular basis. + Always consider using :func:`trex_client.CTRexClient.stop_trex` instead. + + In the end of this method, TRex will return to IDLE state with no reservation. + + :parameters: + confirm : bool + Prompt a user confirmation before continue terminating TRex session + + :return: + + **True** on successful termination + + **False** otherwise. + + :raises: + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + if confirm: + prompt = "WARNING: This will terminate active TRex session indiscriminately.\nAre you sure? " + sys.stdout.write('%s [y/n]\n' % prompt) + while True: + try: + if strtobool(user_input().lower()): + break + else: + return + except ValueError: + sys.stdout.write('Please respond with \'y\' or \'n\'.\n') + try: + return self.server.force_trex_kill() + except AppError as err: + # Silence any kind of application errors- by design + return False + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def wait_until_kickoff_finish(self, timeout = 40): + """ + Block the client application until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + The request is only valid if the stop initiator is the same client as the TRex run initiator. + + :parameters: + timeout : int + maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + :return: + + **True** on successful termination + + **False** if request issued but TRex wasn't running. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + ProtocolError, in case of error in JSON-RPC protocol. + + .. note:: Exceptions are throws only when start_trex did not block in the first place, i.e. `block_to_success` parameter was set to `False` + + """ + + try: + return self.server.wait_until_kickoff_finish(timeout) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def is_running (self, dump_out = False): + """ + Poll for TRex running status. + + If TRex is running, a history item will be added into result_obj and processed. + + .. tip:: This method is especially useful for iterating until TRex run is finished. + + :parameters: + dump_out : dict + if passed, the pointer object is cleared and the latest dump stored in it. + + :return: + + **True** if TRex is running. + + **False** if TRex is not running. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + res = self.get_running_info() + if res == {}: + return False + if (dump_out != False) and (isinstance(dump_out, dict)): # save received dump to given 'dump_out' pointer + dump_out.clear() + dump_out.update(res) + return True + except TRexWarning as err: + if err.code == -12: # TRex is either still at 'Starting' state or in Idle state, however NO error occured + return False + except TRexException: + raise + except ProtocolError as err: + raise + finally: + self.prompt_verbose_data() + + def is_idle (self): + """ + Poll for TRex running status, check if TRex is in Idle state. + + :parameters: + None + + :return: + + **True** if TRex is idle. + + **False** if TRex is starting or running. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + if self.get_running_status()['state'] == TRexStatus.Idle: + return True + return False + except TRexException: + raise + except ProtocolError as err: + raise + finally: + self.prompt_verbose_data() + + def get_trex_files_path (self): + """ + Fetches the local path in which files are stored when pushed to TRex server from client. + + :parameters: + None + + :return: + string representation of the desired path + + .. note:: The returned path represents a path on the TRex server **local machine** + + :raises: + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return (self.server.get_files_path() + '/') + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_running_status (self): + """ + Fetches the current TRex status. + + If available, a verbose data will accompany the state itself. + + :parameters: + None + + :return: + dictionary with 'state' and 'verbose' keys. + + :raises: + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + res = self.server.get_running_status() + res['state'] = TRexStatus(res['state']) + return res + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_running_info (self): + """ + Performs single poll of TRex running data and process it into the result object (named `result_obj`). + + .. tip:: This method will throw an exception if TRex isn't running. Always consider using :func:`trex_client.CTRexClient.is_running` which handles a single poll operation in safer manner. + + :parameters: + None + + :return: + dictionary containing the most updated data dump from TRex. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + if not self.is_query_relevance(): + # if requested in timeframe smaller than the original sample rate, return the last known data without interacting with server + return self.result_obj.get_latest_dump() + else: + try: + latest_dump = self.decoder.decode( self.server.get_running_info() ) # latest dump is not a dict, but json string. decode it. + self.result_obj.update_result_data(latest_dump) + return latest_dump + except TypeError as inst: + raise TypeError('JSON-RPC data decoding failed. Check out incoming JSON stream.') + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def sample_until_condition (self, condition_func, time_between_samples = 5): + """ + Automatically sets ongoing sampling of TRex data, with sampling rate described by time_between_samples. + + On each fetched dump, the condition_func is applied on the result objects, and if returns True, the sampling will stop. + + :parameters: + condition_func : function + function that operates on result_obj and checks if a condition has been met + + .. note:: `condition_finc` is applied on `CTRexResult` object. Make sure to design a relevant method. + time_between_samples : int + determines the time between each sample of the server + + default value : **5** + + :return: + the first result object (see :class:`CTRexResult` for further details) of the TRex run on which the condition has been met. + + :raises: + + :exc:`UserWarning`, in case the condition_func method condition hasn't been met + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + :exc:`Exception`, in case the condition_func suffered from any kind of exception + + """ + # make sure TRex is running. raise exceptions here if any + self.wait_until_kickoff_finish() + try: + while self.is_running(): + results = self.get_result_obj() + if condition_func(results): + # if condition satisfied, stop TRex and return result object + self.stop_trex() + return results + time.sleep(time_between_samples) + except TRexWarning: + # means we're back to Idle state, and didn't meet our condition + raise UserWarning("TRex results condition wasn't met during TRex run.") + except Exception: + # this could come from provided method 'condition_func' + raise + + def sample_to_run_finish (self, time_between_samples = 5): + """ + Automatically sets automatically sampling of TRex data with sampling rate described by time_between_samples until TRex run finished. + + :parameters: + time_between_samples : int + determines the time between each sample of the server + + default value : **5** + + :return: + the latest result object (see :class:`CTRexResult` for further details) with sampled data. + + :raises: + + :exc:`UserWarning`, in case the condition_func method condition hasn't been met + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + self.wait_until_kickoff_finish() + + try: + while self.is_running(): + time.sleep(time_between_samples) + except TRexWarning: + pass + results = self.get_result_obj() + return results + + def sample_x_seconds (self, sample_time, time_between_samples = 5): + """ + Automatically sets ongoing sampling of TRex data for sample_time seconds, with sampling rate described by time_between_samples. + Does not stop the TRex afterwards! + + .. tip:: Useful for changing the device (Router, ASA etc.) configuration after given time. + + :parameters: + sample_time : int + sample the TRex this number of seconds + + time_between_samples : int + determines the time between each sample of the server + + default value : **5** + + :return: + the first result object (see :class:`CTRexResult` for further details) of the TRex run after given sample_time. + + :raises: + + :exc:`UserWarning`, in case the TRex run ended before sample_time duration + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + # make sure TRex is running. raise exceptions here if any + self.wait_until_kickoff_finish() + elapsed_time = 0 + while self.is_running(): + if elapsed_time >= sample_time: + return self.get_result_obj() + time.sleep(time_between_samples) + elapsed_time += time_between_samples + raise UserWarning("TRex has stopped at %s seconds (before expected %s seconds)\nTry increasing test duration or decreasing sample_time" % (elapsed_time, sample_time)) + + def get_result_obj (self, copy_obj = True): + """ + Returns the result object of the trex_client's instance. + + By default, returns a **copy** of the objects (so that changes to the original object are masked). + + :parameters: + copy_obj : bool + False means that a reference to the original (possibly changing) object are passed + + defaul value : **True** + + :return: + the latest result object (see :class:`CTRexResult` for further details) with sampled data. + + """ + if copy_obj: + return copy.deepcopy(self.result_obj) + else: + return self.result_obj + + def is_reserved (self): + """ + Checks if TRex is currently reserved to any user or not. + + :parameters: + None + + :return: + + **True** if TRex is reserved. + + **False** otherwise. + + :raises: + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return self.server.is_reserved() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_trex_daemon_log (self): + """ + Get Trex daemon log. + + :return: + String representation of TRex daemon log + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case file could not be read. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return binascii.a2b_base64(self.server.get_trex_daemon_log()) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_trex_log (self): + """ + Get TRex CLI output log + + :return: + String representation of TRex log + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case file could not be fetched at server side. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return binascii.a2b_base64(self.server.get_trex_log()) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_trex_version (self): + """ + Get TRex version details. + + :return: + Trex details (Version, User, Date, Uuid, Git SHA) as ordered dictionary + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex version could not be determined. + + ProtocolError, in case of error in JSON-RPC protocol. + + General Exception is case one of the keys is missing in response + """ + + try: + version_dict = OrderedDict() + result_lines = binascii.a2b_base64(self.server.get_trex_version()).split('\n') + for line in result_lines: + if not line: + continue + key, value = line.strip().split(':', 1) + version_dict[key.strip()] = value.strip() + for key in ('Version', 'User', 'Date', 'Uuid', 'Git SHA'): + if key not in version_dict: + raise Exception('get_trex_version: got server response without key: {0}'.format(key)) + return version_dict + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def reserve_trex (self, user = None): + """ + Reserves the usage of TRex to a certain user. + + When TRex is reserved, it can't be reserved. + + :parameters: + user : str + a username of the desired owner of TRex + + default: current logged user + + :return: + **True** if reservation made successfully + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying to make the reservation. + + :exc:`trex_exceptions.TRexInUseError`, in case TRex is currently running. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + username = user or self.__default_user + try: + return self.server.reserve_trex(user = username) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def cancel_reservation (self, user = None): + """ + Cancels a current reservation of TRex to a certain user. + + When TRex is reserved, no other user can start new TRex runs. + + + :parameters: + user : str + a username of the desired owner of TRex + + default: current logged user + + :return: + + **True** if reservation canceled successfully, + + **False** if there was no reservation at all. + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying to cancel the reservation. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + + username = user or self.__default_user + try: + return self.server.cancel_reservation(user = username) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def push_files (self, filepaths): + """ + Pushes a file (or a list of files) to store locally on server. + + :parameters: + filepaths : str or list + a path to a file to be pushed to server. + if a list of paths is passed, all of those will be pushed to server + + :return: + + **True** if file(s) copied successfully. + + **False** otherwise. + + :raises: + + :exc:`IOError`, in case specified file wasn't found or could not be accessed. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + paths_list = None + if isinstance(filepaths, str): + paths_list = [filepaths] + elif isinstance(filepaths, list): + paths_list = filepaths + else: + raise TypeError("filepaths argument must be of type str or list") + + for filepath in paths_list: + try: + if not os.path.exists(filepath): + raise IOError(errno.ENOENT, "The requested `{fname}` file wasn't found. Operation aborted.".format( + fname = filepath) ) + else: + filename = os.path.basename(filepath) + with open(filepath, 'rb') as f: + file_content = f.read() + self.server.push_file(filename, binascii.b2a_base64(file_content)) + finally: + self.prompt_verbose_data() + return True + + def is_query_relevance(self): + """ + Checks if time between any two consecutive server queries (asking for live running data) passed. + + .. note:: The allowed minimum time between each two consecutive samples is 0.5 seconds. + + :parameters: + None + + :return: + + **True** if more than 0.5 seconds has been past from last server query. + + **False** otherwise. + + """ + cur_time = time.time() + if cur_time-self._last_sample < 0.5: + return False + else: + self._last_sample = cur_time + return True + + def call_server_mathod_safely (self, method_to_call): + try: + return method_to_call() + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + raise SocketError(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") + + def check_server_connectivity (self): + """ + Checks for server valid connectivity. + """ + try: + socket.gethostbyname(self.trex_host) + return self.server.connectivity_check() + except socket.gaierror as e: + raise socket.gaierror(e.errno, "Could not resolve server hostname. Please make sure hostname entered correctly.") + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + raise socket.error(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") + finally: + self.prompt_verbose_data() + + def prompt_verbose_data(self): + """ + This method prompts any verbose data available, only if `verbose` option has been turned on. + """ + if self.verbose: + print ('\n') + print ("(*) JSON-RPC request:", self.history.request) + print ("(*) JSON-RPC response:", self.history.response) + + def __verbose_print(self, print_str): + """ + This private method prints the `print_str` string only in case self.verbose flag is turned on. + + :parameters: + print_str : str + a string to be printed + + :returns: + None + """ + if self.verbose: + print (print_str) + + + + def _handle_AppError_exception(self, err): + """ + This private method triggres the TRex dedicated exception generation in case a general ProtocolError has been raised. + """ + # handle known exceptions based on known error codes. + # if error code is not known, raise ProtocolError + raise exception_handler.gen_exception(err) + + +class CTRexResult(object): + """ + A class containing all results received from TRex. + + Ontop to containing the results, this class offers easier data access and extended results processing options + """ + def __init__(self, max_history_size): + """ + Instatiate a TRex result object + + :parameters: + max_history_size : int + a number to set the maximum history size of a single TRex run. Each sampling adds a new item to history. + + """ + self._history = deque(maxlen = max_history_size) + self.clear_results() + self.latency_checked = True + + def __repr__(self): + return ("Is valid history? {arg}\n".format( arg = self.is_valid_hist() ) + + "Done warmup? {arg}\n".format( arg = self.is_done_warmup() ) + + "Expected tx rate: {arg}\n".format( arg = self.get_expected_tx_rate() ) + + "Current tx rate: {arg}\n".format( arg = self.get_current_tx_rate() ) + + "Maximum latency: {arg}\n".format( arg = self.get_max_latency() ) + + "Average latency: {arg}\n".format( arg = self.get_avg_latency() ) + + "Average window latency: {arg}\n".format( arg = self.get_avg_window_latency() ) + + "Total drops: {arg}\n".format( arg = self.get_total_drops() ) + + "Drop rate: {arg}\n".format( arg = self.get_drop_rate() ) + + "History size so far: {arg}\n".format( arg = len(self._history) ) ) + + def get_expected_tx_rate (self): + """ + Fetches the expected TX rate in various units representation + + :parameters: + None + + :return: + dictionary containing the expected TX rate, where the key is the measurement units, and the value is the measurement value. + + """ + return self._expected_tx_rate + + def get_current_tx_rate (self): + """ + Fetches the current TX rate in various units representation + + :parameters: + None + + :return: + dictionary containing the current TX rate, where the key is the measurement units, and the value is the measurement value. + + """ + return self._current_tx_rate + + def get_max_latency (self): + """ + Fetches the maximum latency measured on each of the interfaces + + :parameters: + None + + :return: + dictionary containing the maximum latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. + + """ + return self._max_latency + + def get_avg_latency (self): + """ + Fetches the average latency measured on each of the interfaces from the start of TRex run + + :parameters: + None + + :return: + dictionary containing the average latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. + + The `all` key represents the average of all interfaces' average + + """ + return self._avg_latency + + def get_avg_window_latency (self): + """ + Fetches the average latency measured on each of the interfaces from all the sampled currently stored in window. + + :parameters: + None + + :return: + dictionary containing the average latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. + + The `all` key represents the average of all interfaces' average + + """ + return self._avg_window_latency + + def get_total_drops (self): + """ + Fetches the total number of drops identified from the moment TRex run began. + + :parameters: + None + + :return: + total drops count (as int) + + """ + return self._total_drops + + def get_drop_rate (self): + """ + Fetches the most recent drop rate in pkts/sec units. + + :parameters: + None + + :return: + current drop rate (as float) + + """ + return self._drop_rate + + def is_valid_hist (self): + """ + Checks if result obejct contains valid data. + + :parameters: + None + + :return: + + **True** if history is valid. + + **False** otherwise. + + """ + return self.valid + + def set_valid_hist (self, valid_stat = True): + """ + Sets result obejct validity status. + + :parameters: + valid_stat : bool + defines the validity status + + dafault value : **True** + + :return: + None + + """ + self.valid = valid_stat + + def is_done_warmup (self): + """ + Checks if TRex latest results TX-rate indicates that TRex has reached its expected TX-rate. + + :parameters: + None + + :return: + + **True** if expected TX-rate has been reached. + + **False** otherwise. + + """ + return self._done_warmup + + def get_last_value (self, tree_path_to_key, regex = None): + """ + A dynamic getter from the latest sampled data item stored in the result object. + + :parameters: + tree_path_to_key : str + defines a path to desired data. + + .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. + | Use '[i]' to access the i'th indexed object of an array. + + tree_path_to_key : regex + apply a regex to filter results out from a multiple results set. + + Filter applies only on keys of dictionary type. + + dafault value : **None** + + :return: + + a list of values relevant to the specified path + + None if no results were fetched or the history isn't valid. + + """ + if not self.is_valid_hist(): + return None + else: + return CTRexResult.__get_value_by_path(self._history[len(self._history)-1], tree_path_to_key, regex) + + def get_value_list (self, tree_path_to_key, regex = None, filter_none = True): + """ + A dynamic getter from all sampled data items stored in the result object. + + :parameters: + tree_path_to_key : str + defines a path to desired data. + + .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. + | Use '[i]' to access the i'th indexed object of an array. + + tree_path_to_key : regex + apply a regex to filter results out from a multiple results set. + + Filter applies only on keys of dictionary type. + + dafault value : **None** + + filter_none : bool + specify if None results should be filtered out or not. + + dafault value : **True** + + :return: + + a list of values relevant to the specified path. Each item on the list refers to a single server sample. + + None if no results were fetched or the history isn't valid. + """ + + if not self.is_valid_hist(): + return None + else: + raw_list = list( map(lambda x: CTRexResult.__get_value_by_path(x, tree_path_to_key, regex), self._history) ) + if filter_none: + return list (filter(lambda x: x!=None, raw_list) ) + else: + return raw_list + + def get_latest_dump(self): + """ + A getter to the latest sampled data item stored in the result object. + + :parameters: + None + + :return: + + a dictionary of the latest data item + + an empty dictionary if history is empty. + + """ + history_size = len(self._history) + if history_size != 0: + return self._history[len(self._history) - 1] + else: + return {} + + def update_result_data (self, latest_dump): + """ + Integrates a `latest_dump` dictionary into the CTRexResult object. + + :parameters: + latest_dump : dict + a dictionary with the items desired to be integrated into the object history and stats + + :return: + None + + """ + # add latest dump to history + if latest_dump != {}: + self._history.append(latest_dump) + if not self.valid: + self.valid = True + + # parse important fields and calculate averages and others + if self._expected_tx_rate is None: + # get the expected data only once since it doesn't change + self._expected_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_expected_\w+") + + self._current_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_(?!expected_)\w+") + if not self._done_warmup and self._expected_tx_rate is not None: + # check for up to 2% change between expected and actual + if (self._current_tx_rate['m_tx_bps']/self._expected_tx_rate['m_tx_expected_bps'] > 0.98): + self._done_warmup = True + + # handle latency data + if self.latency_checked: + latency_pre = "trex-latency" + self._max_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "max-")#None # TBC + # support old typo + if self._max_latency is None: + latency_pre = "trex-latecny" + self._max_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "max-") + + self._avg_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "avg-")#None # TBC + self._avg_latency = CTRexResult.__avg_all_and_rename_keys(self._avg_latency) + + avg_win_latency_list = self.get_value_list("{latency}.data".format(latency = latency_pre), "avg-") + self._avg_window_latency = CTRexResult.__calc_latency_win_stats(avg_win_latency_list) + + tx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_tx_pkts") + rx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_rx_pkts") + if tx_pkts is not None and rx_pkts is not None: + self._total_drops = tx_pkts - rx_pkts + self._drop_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_rx_drop_bps") + + def clear_results (self): + """ + Clears all results and sets the history's validity to `False` + + :parameters: + None + + :return: + None + + """ + self.valid = False + self._done_warmup = False + self._expected_tx_rate = None + self._current_tx_rate = None + self._max_latency = None + self._avg_latency = None + self._avg_window_latency = None + self._total_drops = None + self._drop_rate = None + self._history.clear() + + @staticmethod + def __get_value_by_path (dct, tree_path, regex = None): + try: + for i, p in re.findall(r'(\d+)|([\w|-]+)', tree_path): + dct = dct[p or int(i)] + if regex is not None and isinstance(dct, dict): + res = {} + for key,val in dct.items(): + match = re.match(regex, key) + if match: + res[key]=val + return res + else: + return dct + except (KeyError, TypeError): + return None + + @staticmethod + def __calc_latency_win_stats (latency_win_list): + res = {'all' : None } + port_dict = {'all' : []} + list( map(lambda x: CTRexResult.__update_port_dict(x, port_dict), latency_win_list) ) + + # finally, calculate everages for each list + res['all'] = float("%.3f" % (sum(port_dict['all'])/float(len(port_dict['all']))) ) + port_dict.pop('all') + for port, avg_list in port_dict.items(): + res[port] = float("%.3f" % (sum(avg_list)/float(len(avg_list))) ) + + return res + + @staticmethod + def __update_port_dict (src_avg_dict, dest_port_dict): + all_list = src_avg_dict.values() + dest_port_dict['all'].extend(all_list) + for key, val in src_avg_dict.items(): + reg_res = re.match("avg-(\d+)", key) + if reg_res: + tmp_key = "port"+reg_res.group(1) + if tmp_key in dest_port_dict: + dest_port_dict[tmp_key].append(val) + else: + dest_port_dict[tmp_key] = [val] + + @staticmethod + def __avg_all_and_rename_keys (src_dict): + res = {} + all_list = src_dict.values() + res['all'] = float("%.3f" % (sum(all_list)/float(len(all_list))) ) + for key, val in src_dict.items(): + reg_res = re.match("avg-(\d+)", key) + if reg_res: + tmp_key = "port"+reg_res.group(1) + res[tmp_key] = val # don't touch original fields values + return res + + + +if __name__ == "__main__": + pass + diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_daemon_server.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_daemon_server.py new file mode 100755 index 00000000..9784d42a --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_daemon_server.py @@ -0,0 +1,79 @@ +#!/usr/bin/python + +import outer_packages +import daemon +from trex_server import do_main_program, trex_parser +import CCustomLogger + +import logging +import time +import sys +import os, errno +import grp +import signal +from daemon import runner +from extended_daemon_runner import ExtendedDaemonRunner +import lockfile +import errno + +class TRexServerApp(object): + def __init__(self): + TRexServerApp.create_working_dirs() + self.stdin_path = '/dev/null' + self.stdout_path = '/dev/tty' # All standard prints will come up from this source. + self.stderr_path = "/var/log/trex/trex_daemon_server.log" # All log messages will come up from this source + self.pidfile_path = '/var/run/trex/trex_daemon_server.pid' + self.pidfile_timeout = 5 # timeout in seconds + + def run(self): + do_main_program() + + + @staticmethod + def create_working_dirs(): + if not os.path.exists('/var/log/trex'): + os.mkdir('/var/log/trex') + if not os.path.exists('/var/run/trex'): + os.mkdir('/var/run/trex') + + + +def main (): + + trex_app = TRexServerApp() + + # setup the logger + default_log_path = '/var/log/trex/trex_daemon_server.log' + + try: + CCustomLogger.setup_daemon_logger('TRexServer', default_log_path) + logger = logging.getLogger('TRexServer') + logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s") + handler = logging.FileHandler("/var/log/trex/trex_daemon_server.log") + logger.addHandler(handler) + except EnvironmentError, e: + if e.errno == errno.EACCES: # catching permission denied error + print "Launching user must have sudo privileges in order to run TRex daemon.\nTerminating daemon process." + exit(-1) + + daemon_runner = ExtendedDaemonRunner(trex_app, trex_parser) + + #This ensures that the logger file handle does not get closed during daemonization + daemon_runner.daemon_context.files_preserve=[handler.stream] + + try: + if not set(['start', 'stop']).isdisjoint(set(sys.argv)): + print "Logs are saved at: {log_path}".format( log_path = default_log_path ) + daemon_runner.do_action() + + except lockfile.LockTimeout as inst: + logger.error(inst) + print inst + print """ + Please try again once the timeout has been reached. + If this error continues, consider killing the process manually and restart the daemon.""" + + +if __name__ == "__main__": + main() diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_exceptions.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_exceptions.py new file mode 100755 index 00000000..0de38411 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_exceptions.py @@ -0,0 +1,140 @@ +#!/router/bin/python + +#from rpc_exceptions import RPCExceptionHandler, WrappedRPCError + +from jsonrpclib import Fault, ProtocolError, AppError + +class RPCError(Exception): + """ + This is the general RPC error exception class from which :exc:`trex_exceptions.TRexException` inherits. + + Every exception in this class has as error format according to JSON-RPC convention convention: code, message and data. + + """ + def __init__(self, code, message, remote_data = None): + self.code = code + self.msg = message or self._default_message + self.data = remote_data + self.args = (code, self.msg, remote_data) + + def __str__(self): + return self.__repr__() + def __repr__(self): + if self.args[2] is not None: + return u"[errcode:%r] %r. Extended data: %r" % (self.args[0], self.args[1], self.args[2]) + else: + return u"[errcode:%r] %r" % (self.args[0], self.args[1]) + +class TRexException(RPCError): + """ + This is the most general TRex exception. + + All exceptions inherits from this class has an error code and a default message which describes the most common use case of the error. + + This exception isn't used by default and will only when an unrelated to ProtocolError will occur, and it can't be resolved to any of the deriviate exceptions. + + """ + code = -10 + _default_message = 'TRex encountered an unexpected error. please contact TRex dev team.' + # api_name = 'TRex' + +class TRexError(TRexException): + """ + This is the most general TRex exception. + + This exception isn't used by default and will only when an unrelated to ProtocolError will occur, and it can't be resolved to any of the deriviate exceptions. + """ + code = -11 + _default_message = 'TRex run failed due to wrong input parameters, or due to reachability issues.' + +class TRexWarning(TRexException): + """ Indicates a warning from TRex server. When this exception raises it normally used to indicate required data isn't ready yet """ + code = -12 + _default_message = 'TRex is starting (data is not available yet).' + +class TRexRequestDenied(TRexException): + """ Indicates the desired reques was denied by the server """ + code = -33 + _default_message = 'TRex desired request denied because the requested resource is already taken. Try again once TRex is back in IDLE state.' + +class TRexInUseError(TRexException): + """ + Indicates that TRex is currently in use + + """ + code = -13 + _default_message = 'TRex is already being used by another user or process. Try again once TRex is back in IDLE state.' + +class TRexRunFailedError(TRexException): + """ Indicates that TRex has failed due to some reason. This Exception is used when TRex process itself terminates due to unknown reason """ + code = -14 + _default_message = '' + +class TRexIncompleteRunError(TRexException): + """ + Indicates that TRex has failed due to some reason. + This Exception is used when TRex process itself terminated with error fault or it has been terminated by an external intervention in the OS. + + """ + code = -15 + _default_message = 'TRex run was terminated unexpectedly by outer process or by the hosting OS' + +EXCEPTIONS = [TRexException, TRexError, TRexWarning, TRexInUseError, TRexRequestDenied, TRexRunFailedError, TRexIncompleteRunError] + +class CExceptionHandler(object): + """ + CExceptionHandler is responsible for generating TRex API related exceptions in client side. + """ + def __init__(self, exceptions): + """ + Instatiate a CExceptionHandler object + + :parameters: + + exceptions : list + a list of all TRex acceptable exception objects. + + default list: + - :exc:`trex_exceptions.TRexException` + - :exc:`trex_exceptions.TRexError` + - :exc:`trex_exceptions.TRexWarning` + - :exc:`trex_exceptions.TRexInUseError` + - :exc:`trex_exceptions.TRexRequestDenied` + - :exc:`trex_exceptions.TRexRunFailedError` + - :exc:`trex_exceptions.TRexIncompleteRunError` + + """ + if isinstance(exceptions, type): + exceptions = [ exceptions, ] + self.exceptions = exceptions + self.exceptions_dict = dict((e.code, e) for e in self.exceptions) + + def gen_exception (self, err): + """ + Generates an exception based on a general ProtocolError exception object `err`. + + When TRex is reserved, no other user can start new TRex runs. + + + :parameters: + + err : exception + a ProtocolError exception raised by :class:`trex_client.CTRexClient` class + + :return: + A TRex exception from the exception list defined in class creation. + + If such exception wasn't found, returns a TRexException exception + + """ + code, message, data = err + try: + exp = self.exceptions_dict[code] + return exp(exp.code, message, data) + except KeyError: + # revert to TRexException when unknown error application raised + return TRexException(err) + + +exception_handler = CExceptionHandler( EXCEPTIONS ) + diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status.py new file mode 100644 index 00000000..f132720c --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status.py @@ -0,0 +1,8 @@ +#!/router/bin/python + +# define the states in which a T-Rex can hold during its lifetime +# TRexStatus = Enum('TRexStatus', 'Idle Starting Running') + +IDLE = 1 +STARTING = 2 +RUNNING = 3 diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status_e.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status_e.py new file mode 100755 index 00000000..79a25acc --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_status_e.py @@ -0,0 +1,11 @@ +#!/router/bin/python + +try: + from . import outer_packages +except: + import outer_packages +from enum import Enum + + +# define the states in which a TRex can hold during its lifetime +TRexStatus = Enum('TRexStatus', 'Idle Starting Running') -- cgit 1.2.3-korg