From 59f00264f39dfb95a62401776b4ed8c433682fa7 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Thu, 24 Mar 2016 20:00:27 +0200 Subject: gather stateful client to stf folder, create trex_client package --- .gitignore | 1 + linux_dpdk/ws_main.py | 19 +- scripts/automation/regression/CPlatform.py | 15 +- scripts/automation/regression/trex_unit_test.py | 4 +- .../trex_control_plane/client/__init__.py | 1 - .../trex_control_plane/client/trex_adv_client.py | 70 -- .../trex_control_plane/client/trex_root_path.py | 15 - .../trex_control_plane/client_utils/__init__.py | 1 - .../client_utils/external_packages.py | 72 -- .../client_utils/general_utils.py | 95 -- .../client_utils/trex_yaml_gen.py | 212 ---- .../trex_control_plane/client_utils/yaml_utils.py | 163 --- .../trex_control_plane/dirtree_no_files.txt | 11 - .../trex_control_plane/dirtree_with_files.txt | 31 - .../trex_control_plane/stf/CCustomLogger.py | 100 ++ .../automation/trex_control_plane/stf/__init__.py | 1 + .../trex_control_plane/stf/external_packages.py | 28 + .../trex_control_plane/stf/general_utils.py | 95 ++ .../trex_control_plane/stf/outer_packages.py | 30 + .../automation/trex_control_plane/stf/text_opts.py | 192 ++++ .../trex_control_plane/stf/trex_client.py | 1216 ++++++++++++++++++++ .../trex_control_plane/stf/trex_daemon_server.py | 79 ++ .../trex_control_plane/stf/trex_exceptions.py | 140 +++ .../trex_control_plane/stf/trex_status.py | 8 + .../trex_control_plane/stf/trex_status_e.py | 11 + .../stl/examples/hlt_udp_simple.py | 12 +- .../trex_control_plane/stl/examples/stl_pcap.py | 52 +- .../stl/trex_stl_lib/trex_stl_ext.py | 9 +- .../stl/trex_stl_lib/trex_stl_hltapi.py | 6 +- .../scapy-2.3.1/python2/scapy/data.py | 1 + .../scapy-2.3.1/python3/scapy/data.py | 1 + scripts/find_python.sh | 27 +- scripts/run_functional_tests | 7 +- 33 files changed, 2013 insertions(+), 712 deletions(-) delete mode 100755 scripts/automation/trex_control_plane/client/__init__.py delete mode 100755 scripts/automation/trex_control_plane/client/trex_adv_client.py delete mode 100644 scripts/automation/trex_control_plane/client/trex_root_path.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/__init__.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/external_packages.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/general_utils.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/yaml_utils.py delete mode 100755 scripts/automation/trex_control_plane/dirtree_no_files.txt delete mode 100755 scripts/automation/trex_control_plane/dirtree_with_files.txt create mode 100755 scripts/automation/trex_control_plane/stf/CCustomLogger.py create mode 100755 scripts/automation/trex_control_plane/stf/__init__.py create mode 100755 scripts/automation/trex_control_plane/stf/external_packages.py create mode 100755 scripts/automation/trex_control_plane/stf/general_utils.py create mode 100755 scripts/automation/trex_control_plane/stf/outer_packages.py create mode 100755 scripts/automation/trex_control_plane/stf/text_opts.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_client.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_daemon_server.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_exceptions.py create mode 100644 scripts/automation/trex_control_plane/stf/trex_status.py create mode 100755 scripts/automation/trex_control_plane/stf/trex_status_e.py diff --git a/.gitignore b/.gitignore index 39995917..0b02fb21 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ src/GTAGS #generated +*.bak *.pyc __pycache__ *_GENERATED.py diff --git a/linux_dpdk/ws_main.py b/linux_dpdk/ws_main.py index 2aa06e3b..6310a8c2 100755 --- a/linux_dpdk/ws_main.py +++ b/linux_dpdk/ws_main.py @@ -546,7 +546,14 @@ dpdk_includes_path =''' ../src/ DPDK_FLAGS=['-D_GNU_SOURCE', '-DPF_DRIVER', '-DX722_SUPPORT', '-DVF_DRIVER', '-DINTEGRATED_VF']; - +client_external_libs = [ + 'enum34-1.0.4', + 'jsonrpclib-pelix-0.2.5', + 'pyyaml-3.11', + 'pyzmq-14.5.0', + 'scapy-2.3.1', + 'texttable-0.8.4', + ] RELEASE_ = "release" @@ -956,6 +963,16 @@ def release(bld, custom_dir = None): os.system("cp -rv %s %s " %(src_file,dest_file)); os.system("chmod 755 %s " %(dest_file)); + # create client package + os.system('mkdir -p %s/trex_client/external_libs' % exec_p) + os.system('touch %s/trex_client/__init__.py' % exec_p) + for ext_lib in client_external_libs: + os.system('cp ../scripts/external_libs/%s %s/trex_client/external_libs/ -r' % (ext_lib, exec_p)) + os.system('cp ../scripts/automation/trex_control_plane/stf %s/trex_client/ -r' % exec_p) + os.system('cp ../scripts/automation/trex_control_plane/stl/trex_stl_lib %s/trex_client/stl -r' % exec_p) + shutil.make_archive(os.path.join(exec_p, 'trex_client'), 'gztar', exec_p, 'trex_client') + os.system('rm -r %s/trex_client' % exec_p) + rel=get_build_num () os.system('cd %s/..;tar --exclude="*.pyc" -zcvf %s/%s.tar.gz %s' %(exec_p,os.getcwd(),rel,rel)) os.system("mv %s/%s.tar.gz %s" % (os.getcwd(),rel,exec_p)); diff --git a/scripts/automation/regression/CPlatform.py b/scripts/automation/regression/CPlatform.py index 6741d5c1..da056d23 100755 --- a/scripts/automation/regression/CPlatform.py +++ b/scripts/automation/regression/CPlatform.py @@ -650,7 +650,7 @@ class CPlatform(object): command = "dir {drive}: | include {image}".format(drive = search_drive, image = img_name) response = self.cmd_link.run_single_command(command, timeout = 10) if CShowParser.parse_image_existence(response, img_name): - self.needed_image_path = '%s:%s' % (search_drive, img_name) + self.needed_image_path = '%s:/%s' % (search_drive, img_name) print('Found image in platform:', self.needed_image_path) return True return False @@ -734,8 +734,10 @@ class CPlatform(object): boot_img_cmd = "boot system flash %s" % self.needed_image_path config_register_cmd = "config-register 0x2021" - cache.add('CONF', ["no boot system", boot_img_cmd, config_register_cmd]) - self.cmd_link.run_single_command( cache ) + cache.add('CONF', ["no boot system", boot_img_cmd, config_register_cmd, '\r']) + response = self.cmd_link.run_single_command( cache ) + print("RESPONSE:") + print(response) self.save_config_to_startup_config() def is_image_matches(self, needed_image): @@ -777,7 +779,9 @@ class CPlatform(object): Copies running-config into startup-config. """ - self.cmd_link.run_single_command('wr') + cache = CCommandCache() + cache.add('EXEC', ['wr', '\r'] ) + self.cmd_link.run_single_command(cache) def reload_platform(self, device_cfg_obj): """ reload_platform(self) -> None @@ -814,8 +818,9 @@ class CPlatform(object): time.sleep(30) self.reload_connection(device_cfg_obj) - finally: progress_thread.join() + except Exception as e: + print e def get_if_manager(self): return self.if_mngr diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py index 4348d004..fb666382 100755 --- a/scripts/automation/regression/trex_unit_test.py +++ b/scripts/automation/regression/trex_unit_test.py @@ -36,8 +36,8 @@ import misc_methods from rednose import RedNose import termstyle from trex import CTRexScenario -from client.trex_client import * -from common.trex_exceptions import * +from stf.trex_client import * +from stf.trex_exceptions import * from trex_stl_lib.api import * import trex import socket diff --git a/scripts/automation/trex_control_plane/client/__init__.py b/scripts/automation/trex_control_plane/client/__init__.py deleted file mode 100755 index e1d24710..00000000 --- a/scripts/automation/trex_control_plane/client/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ["trex_client"] diff --git a/scripts/automation/trex_control_plane/client/trex_adv_client.py b/scripts/automation/trex_control_plane/client/trex_adv_client.py deleted file mode 100755 index bf7ccf58..00000000 --- a/scripts/automation/trex_control_plane/client/trex_adv_client.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/router/bin/python - -import trex_client -from jsonrpclib import ProtocolError, AppError - -class CTRexAdvClient(trex_client.CTRexClient): - def __init__ (self, trex_host, max_history_size = 100, trex_daemon_port = 8090, trex_zmq_port = 4500, verbose = False): - super(CTRexAdvClient, self).__init__(trex_host, max_history_size, trex_daemon_port, trex_zmq_port, verbose) - pass - - # TRex KIWI advanced methods - def start_quick_trex(self, pcap_file, d, delay, dual, ipv6, times, interfaces): - try: - return self.server.start_quick_trex(pcap_file = pcap_file, duration = d, dual = dual, delay = delay, ipv6 = ipv6, times = times, interfaces = interfaces) - except AppError as err: - self.__handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def stop_quick_trex(self): - try: - return self.server.stop_quick_trex() - except AppError as err: - self.__handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - -# def is_running(self): -# pass - - def get_running_stats(self): - try: - return self.server.get_running_stats() - except AppError as err: - self.__handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - def clear_counters(self): - try: - return self.server.clear_counters() - except AppError as err: - self.__handle_AppError_exception(err.args[0]) - except ProtocolError: - raise - finally: - self.prompt_verbose_data() - - -if __name__ == "__main__": - trex = CTRexAdvClient('trex-dan', trex_daemon_port = 8383, verbose = True) - print trex.start_quick_trex(delay = 10, - dual = True, - d = 20, - interfaces = ["gig0/0/1", "gig0/0/2"], - ipv6 = False, - pcap_file="avl/http_browsing.pcap", - times=3) - print trex.stop_quick_trex() - print trex.get_running_stats() - print trex.clear_counters() - pass - - diff --git a/scripts/automation/trex_control_plane/client/trex_root_path.py b/scripts/automation/trex_control_plane/client/trex_root_path.py deleted file mode 100644 index de4ec03b..00000000 --- a/scripts/automation/trex_control_plane/client/trex_root_path.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/router/bin/python - -import os -import sys - -def add_root_to_path (): - """adds trex_control_plane root dir to script path, up to `depth` parent dirs""" - root_dirname = 'trex_control_plane' - file_path = os.path.dirname(os.path.realpath(__file__)) - - components = file_path.split(os.sep) - sys.path.append( str.join(os.sep, components[:components.index(root_dirname)+1]) ) - return - -add_root_to_path() diff --git a/scripts/automation/trex_control_plane/client_utils/__init__.py b/scripts/automation/trex_control_plane/client_utils/__init__.py deleted file mode 100755 index c38c2cca..00000000 --- a/scripts/automation/trex_control_plane/client_utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ["general_utils", "trex_yaml_gen"] diff --git a/scripts/automation/trex_control_plane/client_utils/external_packages.py b/scripts/automation/trex_control_plane/client_utils/external_packages.py deleted file mode 100755 index c682dc18..00000000 --- a/scripts/automation/trex_control_plane/client_utils/external_packages.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/router/bin/python - -import sys -import os -import warnings - -CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) -ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory -PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) - -CLIENT_UTILS_MODULES = ['dpkt-1.8.6', - 'yaml-3.11', - 'texttable-0.8.4', - 'scapy-2.3.1' - ] - -def import_client_utils_modules(): - - # must be in a higher priority - sys.path.insert(0, PATH_TO_PYTHON_LIB) - - sys.path.append(ROOT_PATH) - import_module_list(CLIENT_UTILS_MODULES) - - -def import_module_list(modules_list): - assert(isinstance(modules_list, list)) - for p in modules_list: - full_path = os.path.join(PATH_TO_PYTHON_LIB, p) - fix_path = os.path.normcase(full_path) - sys.path.insert(1, full_path) - - - import_platform_dirs() - - - -def import_platform_dirs (): - # handle platform dirs - - # try fedora 18 first and then cel5.9 - # we are using the ZMQ module to determine the right platform - - full_path = os.path.join(PATH_TO_PYTHON_LIB, 'platform/fedora18') - fix_path = os.path.normcase(full_path) - sys.path.insert(0, full_path) - try: - # try to import and delete it from the namespace - import zmq - del zmq - return - except: - sys.path.pop(0) - pass - - full_path = os.path.join(PATH_TO_PYTHON_LIB, 'platform/cel59') - fix_path = os.path.normcase(full_path) - sys.path.insert(0, full_path) - try: - # try to import and delete it from the namespace - import zmq - del zmq - return - - except: - sys.path.pop(0) - sys.modules['zmq'] = None - warnings.warn("unable to determine platform type for ZMQ import") - - - -import_client_utils_modules() diff --git a/scripts/automation/trex_control_plane/client_utils/general_utils.py b/scripts/automation/trex_control_plane/client_utils/general_utils.py deleted file mode 100755 index d2521f02..00000000 --- a/scripts/automation/trex_control_plane/client_utils/general_utils.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/router/bin/python - -import sys -import site -import string -import random -import os - -try: - import pwd -except ImportError: - import getpass - pwd = None - -using_python_3 = True if sys.version_info.major == 3 else False - - -def user_input(): - if using_python_3: - return input() - else: - # using python version 2 - return raw_input() - -def get_current_user(): - if pwd: - return pwd.getpwuid(os.geteuid()).pw_name - else: - return getpass.getuser() - -def import_module_list_by_path (modules_list): - assert(isinstance(modules_list, list)) - for full_path in modules_list: - site.addsitedir(full_path) - -def find_path_to_pardir (pardir, base_path = os.getcwd() ): - """ - Finds the absolute path for some parent dir `pardir`, starting from base_path - - The request is only valid if the stop initiator is the same client as the TRex run initiator. - - :parameters: - pardir : str - name of an upper-level directory to which we want to find an absolute path for - base_path : str - a full (usually nested) path from which we want to find a parent folder. - - default value : **current working dir** - - :return: - string representation of the full path to - - """ - components = base_path.split(os.sep) - return str.join(os.sep, components[:components.index(pardir)+1]) - - -def random_id_gen(length=8): - """ - A generator for creating a random chars id of specific length - - :parameters: - length : int - the desired length of the generated id - - default: 8 - - :return: - a random id with each next() request. - """ - id_chars = string.ascii_lowercase + string.digits - while True: - return_id = '' - for i in range(length): - return_id += random.choice(id_chars) - yield return_id - -def id_count_gen(): - """ - A generator for creating an increasing id for objects, starting from 0 - - :parameters: - None - - :return: - an id (unsigned int) with each next() request. - """ - return_id = 0 - while True: - yield return_id - return_id += 1 - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py b/scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py deleted file mode 100755 index c26fef29..00000000 --- a/scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/router/bin/python - -import pprint -import yaml -import os -# import bisect - -class CTRexYaml(object): - """ - This class functions as a YAML generator according to TRex YAML format. - - CTRexYaml is compatible with both Python 2 and Python 3. - """ - YAML_TEMPLATE = [{'cap_info': [], - 'duration': 10.0, - 'generator': {'clients_end': '16.0.1.255', - 'clients_per_gb': 201, - 'clients_start': '16.0.0.1', - 'distribution': 'seq', - 'dual_port_mask': '1.0.0.0', - 'min_clients': 101, - 'servers_end': '48.0.0.255', - 'servers_start': '48.0.0.1', - 'tcp_aging': 1, - 'udp_aging': 1}, - 'mac' : [0x00,0x00,0x00,0x01,0x00,0x00]}] - PCAP_TEMPLATE = {'cps': 1.0, - 'ipg': 10000, - 'name': '', - 'rtt': 10000, - 'w': 1} - - def __init__ (self, trex_files_path): - """ - The initialization of this class creates a CTRexYaml object with **empty** 'cap-info', and with default client-server configuration. - - Use class methods to add and assign pcap files and export the data to a YAML file. - - :parameters: - trex_files_path : str - a path (on TRex server side) for the pcap files using which TRex can access it. - - """ - self.yaml_obj = list(CTRexYaml.YAML_TEMPLATE) - self.empty_cap = True - self.file_list = [] - self.yaml_dumped = False - self.trex_files_path = trex_files_path - - def add_pcap_file (self, local_pcap_path): - """ - Adds a .pcap file with recorded traffic to the yaml object by linking the file with 'cap-info' template key fields. - - :parameters: - local_pcap_path : str - a path (on client side) for the pcap file to be added. - - :return: - + The index of the inserted item (as int) if item added successfully - + -1 if pcap file already exists in 'cap_info'. - - """ - new_pcap = dict(CTRexYaml.PCAP_TEMPLATE) - new_pcap['name'] = self.trex_files_path + os.path.basename(local_pcap_path) - if self.get_pcap_idx(new_pcap['name']) != -1: - # pcap already exists in 'cap_info' - return -1 - else: - self.yaml_obj[0]['cap_info'].append(new_pcap) - if self.empty_cap: - self.empty_cap = False - self.file_list.append(local_pcap_path) - return ( len(self.yaml_obj[0]['cap_info']) - 1) - - - def get_pcap_idx (self, pcap_name): - """ - Checks if a certain .pcap file has been added into the yaml object. - - :parameters: - pcap_name : str - the name of the pcap file to be searched - - :return: - + The index of the pcap file (as int) if exists - + -1 if not exists. - - """ - comp_pcap = pcap_name if pcap_name.startswith(self.trex_files_path) else (self.trex_files_path + pcap_name) - for idx, pcap in enumerate(self.yaml_obj[0]['cap_info']): - print (pcap['name'] == comp_pcap) - if pcap['name'] == comp_pcap: - return idx - # pcap file wasn't found - return -1 - - def dump_as_python_obj (self): - """ - dumps with nice indentation the pythonic format (dictionaries and lists) of the currently built yaml object. - - :parameters: - None - - :return: - None - - """ - pprint.pprint(self.yaml_obj) - - def dump(self): - """ - dumps with nice indentation the YAML format of the currently built yaml object. - - :parameters: - None - - :return: - None - - """ - print (yaml.safe_dump(self.yaml_obj, default_flow_style = False)) - - def to_yaml(self, filename): - """ - Exports to YAML file the built configuration into an actual YAML file. - - :parameters: - filename : str - a path (on client side, including filename) to store the generated yaml file. - - :return: - None - - :raises: - + :exc:`ValueError`, in case no pcap files has been added to the object. - + :exc:`EnvironmentError`, in case of any IO error of writing to the files or OSError when trying to open it for writing. - - """ - if self.empty_cap: - raise ValueError("No .pcap file has been assigned to yaml object. Must add at least one") - else: - try: - with open(filename, 'w') as yaml_file: - yaml_file.write( yaml.safe_dump(self.yaml_obj, default_flow_style = False) ) - self.yaml_dumped = True - self.file_list.append(filename) - except EnvironmentError as inst: - raise - - def set_cap_info_param (self, param, value, seq): - """ - Set cap-info parameters' value of a specific pcap file. - - :parameters: - param : str - the name of the parameters to be set. - value : int/float - the desired value to be set to `param` key. - seq : int - an index to the relevant caps array to be changed (index supplied when adding new pcap file, see :func:`add_pcap_file`). - - :return: - **True** on success - - :raises: - :exc:`IndexError`, in case an out-of range index was given. - - """ - try: - self.yaml_obj[0]['cap_info'][seq][param] = value - - return True - except IndexError: - return False - - def set_generator_param (self, param, value): - """ - Set generator parameters' value of the yaml object. - - :parameters: - param : str - the name of the parameters to be set. - value : int/float/str - the desired value to be set to `param` key. - - :return: - None - - """ - self.yaml_obj[0]['generator'][param] = value - - def get_file_list(self): - """ - Returns a list of all files related to the YAML object, including the YAML filename itself. - - .. tip:: This method is especially useful for listing all the files that should be pushed to TRex server as part of the same yaml selection. - - :parameters: - None - - :return: - a list of filepaths, each is a local client-machine file path. - - """ - if not self.yaml_dumped: - print ("WARNING: .yaml file wasn't dumped yet. Files list contains only .pcap files") - return self.file_list - - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/client_utils/yaml_utils.py b/scripts/automation/trex_control_plane/client_utils/yaml_utils.py deleted file mode 100755 index 776a51a7..00000000 --- a/scripts/automation/trex_control_plane/client_utils/yaml_utils.py +++ /dev/null @@ -1,163 +0,0 @@ - -""" -Dan Klein -Cisco Systems, Inc. - -Copyright (c) 2015-2015 Cisco Systems, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import traceback -import sys -import yaml - - -class CTRexYAMLLoader(object): - TYPE_DICT = {"double":float, - "int":int, - "array":list, - "string":str, - "boolean":bool} - - def __init__(self, yaml_ref_file_path): - self.yaml_path = yaml_ref_file_path - self.ref_obj = None - - def check_term_param_type(self, val, val_field, ref_val, multiplier): - # print val, val_field, ref_val - tmp_type = ref_val.get('type') - if isinstance(tmp_type, list): - # item can be one of multiple types - # print "multiple choice!" - python_types = set() - for t in tmp_type: - if t in self.TYPE_DICT: - python_types.add(self.TYPE_DICT.get(t)) - else: - return False, TypeError("Unknown resolving for type {0}".format(t)) - # print "python legit types: ", python_types - if type(val) not in python_types: - return False, TypeError("Type of object field '{0}' is not allowed".format(val_field)) - else: - # WE'RE OK! - return True, CTRexYAMLLoader._calc_final_value(val, multiplier, ref_val.get('multiply', False)) - else: - # this is a single type field - python_type = self.TYPE_DICT.get(tmp_type) - if not isinstance(val, python_type): - return False, TypeError("Type of object field '{0}' is not allowed".format(val_field)) - else: - # WE'RE OK! - return True, CTRexYAMLLoader._calc_final_value(val, multiplier, ref_val.get('multiply', False)) - - def get_reference_default(self, root_obj, sub_obj, key): - # print root_obj, sub_obj, key - if sub_obj: - ref_field = self.ref_obj.get(root_obj).get(sub_obj).get(key) - else: - ref_field = self.ref_obj.get(root_obj).get(key) - if 'has_default' in ref_field: - if ref_field.get('has_default'): - # WE'RE OK! - return True, ref_field.get('default') - else: - # This is a mandatory field! - return False, ValueError("The {0} field is mandatory and must be specified explicitly".format(key)) - else: - return False, ValueError("The {0} field has no indication about default value".format(key)) - - def validate_yaml(self, evaluated_obj, root_obj, fill_defaults=True, multiplier=1): - if isinstance(evaluated_obj, dict) and evaluated_obj.keys() == [root_obj]: - evaluated_obj = evaluated_obj.get(root_obj) - if not self.ref_obj: - self.ref_obj = load_yaml_to_obj(self.yaml_path) - # self.load_reference() - ref_item = self.ref_obj.get(root_obj) - if ref_item is not None: - try: - typed_obj = [False, None] # first item stores validity (multiple object "shapes"), second stored type - if "type" in evaluated_obj: - ref_item = ref_item[evaluated_obj.get("type")] - # print "lower resolution with typed object" - typed_obj = [True, evaluated_obj.get("type")] - if isinstance(ref_item, dict) and "type" not in ref_item: # this is not a terminal - result_obj = {} - if typed_obj[0]: - result_obj["type"] = typed_obj[1] - # print "processing dictionary non-terminal value" - for k, v in ref_item.items(): - # print "processing element '{0}' with value '{1}'".format(k,v) - if k in evaluated_obj: - # validate with ref obj - # print "found in evaluated object!" - tmp_type = v.get('type') - # print tmp_type - # print evaluated_obj - if tmp_type == "object": - # go deeper into nesting hierarchy - # print "This is an object type, recursion!" - result_obj[k] = self.validate_yaml(evaluated_obj.get(k), k, fill_defaults, multiplier) - else: - # validation on terminal type - # print "Validating terminal type %s" % k - res_ok, data = self.check_term_param_type(evaluated_obj.get(k), k, v, multiplier) - if res_ok: - # data field contains the value to save - result_obj[k] = data - else: - # data var contains the exception to throw - raise data - elif fill_defaults: - # complete missing values with default value, if exists - sub_obj = typed_obj[1] if typed_obj[0] else None - res_ok, data = self.get_reference_default(root_obj, sub_obj, k) - if res_ok: - # data field contains the value to save - result_obj[k] = data - else: - # data var contains the exception to throw - raise data - return result_obj - elif isinstance(ref_item, list): - # currently not handling list objects - return NotImplementedError("List object are currently unsupported") - else: - raise TypeError("Unknown parse tree object type.") - except KeyError as e: - raise - else: - raise KeyError("The given root_key '{key}' does not exists on reference object".format(key=root_obj)) - - @staticmethod - def _calc_final_value(val, multiplier, multiply): - def to_num(s): - try: - return int(s) - except ValueError: - return float(s) - if multiply: - return val * to_num(multiplier) - else: - return val - - -def load_yaml_to_obj(file_path): - try: - return yaml.load(file(file_path, 'r')) - except yaml.YAMLError as e: - raise - except Exception as e: - raise - -def yaml_exporter(file_path): - pass - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/dirtree_no_files.txt b/scripts/automation/trex_control_plane/dirtree_no_files.txt deleted file mode 100755 index b87c4167..00000000 --- a/scripts/automation/trex_control_plane/dirtree_no_files.txt +++ /dev/null @@ -1,11 +0,0 @@ -trex_control_plane/ -|-- Client -|-- Server -|-- common -`-- python_lib - |-- enum34-1.0.4 - |-- jsonrpclib-0.1.3 - |-- lockfile-0.10.2 - |-- python-daemon-2.0.5 - `-- zmq - diff --git a/scripts/automation/trex_control_plane/dirtree_with_files.txt b/scripts/automation/trex_control_plane/dirtree_with_files.txt deleted file mode 100755 index 5ce7cdfc..00000000 --- a/scripts/automation/trex_control_plane/dirtree_with_files.txt +++ /dev/null @@ -1,31 +0,0 @@ -trex_control_plane/ -|-- Client -| |-- __init__.py -| |-- outer_packages.py -| |-- python_lib -| `-- trex_client.py -|-- Server -| |-- CCustomLogger.py -| |-- outer_packages.py -| |-- trex_daemon_server -| |-- trex_daemon_server.py -| |-- trex_launch_thread.py -| |-- trex_server.py -| `-- zmq_monitor_thread.py -|-- __init__.py -|-- common -| |-- __init__.py -| |-- __init__.pyc -| |-- trex_status_e.py -| `-- trex_status_e.pyc -|-- dirtree_no_files.txt -|-- dirtree_with_files.txt -`-- python_lib - |-- __init__.py - |-- enum34-1.0.4 - |-- jsonrpclib-0.1.3 - |-- lockfile-0.10.2 - |-- python-daemon-2.0.5 - |-- zmq - `-- zmq_fedora.tar.gz - diff --git a/scripts/automation/trex_control_plane/stf/CCustomLogger.py b/scripts/automation/trex_control_plane/stf/CCustomLogger.py new file mode 100755 index 00000000..ecf7d519 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/CCustomLogger.py @@ -0,0 +1,100 @@ + +import sys +import os +import logging + + +def setup_custom_logger(name, log_path = None): + # first make sure path availabe +# if log_path is None: +# log_path = os.getcwd()+'/trex_log.log' +# else: +# directory = os.path.dirname(log_path) +# if not os.path.exists(directory): +# os.makedirs(directory) + logging.basicConfig(level = logging.INFO, + format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', + datefmt = '%m-%d %H:%M') +# filename= log_path, +# filemode= 'w') +# +# # define a Handler which writes INFO messages or higher to the sys.stderr +# consoleLogger = logging.StreamHandler() +# consoleLogger.setLevel(logging.ERROR) +# # set a format which is simpler for console use +# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') +# # tell the handler to use this format +# consoleLogger.setFormatter(formatter) +# +# # add the handler to the logger +# logging.getLogger(name).addHandler(consoleLogger) + +def setup_daemon_logger (name, log_path = None): + # first make sure path availabe + logging.basicConfig(level = logging.INFO, + format = '%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s', + datefmt = '%m-%d %H:%M', + filename= log_path, + filemode= 'w') + +class CustomLogger(object): + + def __init__(self, log_filename): + # Store the original stdout and stderr + sys.stdout.flush() + sys.stderr.flush() + + self.stdout_fd = os.dup(sys.stdout.fileno()) + self.devnull = os.open('/dev/null', os.O_WRONLY) + self.log_file = open(log_filename, 'w') + self.silenced = False + self.pending_log_file_prints = 0 + + # silence all prints from stdout + def silence(self): + os.dup2(self.devnull, sys.stdout.fileno()) + self.silenced = True + + # restore stdout status + def restore(self): + sys.stdout.flush() + sys.stderr.flush() + # Restore normal stdout + os.dup2(self.stdout_fd, sys.stdout.fileno()) + self.silenced = False + + #print a message to the log (both stdout / log file) + def log(self, text, force = False, newline = True): + self.log_file.write((text + "\n") if newline else text) + self.pending_log_file_prints += 1 + + if (self.pending_log_file_prints >= 10): + self.log_file.flush() + self.pending_log_file_prints = 0 + + self.console(text, force, newline) + + # print a message to the console alone + def console(self, text, force = False, newline = True): + _text = (text + "\n") if newline else text + # if we are silenced and not forced - go home + if self.silenced and not force: + return + + if self.silenced: + os.write(self.stdout_fd, _text) + else: + sys.stdout.write(_text) + + sys.stdout.flush() + + # flush + def flush(self): + sys.stdout.flush() + self.log_file.flush() + + def __exit__(self, type, value, traceback): + sys.stdout.flush() + self.log_file.flush() + os.close(self.devnull) + os.close(self.log_file) diff --git a/scripts/automation/trex_control_plane/stf/__init__.py b/scripts/automation/trex_control_plane/stf/__init__.py new file mode 100755 index 00000000..5a1da046 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/__init__.py @@ -0,0 +1 @@ +__all__ = ["trex_status_e", "trex_exceptions"] diff --git a/scripts/automation/trex_control_plane/stf/external_packages.py b/scripts/automation/trex_control_plane/stf/external_packages.py new file mode 100755 index 00000000..7353c397 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/external_packages.py @@ -0,0 +1,28 @@ +#!/router/bin/python + +import sys +import os + +CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) +ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory +PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) + +CLIENT_UTILS_MODULES = ['yaml-3.11' + ] + +def import_common_modules(): + # must be in a higher priority + sys.path.insert(0, PATH_TO_PYTHON_LIB) + sys.path.append(ROOT_PATH) + import_module_list(CLIENT_UTILS_MODULES) + + +def import_module_list(modules_list): + assert(isinstance(modules_list, list)) + for p in modules_list: + full_path = os.path.join(PATH_TO_PYTHON_LIB, p) + fix_path = os.path.normcase(full_path) + sys.path.insert(1, full_path) + +import_common_modules() + diff --git a/scripts/automation/trex_control_plane/stf/general_utils.py b/scripts/automation/trex_control_plane/stf/general_utils.py new file mode 100755 index 00000000..d2521f02 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/general_utils.py @@ -0,0 +1,95 @@ +#!/router/bin/python + +import sys +import site +import string +import random +import os + +try: + import pwd +except ImportError: + import getpass + pwd = None + +using_python_3 = True if sys.version_info.major == 3 else False + + +def user_input(): + if using_python_3: + return input() + else: + # using python version 2 + return raw_input() + +def get_current_user(): + if pwd: + return pwd.getpwuid(os.geteuid()).pw_name + else: + return getpass.getuser() + +def import_module_list_by_path (modules_list): + assert(isinstance(modules_list, list)) + for full_path in modules_list: + site.addsitedir(full_path) + +def find_path_to_pardir (pardir, base_path = os.getcwd() ): + """ + Finds the absolute path for some parent dir `pardir`, starting from base_path + + The request is only valid if the stop initiator is the same client as the TRex run initiator. + + :parameters: + pardir : str + name of an upper-level directory to which we want to find an absolute path for + base_path : str + a full (usually nested) path from which we want to find a parent folder. + + default value : **current working dir** + + :return: + string representation of the full path to + + """ + components = base_path.split(os.sep) + return str.join(os.sep, components[:components.index(pardir)+1]) + + +def random_id_gen(length=8): + """ + A generator for creating a random chars id of specific length + + :parameters: + length : int + the desired length of the generated id + + default: 8 + + :return: + a random id with each next() request. + """ + id_chars = string.ascii_lowercase + string.digits + while True: + return_id = '' + for i in range(length): + return_id += random.choice(id_chars) + yield return_id + +def id_count_gen(): + """ + A generator for creating an increasing id for objects, starting from 0 + + :parameters: + None + + :return: + an id (unsigned int) with each next() request. + """ + return_id = 0 + while True: + yield return_id + return_id += 1 + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/stf/outer_packages.py b/scripts/automation/trex_control_plane/stf/outer_packages.py new file mode 100755 index 00000000..5e29f8d6 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/outer_packages.py @@ -0,0 +1,30 @@ +#!/router/bin/python + +import sys +import os + + +CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) +PARENT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, 'external_libs')) +SCRIPTS_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, 'external_libs')) + +CLIENT_MODULES = ['enum34-1.0.4', + 'jsonrpclib-pelix-0.2.5', +# 'termstyle', +# 'yaml-3.11' + ] + + +def import_module_list(ext_libs_path): + for p in CLIENT_MODULES: + full_path = os.path.join(ext_libs_path, p) + if not os.path.exists(full_path): + raise Exception('Library %s is absent in path %s' % (p, ext_libs_path)) + sys.path.insert(1, full_path) + +if os.path.exists(PARENT_PATH): + import_module_list(PARENT_PATH) +elif os.path.exists(SCRIPTS_PATH): + import_module_list(SCRIPTS_PATH) +else: + raise Exception('Could not find external libs in path: %s' % [PARENT_PATH, SCRIPTS_PATH]) diff --git a/scripts/automation/trex_control_plane/stf/text_opts.py b/scripts/automation/trex_control_plane/stf/text_opts.py new file mode 100755 index 00000000..78a0ab1f --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/text_opts.py @@ -0,0 +1,192 @@ +import json +import re + +TEXT_CODES = {'bold': {'start': '\x1b[1m', + 'end': '\x1b[22m'}, + 'cyan': {'start': '\x1b[36m', + 'end': '\x1b[39m'}, + 'blue': {'start': '\x1b[34m', + 'end': '\x1b[39m'}, + 'red': {'start': '\x1b[31m', + 'end': '\x1b[39m'}, + 'magenta': {'start': '\x1b[35m', + 'end': '\x1b[39m'}, + 'green': {'start': '\x1b[32m', + 'end': '\x1b[39m'}, + 'yellow': {'start': '\x1b[33m', + 'end': '\x1b[39m'}, + 'underline': {'start': '\x1b[4m', + 'end': '\x1b[24m'}} + +class TextCodesStripper: + keys = [re.escape(v['start']) for k,v in TEXT_CODES.iteritems()] + keys += [re.escape(v['end']) for k,v in TEXT_CODES.iteritems()] + pattern = re.compile("|".join(keys)) + + @staticmethod + def strip (s): + return re.sub(TextCodesStripper.pattern, '', s) + +def format_num (size, suffix = "", compact = True, opts = ()): + txt = "NaN" + + if type(size) == str: + return "N/A" + + u = '' + + if compact: + for unit in ['','K','M','G','T','P']: + if abs(size) < 1000.0: + u = unit + break + size /= 1000.0 + + if isinstance(size, float): + txt = "%3.2f" % (size) + else: + txt = "{:,}".format(size) + + if u or suffix: + txt += " {:}{:}".format(u, suffix) + + if isinstance(opts, tuple): + return format_text(txt, *opts) + else: + return format_text(txt, (opts)) + + + +def format_time (t_sec): + if t_sec < 0: + return "infinite" + + if t_sec < 1: + # low numbers + for unit in ['ms', 'usec', 'ns']: + t_sec *= 1000.0 + if t_sec >= 1.0: + return '{:,.2f} [{:}]'.format(t_sec, unit) + + return "NaN" + + else: + # seconds + if t_sec < 60.0: + return '{:,.2f} [{:}]'.format(t_sec, 'sec') + + # minutes + t_sec /= 60.0 + if t_sec < 60.0: + return '{:,.2f} [{:}]'.format(t_sec, 'minutes') + + # hours + t_sec /= 60.0 + if t_sec < 24.0: + return '{:,.2f} [{:}]'.format(t_sec, 'hours') + + # days + t_sec /= 24.0 + return '{:,.2f} [{:}]'.format(t_sec, 'days') + + +def format_percentage (size): + return "%0.2f %%" % (size) + +def bold(text): + return text_attribute(text, 'bold') + + +def cyan(text): + return text_attribute(text, 'cyan') + + +def blue(text): + return text_attribute(text, 'blue') + + +def red(text): + return text_attribute(text, 'red') + + +def magenta(text): + return text_attribute(text, 'magenta') + + +def green(text): + return text_attribute(text, 'green') + +def yellow(text): + return text_attribute(text, 'yellow') + +def underline(text): + return text_attribute(text, 'underline') + + +def text_attribute(text, attribute): + if isinstance(text, str): + return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], + txt=text, + stop=TEXT_CODES[attribute]['end']) + elif isinstance(text, unicode): + return u"{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], + txt=text, + stop=TEXT_CODES[attribute]['end']) + else: + raise Exception("not a string") + + +FUNC_DICT = {'blue': blue, + 'bold': bold, + 'green': green, + 'yellow': yellow, + 'cyan': cyan, + 'magenta': magenta, + 'underline': underline, + 'red': red} + + +def format_text(text, *args): + return_string = text + for i in args: + func = FUNC_DICT.get(i) + if func: + return_string = func(return_string) + + return return_string + + +def format_threshold (value, red_zone, green_zone): + if value >= red_zone[0] and value <= red_zone[1]: + return format_text("{0}".format(value), 'red') + + if value >= green_zone[0] and value <= green_zone[1]: + return format_text("{0}".format(value), 'green') + + return "{0}".format(value) + +# pretty print for JSON +def pretty_json (json_str, use_colors = True): + pretty_str = json.dumps(json.loads(json_str), indent = 4, separators=(',', ': '), sort_keys = True) + + if not use_colors: + return pretty_str + + try: + # int numbers + pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*[^.])',r'\1{0}'.format(blue(r'\2')), pretty_str) + # float + pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*\.[0-9]+)',r'\1{0}'.format(magenta(r'\2')), pretty_str) + # # strings + # + pretty_str = re.sub(r'([ ]*:[ ]+)("[^"]*")',r'\1{0}'.format(red(r'\2')), pretty_str) + pretty_str = re.sub(r"('[^']*')", r'{0}\1{1}'.format(TEXT_CODES['magenta']['start'], + TEXT_CODES['red']['start']), pretty_str) + except : + pass + + return pretty_str + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/stf/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_client.py new file mode 100755 index 00000000..919253d1 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_client.py @@ -0,0 +1,1216 @@ +#!/router/bin/python + +# internal libs +import sys +import os +import socket +import errno +import time +import re +import copy +import binascii +from distutils.util import strtobool +from collections import deque, OrderedDict +from json import JSONDecoder +import traceback + +try: + from . import outer_packages + from .trex_status_e import TRexStatus + from .trex_exceptions import * + from .trex_exceptions import exception_handler + from .general_utils import * +except Exception as e: # is __main__ + import outer_packages + from trex_status_e import TRexStatus + from trex_exceptions import * + from trex_exceptions import exception_handler + from general_utils import * + +# external libs +import jsonrpclib +from jsonrpclib import ProtocolError, AppError +from enum import Enum + + + +class CTRexClient(object): + """ + This class defines the client side of the RESTfull interaction with TRex + """ + + def __init__(self, trex_host, max_history_size = 100, trex_daemon_port = 8090, trex_zmq_port = 4500, verbose = False): + """ + Instantiate a TRex client object, and connecting it to listening daemon-server + + :parameters: + trex_host : str + a string of the TRex ip address or hostname. + max_history_size : int + a number to set the maximum history size of a single TRex run. Each sampling adds a new item to history. + + default value : **100** + trex_daemon_port : int + the port number on which the trex-daemon server can be reached + + default value: **8090** + trex_zmq_port : int + the port number on which trex's zmq module will interact with daemon server + + default value: **4500** + verbose : bool + sets a verbose output on supported class method. + + default value : **False** + + :raises: + socket errors, in case server could not be reached. + + """ + try: + self.trex_host = socket.gethostbyname(trex_host) + except: # give it another try + self.trex_host = socket.gethostbyname(trex_host) + self.trex_daemon_port = trex_daemon_port + self.trex_zmq_port = trex_zmq_port + self.seq = None + self.verbose = verbose + self.result_obj = CTRexResult(max_history_size) + self.decoder = JSONDecoder() + self.trex_server_path = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port ) + self.__verbose_print("Connecting to TRex @ {trex_path} ...".format( trex_path = self.trex_server_path ) ) + self.history = jsonrpclib.history.History() + self.server = jsonrpclib.Server(self.trex_server_path, history = self.history) + self.check_server_connectivity() + self.__verbose_print("Connection established successfully!") + self._last_sample = time.time() + self.__default_user = get_current_user() + + + def add (self, x, y): + try: + return self.server.add(x,y) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def start_trex (self, f, d, block_to_success = True, timeout = 40, user = None, trex_development = False, **trex_cmd_options): + """ + Request to start a TRex run on server. + + :parameters: + f : str + a path (on server) for the injected traffic data (.yaml file) + d : int + the desired duration of the test. must be at least 30 seconds long. + block_to_success : bool + determine if this method blocks until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + default value : **True** + timeout : int + maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + default value: **40** + user : str + the identity of the the run issuer. + trex_cmd_options : key, val + sets desired TRex options using key=val syntax, separated by comma. + for keys with no value, state key=True + + :return: + **True** on success + + :raises: + + :exc:`ValueError`, in case 'd' parameter inserted with wrong value. + + :exc:`trex_exceptions.TRexError`, in case one of the trex_cmd_options raised an exception at server. + + :exc:`trex_exceptions.TRexInUseError`, in case TRex is already taken. + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying start TRex. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + user = user or self.__default_user + try: + d = int(d) + if d < 30 and not trex_development: # test duration should be at least 30 seconds, unless trex_development flag is specified. + raise ValueError + except ValueError: + raise ValueError('d parameter must be integer, specifying how long TRex run, and must be larger than 30 secs.') + + trex_cmd_options.update( {'f' : f, 'd' : d} ) + if not trex_cmd_options.get('l'): + self.result_obj.latency_checked = False + if 'k' in trex_cmd_options: + timeout += int(trex_cmd_options['k']) # during 'k' seconds TRex stays in 'Starting' state + + self.result_obj.clear_results() + try: + issue_time = time.time() + retval = self.server.start_trex(trex_cmd_options, user, block_to_success, timeout) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + if retval!=0: + self.seq = retval # update seq num only on successful submission + return True + else: # TRex is has been started by another user + raise TRexInUseError('TRex is already being used by another user or process. Try again once TRex is back in IDLE state.') + + def stop_trex (self): + """ + Request to stop a TRex run on server. + + The request is only valid if the stop initiator is the same client as the TRex run initiator. + + :parameters: + None + + :return: + + **True** on successful termination + + **False** if request issued but TRex wasn't running. + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex ir running but started by another user. + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return self.server.stop_trex(self.seq) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def force_kill (self, confirm = True): + """ + Force killing of running TRex process (if exists) on the server. + + .. tip:: This method is a safety method and **overrides any running or reserved resources**, and as such isn't designed to be used on a regular basis. + Always consider using :func:`trex_client.CTRexClient.stop_trex` instead. + + In the end of this method, TRex will return to IDLE state with no reservation. + + :parameters: + confirm : bool + Prompt a user confirmation before continue terminating TRex session + + :return: + + **True** on successful termination + + **False** otherwise. + + :raises: + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + if confirm: + prompt = "WARNING: This will terminate active TRex session indiscriminately.\nAre you sure? " + sys.stdout.write('%s [y/n]\n' % prompt) + while True: + try: + if strtobool(user_input().lower()): + break + else: + return + except ValueError: + sys.stdout.write('Please respond with \'y\' or \'n\'.\n') + try: + return self.server.force_trex_kill() + except AppError as err: + # Silence any kind of application errors- by design + return False + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def wait_until_kickoff_finish(self, timeout = 40): + """ + Block the client application until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + The request is only valid if the stop initiator is the same client as the TRex run initiator. + + :parameters: + timeout : int + maximum time (in seconds) to wait in blocking state until TRex changes state from 'Starting' to either 'Idle' or 'Running' + + :return: + + **True** on successful termination + + **False** if request issued but TRex wasn't running. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + ProtocolError, in case of error in JSON-RPC protocol. + + .. note:: Exceptions are throws only when start_trex did not block in the first place, i.e. `block_to_success` parameter was set to `False` + + """ + + try: + return self.server.wait_until_kickoff_finish(timeout) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def is_running (self, dump_out = False): + """ + Poll for TRex running status. + + If TRex is running, a history item will be added into result_obj and processed. + + .. tip:: This method is especially useful for iterating until TRex run is finished. + + :parameters: + dump_out : dict + if passed, the pointer object is cleared and the latest dump stored in it. + + :return: + + **True** if TRex is running. + + **False** if TRex is not running. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + res = self.get_running_info() + if res == {}: + return False + if (dump_out != False) and (isinstance(dump_out, dict)): # save received dump to given 'dump_out' pointer + dump_out.clear() + dump_out.update(res) + return True + except TRexWarning as err: + if err.code == -12: # TRex is either still at 'Starting' state or in Idle state, however NO error occured + return False + except TRexException: + raise + except ProtocolError as err: + raise + finally: + self.prompt_verbose_data() + + def is_idle (self): + """ + Poll for TRex running status, check if TRex is in Idle state. + + :parameters: + None + + :return: + + **True** if TRex is idle. + + **False** if TRex is starting or running. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + if self.get_running_status()['state'] == TRexStatus.Idle: + return True + return False + except TRexException: + raise + except ProtocolError as err: + raise + finally: + self.prompt_verbose_data() + + def get_trex_files_path (self): + """ + Fetches the local path in which files are stored when pushed to TRex server from client. + + :parameters: + None + + :return: + string representation of the desired path + + .. note:: The returned path represents a path on the TRex server **local machine** + + :raises: + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return (self.server.get_files_path() + '/') + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_running_status (self): + """ + Fetches the current TRex status. + + If available, a verbose data will accompany the state itself. + + :parameters: + None + + :return: + dictionary with 'state' and 'verbose' keys. + + :raises: + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + res = self.server.get_running_status() + res['state'] = TRexStatus(res['state']) + return res + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_running_info (self): + """ + Performs single poll of TRex running data and process it into the result object (named `result_obj`). + + .. tip:: This method will throw an exception if TRex isn't running. Always consider using :func:`trex_client.CTRexClient.is_running` which handles a single poll operation in safer manner. + + :parameters: + None + + :return: + dictionary containing the most updated data dump from TRex. + + :raises: + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + if not self.is_query_relevance(): + # if requested in timeframe smaller than the original sample rate, return the last known data without interacting with server + return self.result_obj.get_latest_dump() + else: + try: + latest_dump = self.decoder.decode( self.server.get_running_info() ) # latest dump is not a dict, but json string. decode it. + self.result_obj.update_result_data(latest_dump) + return latest_dump + except TypeError as inst: + raise TypeError('JSON-RPC data decoding failed. Check out incoming JSON stream.') + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def sample_until_condition (self, condition_func, time_between_samples = 5): + """ + Automatically sets ongoing sampling of TRex data, with sampling rate described by time_between_samples. + + On each fetched dump, the condition_func is applied on the result objects, and if returns True, the sampling will stop. + + :parameters: + condition_func : function + function that operates on result_obj and checks if a condition has been met + + .. note:: `condition_finc` is applied on `CTRexResult` object. Make sure to design a relevant method. + time_between_samples : int + determines the time between each sample of the server + + default value : **5** + + :return: + the first result object (see :class:`CTRexResult` for further details) of the TRex run on which the condition has been met. + + :raises: + + :exc:`UserWarning`, in case the condition_func method condition hasn't been met + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + :exc:`Exception`, in case the condition_func suffered from any kind of exception + + """ + # make sure TRex is running. raise exceptions here if any + self.wait_until_kickoff_finish() + try: + while self.is_running(): + results = self.get_result_obj() + if condition_func(results): + # if condition satisfied, stop TRex and return result object + self.stop_trex() + return results + time.sleep(time_between_samples) + except TRexWarning: + # means we're back to Idle state, and didn't meet our condition + raise UserWarning("TRex results condition wasn't met during TRex run.") + except Exception: + # this could come from provided method 'condition_func' + raise + + def sample_to_run_finish (self, time_between_samples = 5): + """ + Automatically sets automatically sampling of TRex data with sampling rate described by time_between_samples until TRex run finished. + + :parameters: + time_between_samples : int + determines the time between each sample of the server + + default value : **5** + + :return: + the latest result object (see :class:`CTRexResult` for further details) with sampled data. + + :raises: + + :exc:`UserWarning`, in case the condition_func method condition hasn't been met + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + self.wait_until_kickoff_finish() + + try: + while self.is_running(): + time.sleep(time_between_samples) + except TRexWarning: + pass + results = self.get_result_obj() + return results + + def sample_x_seconds (self, sample_time, time_between_samples = 5): + """ + Automatically sets ongoing sampling of TRex data for sample_time seconds, with sampling rate described by time_between_samples. + Does not stop the TRex afterwards! + + .. tip:: Useful for changing the device (Router, ASA etc.) configuration after given time. + + :parameters: + sample_time : int + sample the TRex this number of seconds + + time_between_samples : int + determines the time between each sample of the server + + default value : **5** + + :return: + the first result object (see :class:`CTRexResult` for further details) of the TRex run after given sample_time. + + :raises: + + :exc:`UserWarning`, in case the TRex run ended before sample_time duration + + :exc:`trex_exceptions.TRexIncompleteRunError`, in case one of failed TRex run (unexpected termination). + + :exc:`TypeError`, in case JSON stream decoding error. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + # make sure TRex is running. raise exceptions here if any + self.wait_until_kickoff_finish() + elapsed_time = 0 + while self.is_running(): + if elapsed_time >= sample_time: + return self.get_result_obj() + time.sleep(time_between_samples) + elapsed_time += time_between_samples + raise UserWarning("TRex has stopped at %s seconds (before expected %s seconds)\nTry increasing test duration or decreasing sample_time" % (elapsed_time, sample_time)) + + def get_result_obj (self, copy_obj = True): + """ + Returns the result object of the trex_client's instance. + + By default, returns a **copy** of the objects (so that changes to the original object are masked). + + :parameters: + copy_obj : bool + False means that a reference to the original (possibly changing) object are passed + + defaul value : **True** + + :return: + the latest result object (see :class:`CTRexResult` for further details) with sampled data. + + """ + if copy_obj: + return copy.deepcopy(self.result_obj) + else: + return self.result_obj + + def is_reserved (self): + """ + Checks if TRex is currently reserved to any user or not. + + :parameters: + None + + :return: + + **True** if TRex is reserved. + + **False** otherwise. + + :raises: + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return self.server.is_reserved() + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_trex_daemon_log (self): + """ + Get Trex daemon log. + + :return: + String representation of TRex daemon log + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case file could not be read. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return binascii.a2b_base64(self.server.get_trex_daemon_log()) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_trex_log (self): + """ + Get TRex CLI output log + + :return: + String representation of TRex log + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case file could not be fetched at server side. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + try: + return binascii.a2b_base64(self.server.get_trex_log()) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def get_trex_version (self): + """ + Get TRex version details. + + :return: + Trex details (Version, User, Date, Uuid, Git SHA) as ordered dictionary + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex version could not be determined. + + ProtocolError, in case of error in JSON-RPC protocol. + + General Exception is case one of the keys is missing in response + """ + + try: + version_dict = OrderedDict() + result_lines = binascii.a2b_base64(self.server.get_trex_version()).split('\n') + for line in result_lines: + if not line: + continue + key, value = line.strip().split(':', 1) + version_dict[key.strip()] = value.strip() + for key in ('Version', 'User', 'Date', 'Uuid', 'Git SHA'): + if key not in version_dict: + raise Exception('get_trex_version: got server response without key: {0}'.format(key)) + return version_dict + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def reserve_trex (self, user = None): + """ + Reserves the usage of TRex to a certain user. + + When TRex is reserved, it can't be reserved. + + :parameters: + user : str + a username of the desired owner of TRex + + default: current logged user + + :return: + **True** if reservation made successfully + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying to make the reservation. + + :exc:`trex_exceptions.TRexInUseError`, in case TRex is currently running. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + username = user or self.__default_user + try: + return self.server.reserve_trex(user = username) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def cancel_reservation (self, user = None): + """ + Cancels a current reservation of TRex to a certain user. + + When TRex is reserved, no other user can start new TRex runs. + + + :parameters: + user : str + a username of the desired owner of TRex + + default: current logged user + + :return: + + **True** if reservation canceled successfully, + + **False** if there was no reservation at all. + + :raises: + + :exc:`trex_exceptions.TRexRequestDenied`, in case TRex is reserved for another user than the one trying to cancel the reservation. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + + username = user or self.__default_user + try: + return self.server.cancel_reservation(user = username) + except AppError as err: + self._handle_AppError_exception(err.args[0]) + except ProtocolError: + raise + finally: + self.prompt_verbose_data() + + def push_files (self, filepaths): + """ + Pushes a file (or a list of files) to store locally on server. + + :parameters: + filepaths : str or list + a path to a file to be pushed to server. + if a list of paths is passed, all of those will be pushed to server + + :return: + + **True** if file(s) copied successfully. + + **False** otherwise. + + :raises: + + :exc:`IOError`, in case specified file wasn't found or could not be accessed. + + ProtocolError, in case of error in JSON-RPC protocol. + + """ + paths_list = None + if isinstance(filepaths, str): + paths_list = [filepaths] + elif isinstance(filepaths, list): + paths_list = filepaths + else: + raise TypeError("filepaths argument must be of type str or list") + + for filepath in paths_list: + try: + if not os.path.exists(filepath): + raise IOError(errno.ENOENT, "The requested `{fname}` file wasn't found. Operation aborted.".format( + fname = filepath) ) + else: + filename = os.path.basename(filepath) + with open(filepath, 'rb') as f: + file_content = f.read() + self.server.push_file(filename, binascii.b2a_base64(file_content)) + finally: + self.prompt_verbose_data() + return True + + def is_query_relevance(self): + """ + Checks if time between any two consecutive server queries (asking for live running data) passed. + + .. note:: The allowed minimum time between each two consecutive samples is 0.5 seconds. + + :parameters: + None + + :return: + + **True** if more than 0.5 seconds has been past from last server query. + + **False** otherwise. + + """ + cur_time = time.time() + if cur_time-self._last_sample < 0.5: + return False + else: + self._last_sample = cur_time + return True + + def call_server_mathod_safely (self, method_to_call): + try: + return method_to_call() + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + raise SocketError(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") + + def check_server_connectivity (self): + """ + Checks for server valid connectivity. + """ + try: + socket.gethostbyname(self.trex_host) + return self.server.connectivity_check() + except socket.gaierror as e: + raise socket.gaierror(e.errno, "Could not resolve server hostname. Please make sure hostname entered correctly.") + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + raise socket.error(errno.ECONNREFUSED, "Connection from TRex server was refused. Please make sure the server is up.") + finally: + self.prompt_verbose_data() + + def prompt_verbose_data(self): + """ + This method prompts any verbose data available, only if `verbose` option has been turned on. + """ + if self.verbose: + print ('\n') + print ("(*) JSON-RPC request:", self.history.request) + print ("(*) JSON-RPC response:", self.history.response) + + def __verbose_print(self, print_str): + """ + This private method prints the `print_str` string only in case self.verbose flag is turned on. + + :parameters: + print_str : str + a string to be printed + + :returns: + None + """ + if self.verbose: + print (print_str) + + + + def _handle_AppError_exception(self, err): + """ + This private method triggres the TRex dedicated exception generation in case a general ProtocolError has been raised. + """ + # handle known exceptions based on known error codes. + # if error code is not known, raise ProtocolError + raise exception_handler.gen_exception(err) + + +class CTRexResult(object): + """ + A class containing all results received from TRex. + + Ontop to containing the results, this class offers easier data access and extended results processing options + """ + def __init__(self, max_history_size): + """ + Instatiate a TRex result object + + :parameters: + max_history_size : int + a number to set the maximum history size of a single TRex run. Each sampling adds a new item to history. + + """ + self._history = deque(maxlen = max_history_size) + self.clear_results() + self.latency_checked = True + + def __repr__(self): + return ("Is valid history? {arg}\n".format( arg = self.is_valid_hist() ) + + "Done warmup? {arg}\n".format( arg = self.is_done_warmup() ) + + "Expected tx rate: {arg}\n".format( arg = self.get_expected_tx_rate() ) + + "Current tx rate: {arg}\n".format( arg = self.get_current_tx_rate() ) + + "Maximum latency: {arg}\n".format( arg = self.get_max_latency() ) + + "Average latency: {arg}\n".format( arg = self.get_avg_latency() ) + + "Average window latency: {arg}\n".format( arg = self.get_avg_window_latency() ) + + "Total drops: {arg}\n".format( arg = self.get_total_drops() ) + + "Drop rate: {arg}\n".format( arg = self.get_drop_rate() ) + + "History size so far: {arg}\n".format( arg = len(self._history) ) ) + + def get_expected_tx_rate (self): + """ + Fetches the expected TX rate in various units representation + + :parameters: + None + + :return: + dictionary containing the expected TX rate, where the key is the measurement units, and the value is the measurement value. + + """ + return self._expected_tx_rate + + def get_current_tx_rate (self): + """ + Fetches the current TX rate in various units representation + + :parameters: + None + + :return: + dictionary containing the current TX rate, where the key is the measurement units, and the value is the measurement value. + + """ + return self._current_tx_rate + + def get_max_latency (self): + """ + Fetches the maximum latency measured on each of the interfaces + + :parameters: + None + + :return: + dictionary containing the maximum latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. + + """ + return self._max_latency + + def get_avg_latency (self): + """ + Fetches the average latency measured on each of the interfaces from the start of TRex run + + :parameters: + None + + :return: + dictionary containing the average latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. + + The `all` key represents the average of all interfaces' average + + """ + return self._avg_latency + + def get_avg_window_latency (self): + """ + Fetches the average latency measured on each of the interfaces from all the sampled currently stored in window. + + :parameters: + None + + :return: + dictionary containing the average latency, where the key is the measurement interface (`c` indicates client), and the value is the measurement value. + + The `all` key represents the average of all interfaces' average + + """ + return self._avg_window_latency + + def get_total_drops (self): + """ + Fetches the total number of drops identified from the moment TRex run began. + + :parameters: + None + + :return: + total drops count (as int) + + """ + return self._total_drops + + def get_drop_rate (self): + """ + Fetches the most recent drop rate in pkts/sec units. + + :parameters: + None + + :return: + current drop rate (as float) + + """ + return self._drop_rate + + def is_valid_hist (self): + """ + Checks if result obejct contains valid data. + + :parameters: + None + + :return: + + **True** if history is valid. + + **False** otherwise. + + """ + return self.valid + + def set_valid_hist (self, valid_stat = True): + """ + Sets result obejct validity status. + + :parameters: + valid_stat : bool + defines the validity status + + dafault value : **True** + + :return: + None + + """ + self.valid = valid_stat + + def is_done_warmup (self): + """ + Checks if TRex latest results TX-rate indicates that TRex has reached its expected TX-rate. + + :parameters: + None + + :return: + + **True** if expected TX-rate has been reached. + + **False** otherwise. + + """ + return self._done_warmup + + def get_last_value (self, tree_path_to_key, regex = None): + """ + A dynamic getter from the latest sampled data item stored in the result object. + + :parameters: + tree_path_to_key : str + defines a path to desired data. + + .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. + | Use '[i]' to access the i'th indexed object of an array. + + tree_path_to_key : regex + apply a regex to filter results out from a multiple results set. + + Filter applies only on keys of dictionary type. + + dafault value : **None** + + :return: + + a list of values relevant to the specified path + + None if no results were fetched or the history isn't valid. + + """ + if not self.is_valid_hist(): + return None + else: + return CTRexResult.__get_value_by_path(self._history[len(self._history)-1], tree_path_to_key, regex) + + def get_value_list (self, tree_path_to_key, regex = None, filter_none = True): + """ + A dynamic getter from all sampled data items stored in the result object. + + :parameters: + tree_path_to_key : str + defines a path to desired data. + + .. tip:: | Use '.' to enter one level deeper in dictionary hierarchy. + | Use '[i]' to access the i'th indexed object of an array. + + tree_path_to_key : regex + apply a regex to filter results out from a multiple results set. + + Filter applies only on keys of dictionary type. + + dafault value : **None** + + filter_none : bool + specify if None results should be filtered out or not. + + dafault value : **True** + + :return: + + a list of values relevant to the specified path. Each item on the list refers to a single server sample. + + None if no results were fetched or the history isn't valid. + """ + + if not self.is_valid_hist(): + return None + else: + raw_list = list( map(lambda x: CTRexResult.__get_value_by_path(x, tree_path_to_key, regex), self._history) ) + if filter_none: + return list (filter(lambda x: x!=None, raw_list) ) + else: + return raw_list + + def get_latest_dump(self): + """ + A getter to the latest sampled data item stored in the result object. + + :parameters: + None + + :return: + + a dictionary of the latest data item + + an empty dictionary if history is empty. + + """ + history_size = len(self._history) + if history_size != 0: + return self._history[len(self._history) - 1] + else: + return {} + + def update_result_data (self, latest_dump): + """ + Integrates a `latest_dump` dictionary into the CTRexResult object. + + :parameters: + latest_dump : dict + a dictionary with the items desired to be integrated into the object history and stats + + :return: + None + + """ + # add latest dump to history + if latest_dump != {}: + self._history.append(latest_dump) + if not self.valid: + self.valid = True + + # parse important fields and calculate averages and others + if self._expected_tx_rate is None: + # get the expected data only once since it doesn't change + self._expected_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_expected_\w+") + + self._current_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_(?!expected_)\w+") + if not self._done_warmup and self._expected_tx_rate is not None: + # check for up to 2% change between expected and actual + if (self._current_tx_rate['m_tx_bps']/self._expected_tx_rate['m_tx_expected_bps'] > 0.98): + self._done_warmup = True + + # handle latency data + if self.latency_checked: + latency_pre = "trex-latency" + self._max_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "max-")#None # TBC + # support old typo + if self._max_latency is None: + latency_pre = "trex-latecny" + self._max_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "max-") + + self._avg_latency = self.get_last_value("{latency}.data".format(latency = latency_pre), "avg-")#None # TBC + self._avg_latency = CTRexResult.__avg_all_and_rename_keys(self._avg_latency) + + avg_win_latency_list = self.get_value_list("{latency}.data".format(latency = latency_pre), "avg-") + self._avg_window_latency = CTRexResult.__calc_latency_win_stats(avg_win_latency_list) + + tx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_tx_pkts") + rx_pkts = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_total_rx_pkts") + if tx_pkts is not None and rx_pkts is not None: + self._total_drops = tx_pkts - rx_pkts + self._drop_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data.m_rx_drop_bps") + + def clear_results (self): + """ + Clears all results and sets the history's validity to `False` + + :parameters: + None + + :return: + None + + """ + self.valid = False + self._done_warmup = False + self._expected_tx_rate = None + self._current_tx_rate = None + self._max_latency = None + self._avg_latency = None + self._avg_window_latency = None + self._total_drops = None + self._drop_rate = None + self._history.clear() + + @staticmethod + def __get_value_by_path (dct, tree_path, regex = None): + try: + for i, p in re.findall(r'(\d+)|([\w|-]+)', tree_path): + dct = dct[p or int(i)] + if regex is not None and isinstance(dct, dict): + res = {} + for key,val in dct.items(): + match = re.match(regex, key) + if match: + res[key]=val + return res + else: + return dct + except (KeyError, TypeError): + return None + + @staticmethod + def __calc_latency_win_stats (latency_win_list): + res = {'all' : None } + port_dict = {'all' : []} + list( map(lambda x: CTRexResult.__update_port_dict(x, port_dict), latency_win_list) ) + + # finally, calculate everages for each list + res['all'] = float("%.3f" % (sum(port_dict['all'])/float(len(port_dict['all']))) ) + port_dict.pop('all') + for port, avg_list in port_dict.items(): + res[port] = float("%.3f" % (sum(avg_list)/float(len(avg_list))) ) + + return res + + @staticmethod + def __update_port_dict (src_avg_dict, dest_port_dict): + all_list = src_avg_dict.values() + dest_port_dict['all'].extend(all_list) + for key, val in src_avg_dict.items(): + reg_res = re.match("avg-(\d+)", key) + if reg_res: + tmp_key = "port"+reg_res.group(1) + if tmp_key in dest_port_dict: + dest_port_dict[tmp_key].append(val) + else: + dest_port_dict[tmp_key] = [val] + + @staticmethod + def __avg_all_and_rename_keys (src_dict): + res = {} + all_list = src_dict.values() + res['all'] = float("%.3f" % (sum(all_list)/float(len(all_list))) ) + for key, val in src_dict.items(): + reg_res = re.match("avg-(\d+)", key) + if reg_res: + tmp_key = "port"+reg_res.group(1) + res[tmp_key] = val # don't touch original fields values + return res + + + +if __name__ == "__main__": + pass + diff --git a/scripts/automation/trex_control_plane/stf/trex_daemon_server.py b/scripts/automation/trex_control_plane/stf/trex_daemon_server.py new file mode 100755 index 00000000..9784d42a --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_daemon_server.py @@ -0,0 +1,79 @@ +#!/usr/bin/python + +import outer_packages +import daemon +from trex_server import do_main_program, trex_parser +import CCustomLogger + +import logging +import time +import sys +import os, errno +import grp +import signal +from daemon import runner +from extended_daemon_runner import ExtendedDaemonRunner +import lockfile +import errno + +class TRexServerApp(object): + def __init__(self): + TRexServerApp.create_working_dirs() + self.stdin_path = '/dev/null' + self.stdout_path = '/dev/tty' # All standard prints will come up from this source. + self.stderr_path = "/var/log/trex/trex_daemon_server.log" # All log messages will come up from this source + self.pidfile_path = '/var/run/trex/trex_daemon_server.pid' + self.pidfile_timeout = 5 # timeout in seconds + + def run(self): + do_main_program() + + + @staticmethod + def create_working_dirs(): + if not os.path.exists('/var/log/trex'): + os.mkdir('/var/log/trex') + if not os.path.exists('/var/run/trex'): + os.mkdir('/var/run/trex') + + + +def main (): + + trex_app = TRexServerApp() + + # setup the logger + default_log_path = '/var/log/trex/trex_daemon_server.log' + + try: + CCustomLogger.setup_daemon_logger('TRexServer', default_log_path) + logger = logging.getLogger('TRexServer') + logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(name)-10s %(module)-20s %(levelname)-8s %(message)s") + handler = logging.FileHandler("/var/log/trex/trex_daemon_server.log") + logger.addHandler(handler) + except EnvironmentError, e: + if e.errno == errno.EACCES: # catching permission denied error + print "Launching user must have sudo privileges in order to run TRex daemon.\nTerminating daemon process." + exit(-1) + + daemon_runner = ExtendedDaemonRunner(trex_app, trex_parser) + + #This ensures that the logger file handle does not get closed during daemonization + daemon_runner.daemon_context.files_preserve=[handler.stream] + + try: + if not set(['start', 'stop']).isdisjoint(set(sys.argv)): + print "Logs are saved at: {log_path}".format( log_path = default_log_path ) + daemon_runner.do_action() + + except lockfile.LockTimeout as inst: + logger.error(inst) + print inst + print """ + Please try again once the timeout has been reached. + If this error continues, consider killing the process manually and restart the daemon.""" + + +if __name__ == "__main__": + main() diff --git a/scripts/automation/trex_control_plane/stf/trex_exceptions.py b/scripts/automation/trex_control_plane/stf/trex_exceptions.py new file mode 100755 index 00000000..0de38411 --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_exceptions.py @@ -0,0 +1,140 @@ +#!/router/bin/python + +#from rpc_exceptions import RPCExceptionHandler, WrappedRPCError + +from jsonrpclib import Fault, ProtocolError, AppError + +class RPCError(Exception): + """ + This is the general RPC error exception class from which :exc:`trex_exceptions.TRexException` inherits. + + Every exception in this class has as error format according to JSON-RPC convention convention: code, message and data. + + """ + def __init__(self, code, message, remote_data = None): + self.code = code + self.msg = message or self._default_message + self.data = remote_data + self.args = (code, self.msg, remote_data) + + def __str__(self): + return self.__repr__() + def __repr__(self): + if self.args[2] is not None: + return u"[errcode:%r] %r. Extended data: %r" % (self.args[0], self.args[1], self.args[2]) + else: + return u"[errcode:%r] %r" % (self.args[0], self.args[1]) + +class TRexException(RPCError): + """ + This is the most general TRex exception. + + All exceptions inherits from this class has an error code and a default message which describes the most common use case of the error. + + This exception isn't used by default and will only when an unrelated to ProtocolError will occur, and it can't be resolved to any of the deriviate exceptions. + + """ + code = -10 + _default_message = 'TRex encountered an unexpected error. please contact TRex dev team.' + # api_name = 'TRex' + +class TRexError(TRexException): + """ + This is the most general TRex exception. + + This exception isn't used by default and will only when an unrelated to ProtocolError will occur, and it can't be resolved to any of the deriviate exceptions. + """ + code = -11 + _default_message = 'TRex run failed due to wrong input parameters, or due to reachability issues.' + +class TRexWarning(TRexException): + """ Indicates a warning from TRex server. When this exception raises it normally used to indicate required data isn't ready yet """ + code = -12 + _default_message = 'TRex is starting (data is not available yet).' + +class TRexRequestDenied(TRexException): + """ Indicates the desired reques was denied by the server """ + code = -33 + _default_message = 'TRex desired request denied because the requested resource is already taken. Try again once TRex is back in IDLE state.' + +class TRexInUseError(TRexException): + """ + Indicates that TRex is currently in use + + """ + code = -13 + _default_message = 'TRex is already being used by another user or process. Try again once TRex is back in IDLE state.' + +class TRexRunFailedError(TRexException): + """ Indicates that TRex has failed due to some reason. This Exception is used when TRex process itself terminates due to unknown reason """ + code = -14 + _default_message = '' + +class TRexIncompleteRunError(TRexException): + """ + Indicates that TRex has failed due to some reason. + This Exception is used when TRex process itself terminated with error fault or it has been terminated by an external intervention in the OS. + + """ + code = -15 + _default_message = 'TRex run was terminated unexpectedly by outer process or by the hosting OS' + +EXCEPTIONS = [TRexException, TRexError, TRexWarning, TRexInUseError, TRexRequestDenied, TRexRunFailedError, TRexIncompleteRunError] + +class CExceptionHandler(object): + """ + CExceptionHandler is responsible for generating TRex API related exceptions in client side. + """ + def __init__(self, exceptions): + """ + Instatiate a CExceptionHandler object + + :parameters: + + exceptions : list + a list of all TRex acceptable exception objects. + + default list: + - :exc:`trex_exceptions.TRexException` + - :exc:`trex_exceptions.TRexError` + - :exc:`trex_exceptions.TRexWarning` + - :exc:`trex_exceptions.TRexInUseError` + - :exc:`trex_exceptions.TRexRequestDenied` + - :exc:`trex_exceptions.TRexRunFailedError` + - :exc:`trex_exceptions.TRexIncompleteRunError` + + """ + if isinstance(exceptions, type): + exceptions = [ exceptions, ] + self.exceptions = exceptions + self.exceptions_dict = dict((e.code, e) for e in self.exceptions) + + def gen_exception (self, err): + """ + Generates an exception based on a general ProtocolError exception object `err`. + + When TRex is reserved, no other user can start new TRex runs. + + + :parameters: + + err : exception + a ProtocolError exception raised by :class:`trex_client.CTRexClient` class + + :return: + A TRex exception from the exception list defined in class creation. + + If such exception wasn't found, returns a TRexException exception + + """ + code, message, data = err + try: + exp = self.exceptions_dict[code] + return exp(exp.code, message, data) + except KeyError: + # revert to TRexException when unknown error application raised + return TRexException(err) + + +exception_handler = CExceptionHandler( EXCEPTIONS ) + diff --git a/scripts/automation/trex_control_plane/stf/trex_status.py b/scripts/automation/trex_control_plane/stf/trex_status.py new file mode 100644 index 00000000..f132720c --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_status.py @@ -0,0 +1,8 @@ +#!/router/bin/python + +# define the states in which a T-Rex can hold during its lifetime +# TRexStatus = Enum('TRexStatus', 'Idle Starting Running') + +IDLE = 1 +STARTING = 2 +RUNNING = 3 diff --git a/scripts/automation/trex_control_plane/stf/trex_status_e.py b/scripts/automation/trex_control_plane/stf/trex_status_e.py new file mode 100755 index 00000000..79a25acc --- /dev/null +++ b/scripts/automation/trex_control_plane/stf/trex_status_e.py @@ -0,0 +1,11 @@ +#!/router/bin/python + +try: + from . import outer_packages +except: + import outer_packages +from enum import Enum + + +# define the states in which a TRex can hold during its lifetime +TRexStatus = Enum('TRexStatus', 'Idle Starting Running') diff --git a/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py b/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py index 56e2005a..bdec9999 100644 --- a/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py +++ b/scripts/automation/trex_control_plane/stl/examples/hlt_udp_simple.py @@ -67,15 +67,15 @@ if __name__ == "__main__": args = parser.parse_args(); hltapi = CTRexHltApi() - print 'Connecting to TRex' + print('Connecting to TRex') res = hltapi.connect(device = args.server, port_list = [0, 1], reset = True, break_locks = True) check_res(res) ports = res['port_handle'] if len(ports) < 2: error('Should have at least 2 ports for this test') - print 'Connected, acquired ports: %s' % ports + print('Connected, acquired ports: %s' % ports) - print 'Creating traffic' + print('Creating traffic') res = hltapi.traffic_config(mode = 'create', bidirectional = True, port_handle = ports[0], port_handle2 = ports[1], @@ -91,12 +91,12 @@ if __name__ == "__main__": ) check_res(res) - print 'Starting traffic' + print('Starting traffic') res = hltapi.traffic_control(action = 'run', port_handle = ports[:2]) check_res(res) wait_with_progress(args.duration) - print 'Stopping traffic' + print('Stopping traffic') res = hltapi.traffic_control(action = 'stop', port_handle = ports[:2]) check_res(res) @@ -107,4 +107,4 @@ if __name__ == "__main__": res = hltapi.cleanup_session(port_handle = 'all') check_res(res) - print 'Done' + print('Done') diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py b/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py index 317f44c7..45ddc24b 100644 --- a/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py +++ b/scripts/automation/trex_control_plane/stl/examples/stl_pcap.py @@ -1,7 +1,7 @@ import stl_path from trex_stl_lib.api import * import argparse - +import sys def create_vm (ip_start, ip_end): vm =[ @@ -17,11 +17,23 @@ def create_vm (ip_start, ip_end): return vm - -def inject_pcap (pcap_file, port, loop_count, ipg_usec, use_vm): +# warning: might make test slow +def alter_streams(streams, remove_fcs, vlan_id): + for stream in streams: + packet = Ether(stream.pkt) + if vlan_id >= 0 and vlan_id <= 4096: + packet_l3 = packet.payload + packet = Ether() / Dot1Q(vlan = vlan_id) / packet_l3 + if remove_fcs and packet.lastlayer().name == 'Padding': + packet.lastlayer().underlayer.remove_payload() + packet = STLPktBuilder(packet) + stream.fields['packet'] = packet.dump_pkt() + stream.pkt = base64.b64decode(stream.fields['packet']['binary']) + +def inject_pcap (pcap_file, server, port, loop_count, ipg_usec, use_vm, remove_fcs, vlan_id): # create client - c = STLClient() + c = STLClient(server = server) try: if use_vm: @@ -32,13 +44,16 @@ def inject_pcap (pcap_file, port, loop_count, ipg_usec, use_vm): profile = STLProfile.load_pcap(pcap_file, ipg_usec = ipg_usec, loop_count = loop_count, vm = vm) print("Loaded pcap {0} with {1} packets...\n".format(pcap_file, len(profile))) + streams = profile.get_streams() + if remove_fcs or (vlan_id >= 0 and vlan_id <= 4096): + alter_streams(streams, remove_fcs, vlan_id) # uncomment this for simulator run - #STLSim().run(profile.get_streams(), outfile = 'out.cap') + #STLSim().run(profile.get_streams(), outfile = '/auto/srg-sce-swinfra-usr/emb/users/ybrustin/out.pcap') c.connect() c.reset(ports = [port]) - stream_ids = c.add_streams(profile.get_streams(), ports = [port]) + stream_ids = c.add_streams(streams, ports = [port]) c.clear_stats() @@ -51,6 +66,7 @@ def inject_pcap (pcap_file, port, loop_count, ipg_usec, use_vm): except STLError as e: print(e) + sys.exit(1) finally: c.disconnect() @@ -59,17 +75,22 @@ def inject_pcap (pcap_file, port, loop_count, ipg_usec, use_vm): def setParserOptions(): parser = argparse.ArgumentParser(prog="stl_pcap.py") - parser.add_argument("-f", help = "pcap file to inject", + parser.add_argument("-f", "--file", help = "pcap file to inject", dest = "pcap", required = True, type = str) - parser.add_argument("-p", help = "port to inject on", + parser.add_argument("-s", "--server", help = "TRex server address", + dest = "server", + default = 'localhost', + type = str) + + parser.add_argument("-p", "--port", help = "port to inject on", dest = "port", required = True, type = int) - parser.add_argument("-n", help = "How many times to inject pcap [default is 1, 0 means forever]", + parser.add_argument("-n", "--number", help = "How many times to inject pcap [default is 1, 0 means forever]", dest = "loop_count", default = 1, type = int) @@ -79,19 +100,28 @@ def setParserOptions(): default = 10.0, type = float) - parser.add_argument("-x", help = "Iterate over IP dest", dest = "use_vm", default = False, action = "store_true") + parser.add_argument("-r", "--remove-fcs", help = "Remove FCS if exists. Limited by Scapy capabilities.", + dest = "remove", + default = False, + action = "store_true") + + parser.add_argument("-v", "--vlan", help = "Add VLAN header with this ID. Limited by Scapy capabilities.", + dest = "vlan", + default = -1, + type = int) + return parser def main (): parser = setParserOptions() options = parser.parse_args() - inject_pcap(options.pcap, options.port, options.loop_count, options.ipg, options.use_vm) + inject_pcap(options.pcap, options.server, options.port, options.loop_count, options.ipg, options.use_vm, options.remove, options.vlan) # inject pcap if __name__ == '__main__': diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py index c614c4bd..d6d66ec3 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py @@ -7,11 +7,14 @@ import platform TREX_STL_EXT_PATH = os.environ.get('TREX_STL_EXT_PATH') # take default -if not TREX_STL_EXT_PATH: +if not TREX_STL_EXT_PATH or not os.path.exists(TREX_STL_EXT_PATH): CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) + TREX_STL_EXT_PATH = os.path.normpath(os.path.join(CURRENT_PATH, os.pardir, 'external_libs')) +if not os.path.exists(TREX_STL_EXT_PATH): # ../../../../external_libs - TREX_STL_EXT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs')) - + TREX_STL_EXT_PATH = os.path.normpath(os.path.join(CURRENT_PATH, os.pardir, os.pardir, os.pardir, os.pardir, 'external_libs')) +if not os.path.exists(TREX_STL_EXT_PATH): + raise Exception('Could not determine path of external_libs, try setting TREX_STL_EXT_PATH variable') # the modules required # py-dep requires python2/python3 directories diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py index 9387c3a6..b506137b 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py @@ -174,7 +174,7 @@ import socket import copy from collections import defaultdict -from trex_stl_lib.api import * +from .api import * from .trex_stl_types import * from .utils.common import get_number @@ -249,7 +249,7 @@ def print_brief_stats(res): title_str = ' '*3 tx_str = 'TX:' rx_str = 'RX:' - for port_id, stat in res.iteritems(): + for port_id, stat in res.items(): if type(port_id) is not int: continue title_str += ' '*10 + 'Port%s' % port_id @@ -663,7 +663,7 @@ class CTRexHltApi(object): stats = self.trex_client.get_stats(port_handle) except Exception as e: return HLT_ERR('Could not retrieve stats: %s' % e if isinstance(e, STLError) else traceback.format_exc()) - for port_id, stat_dict in stats.iteritems(): + for port_id, stat_dict in stats.items(): if is_integer(port_id): hlt_stats_dict[port_id] = { 'aggregate': { diff --git a/scripts/external_libs/scapy-2.3.1/python2/scapy/data.py b/scripts/external_libs/scapy-2.3.1/python2/scapy/data.py index de01cfc9..9c5bb6bd 100644 --- a/scripts/external_libs/scapy-2.3.1/python2/scapy/data.py +++ b/scripts/external_libs/scapy-2.3.1/python2/scapy/data.py @@ -105,6 +105,7 @@ def load_services(filename): spaces = re.compile("[ \t]+|\n") tdct=DADict(_name="%s-tcp"%filename) udct=DADict(_name="%s-udp"%filename) + return tdct,udct try: f=open(filename) for l in f: diff --git a/scripts/external_libs/scapy-2.3.1/python3/scapy/data.py b/scripts/external_libs/scapy-2.3.1/python3/scapy/data.py index fc92ebe2..2aded28a 100644 --- a/scripts/external_libs/scapy-2.3.1/python3/scapy/data.py +++ b/scripts/external_libs/scapy-2.3.1/python3/scapy/data.py @@ -105,6 +105,7 @@ def load_services(filename): spaces = re.compile("[ \t]+|\n") tdct=DADict(_name="%s-tcp"%filename) udct=DADict(_name="%s-udp"%filename) + return tdct,udct try: f=open(filename) for l in f: diff --git a/scripts/find_python.sh b/scripts/find_python.sh index 929e873d..e9607fe5 100755 --- a/scripts/find_python.sh +++ b/scripts/find_python.sh @@ -24,12 +24,27 @@ function find_python { exit -1 } -if [ -z "$PYTHON" ]; then - # for development here - move us to python 3 for now - if [ "$USER" == "imarom" ] || [ "$USER" == "hhaim" ] || [ "$USER" == "ybrustin" ] || [ "$USER" == "ibarnea" ]; then - PYTHON=/auto/proj-pcube-b/apps/PL-b/tools/python3.4/bin/python3 - else - find_python +function find_python3 { + MACHINE_PYTHON=python3 + ITAY_PYTHON=/auto/proj-pcube-b/apps/PL-b/tools/python3.4/bin/python3 + PYTHON3=$MACHINE_PYTHON + PCHECK=`$PYTHON3 -c "import sys; ver = sys.version_info[0] * 10 + sys.version_info[1];sys.exit(ver != 34)"` + if [ $? -eq 0 ]; then + return + fi + PYTHON3=$ITAY_PYTHON + PCHECK=`$PYTHON3 -c "import sys; ver = sys.version_info[0] * 10 + sys.version_info[1];sys.exit(ver != 34)"` + if [ $? -eq 0 ]; then + return fi + echo "*** $PYTHON3 - python version does not match, 3.4 is required" + exit -1 +} + +if [ -z "$PYTHON" ]; then + find_python fi +if [ -z "$PYTHON3" ]; then + find_python3 +fi diff --git a/scripts/run_functional_tests b/scripts/run_functional_tests index 995b1b0d..6e6a00a1 100755 --- a/scripts/run_functional_tests +++ b/scripts/run_functional_tests @@ -1,12 +1,10 @@ #!/bin/bash -#source find_python.sh +source find_python.sh cd automation/regression -PYTHON=/usr/bin/python2 -PYTHON3=/auto/proj-pcube-b/apps/PL-b/tools/python3.4/bin/python3 - # Python 2 +echo Python2 test $PYTHON trex_unit_test.py --functional $@ if [ $? -eq 0 ]; then printf "\n$PYTHON test succeeded\n\n" @@ -16,6 +14,7 @@ else fi # Python 3 +echo Python3 test $PYTHON3 trex_unit_test.py --functional $@ if [ $? -eq 0 ]; then printf "\n$PYTHON3 test succeeded\n\n" -- cgit 1.2.3-korg