From 764ef2e77a5f344bf1f30f717f610f49d21f869d Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Thu, 24 Mar 2016 21:08:56 +0200 Subject: return client_utils --- .../trex_control_plane/client_utils/__init__.py | 1 + .../client_utils/external_packages.py | 72 +++++++ .../client_utils/trex_yaml_gen.py | 212 +++++++++++++++++++++ .../trex_control_plane/client_utils/yaml_utils.py | 163 ++++++++++++++++ 4 files changed, 448 insertions(+) create mode 100644 scripts/automation/trex_control_plane/client_utils/__init__.py create mode 100644 scripts/automation/trex_control_plane/client_utils/external_packages.py create mode 100644 scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py create mode 100644 scripts/automation/trex_control_plane/client_utils/yaml_utils.py (limited to 'scripts/automation/trex_control_plane/client_utils') diff --git a/scripts/automation/trex_control_plane/client_utils/__init__.py b/scripts/automation/trex_control_plane/client_utils/__init__.py new file mode 100644 index 00000000..c38c2cca --- /dev/null +++ b/scripts/automation/trex_control_plane/client_utils/__init__.py @@ -0,0 +1 @@ +__all__ = ["general_utils", "trex_yaml_gen"] diff --git a/scripts/automation/trex_control_plane/client_utils/external_packages.py b/scripts/automation/trex_control_plane/client_utils/external_packages.py new file mode 100644 index 00000000..c682dc18 --- /dev/null +++ b/scripts/automation/trex_control_plane/client_utils/external_packages.py @@ -0,0 +1,72 @@ +#!/router/bin/python + +import sys +import os +import warnings + +CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) +ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory +PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) + +CLIENT_UTILS_MODULES = ['dpkt-1.8.6', + 'yaml-3.11', + 'texttable-0.8.4', + 'scapy-2.3.1' + ] + +def import_client_utils_modules(): + + # must be in a higher priority + sys.path.insert(0, PATH_TO_PYTHON_LIB) + + sys.path.append(ROOT_PATH) + import_module_list(CLIENT_UTILS_MODULES) + + +def import_module_list(modules_list): + assert(isinstance(modules_list, list)) + for p in modules_list: + full_path = os.path.join(PATH_TO_PYTHON_LIB, p) + fix_path = os.path.normcase(full_path) + sys.path.insert(1, full_path) + + + import_platform_dirs() + + + +def import_platform_dirs (): + # handle platform dirs + + # try fedora 18 first and then cel5.9 + # we are using the ZMQ module to determine the right platform + + full_path = os.path.join(PATH_TO_PYTHON_LIB, 'platform/fedora18') + fix_path = os.path.normcase(full_path) + sys.path.insert(0, full_path) + try: + # try to import and delete it from the namespace + import zmq + del zmq + return + except: + sys.path.pop(0) + pass + + full_path = os.path.join(PATH_TO_PYTHON_LIB, 'platform/cel59') + fix_path = os.path.normcase(full_path) + sys.path.insert(0, full_path) + try: + # try to import and delete it from the namespace + import zmq + del zmq + return + + except: + sys.path.pop(0) + sys.modules['zmq'] = None + warnings.warn("unable to determine platform type for ZMQ import") + + + +import_client_utils_modules() diff --git a/scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py b/scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py new file mode 100644 index 00000000..c26fef29 --- /dev/null +++ b/scripts/automation/trex_control_plane/client_utils/trex_yaml_gen.py @@ -0,0 +1,212 @@ +#!/router/bin/python + +import pprint +import yaml +import os +# import bisect + +class CTRexYaml(object): + """ + This class functions as a YAML generator according to TRex YAML format. + + CTRexYaml is compatible with both Python 2 and Python 3. + """ + YAML_TEMPLATE = [{'cap_info': [], + 'duration': 10.0, + 'generator': {'clients_end': '16.0.1.255', + 'clients_per_gb': 201, + 'clients_start': '16.0.0.1', + 'distribution': 'seq', + 'dual_port_mask': '1.0.0.0', + 'min_clients': 101, + 'servers_end': '48.0.0.255', + 'servers_start': '48.0.0.1', + 'tcp_aging': 1, + 'udp_aging': 1}, + 'mac' : [0x00,0x00,0x00,0x01,0x00,0x00]}] + PCAP_TEMPLATE = {'cps': 1.0, + 'ipg': 10000, + 'name': '', + 'rtt': 10000, + 'w': 1} + + def __init__ (self, trex_files_path): + """ + The initialization of this class creates a CTRexYaml object with **empty** 'cap-info', and with default client-server configuration. + + Use class methods to add and assign pcap files and export the data to a YAML file. + + :parameters: + trex_files_path : str + a path (on TRex server side) for the pcap files using which TRex can access it. + + """ + self.yaml_obj = list(CTRexYaml.YAML_TEMPLATE) + self.empty_cap = True + self.file_list = [] + self.yaml_dumped = False + self.trex_files_path = trex_files_path + + def add_pcap_file (self, local_pcap_path): + """ + Adds a .pcap file with recorded traffic to the yaml object by linking the file with 'cap-info' template key fields. + + :parameters: + local_pcap_path : str + a path (on client side) for the pcap file to be added. + + :return: + + The index of the inserted item (as int) if item added successfully + + -1 if pcap file already exists in 'cap_info'. + + """ + new_pcap = dict(CTRexYaml.PCAP_TEMPLATE) + new_pcap['name'] = self.trex_files_path + os.path.basename(local_pcap_path) + if self.get_pcap_idx(new_pcap['name']) != -1: + # pcap already exists in 'cap_info' + return -1 + else: + self.yaml_obj[0]['cap_info'].append(new_pcap) + if self.empty_cap: + self.empty_cap = False + self.file_list.append(local_pcap_path) + return ( len(self.yaml_obj[0]['cap_info']) - 1) + + + def get_pcap_idx (self, pcap_name): + """ + Checks if a certain .pcap file has been added into the yaml object. + + :parameters: + pcap_name : str + the name of the pcap file to be searched + + :return: + + The index of the pcap file (as int) if exists + + -1 if not exists. + + """ + comp_pcap = pcap_name if pcap_name.startswith(self.trex_files_path) else (self.trex_files_path + pcap_name) + for idx, pcap in enumerate(self.yaml_obj[0]['cap_info']): + print (pcap['name'] == comp_pcap) + if pcap['name'] == comp_pcap: + return idx + # pcap file wasn't found + return -1 + + def dump_as_python_obj (self): + """ + dumps with nice indentation the pythonic format (dictionaries and lists) of the currently built yaml object. + + :parameters: + None + + :return: + None + + """ + pprint.pprint(self.yaml_obj) + + def dump(self): + """ + dumps with nice indentation the YAML format of the currently built yaml object. + + :parameters: + None + + :return: + None + + """ + print (yaml.safe_dump(self.yaml_obj, default_flow_style = False)) + + def to_yaml(self, filename): + """ + Exports to YAML file the built configuration into an actual YAML file. + + :parameters: + filename : str + a path (on client side, including filename) to store the generated yaml file. + + :return: + None + + :raises: + + :exc:`ValueError`, in case no pcap files has been added to the object. + + :exc:`EnvironmentError`, in case of any IO error of writing to the files or OSError when trying to open it for writing. + + """ + if self.empty_cap: + raise ValueError("No .pcap file has been assigned to yaml object. Must add at least one") + else: + try: + with open(filename, 'w') as yaml_file: + yaml_file.write( yaml.safe_dump(self.yaml_obj, default_flow_style = False) ) + self.yaml_dumped = True + self.file_list.append(filename) + except EnvironmentError as inst: + raise + + def set_cap_info_param (self, param, value, seq): + """ + Set cap-info parameters' value of a specific pcap file. + + :parameters: + param : str + the name of the parameters to be set. + value : int/float + the desired value to be set to `param` key. + seq : int + an index to the relevant caps array to be changed (index supplied when adding new pcap file, see :func:`add_pcap_file`). + + :return: + **True** on success + + :raises: + :exc:`IndexError`, in case an out-of range index was given. + + """ + try: + self.yaml_obj[0]['cap_info'][seq][param] = value + + return True + except IndexError: + return False + + def set_generator_param (self, param, value): + """ + Set generator parameters' value of the yaml object. + + :parameters: + param : str + the name of the parameters to be set. + value : int/float/str + the desired value to be set to `param` key. + + :return: + None + + """ + self.yaml_obj[0]['generator'][param] = value + + def get_file_list(self): + """ + Returns a list of all files related to the YAML object, including the YAML filename itself. + + .. tip:: This method is especially useful for listing all the files that should be pushed to TRex server as part of the same yaml selection. + + :parameters: + None + + :return: + a list of filepaths, each is a local client-machine file path. + + """ + if not self.yaml_dumped: + print ("WARNING: .yaml file wasn't dumped yet. Files list contains only .pcap files") + return self.file_list + + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/client_utils/yaml_utils.py b/scripts/automation/trex_control_plane/client_utils/yaml_utils.py new file mode 100644 index 00000000..776a51a7 --- /dev/null +++ b/scripts/automation/trex_control_plane/client_utils/yaml_utils.py @@ -0,0 +1,163 @@ + +""" +Dan Klein +Cisco Systems, Inc. + +Copyright (c) 2015-2015 Cisco Systems, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import traceback +import sys +import yaml + + +class CTRexYAMLLoader(object): + TYPE_DICT = {"double":float, + "int":int, + "array":list, + "string":str, + "boolean":bool} + + def __init__(self, yaml_ref_file_path): + self.yaml_path = yaml_ref_file_path + self.ref_obj = None + + def check_term_param_type(self, val, val_field, ref_val, multiplier): + # print val, val_field, ref_val + tmp_type = ref_val.get('type') + if isinstance(tmp_type, list): + # item can be one of multiple types + # print "multiple choice!" + python_types = set() + for t in tmp_type: + if t in self.TYPE_DICT: + python_types.add(self.TYPE_DICT.get(t)) + else: + return False, TypeError("Unknown resolving for type {0}".format(t)) + # print "python legit types: ", python_types + if type(val) not in python_types: + return False, TypeError("Type of object field '{0}' is not allowed".format(val_field)) + else: + # WE'RE OK! + return True, CTRexYAMLLoader._calc_final_value(val, multiplier, ref_val.get('multiply', False)) + else: + # this is a single type field + python_type = self.TYPE_DICT.get(tmp_type) + if not isinstance(val, python_type): + return False, TypeError("Type of object field '{0}' is not allowed".format(val_field)) + else: + # WE'RE OK! + return True, CTRexYAMLLoader._calc_final_value(val, multiplier, ref_val.get('multiply', False)) + + def get_reference_default(self, root_obj, sub_obj, key): + # print root_obj, sub_obj, key + if sub_obj: + ref_field = self.ref_obj.get(root_obj).get(sub_obj).get(key) + else: + ref_field = self.ref_obj.get(root_obj).get(key) + if 'has_default' in ref_field: + if ref_field.get('has_default'): + # WE'RE OK! + return True, ref_field.get('default') + else: + # This is a mandatory field! + return False, ValueError("The {0} field is mandatory and must be specified explicitly".format(key)) + else: + return False, ValueError("The {0} field has no indication about default value".format(key)) + + def validate_yaml(self, evaluated_obj, root_obj, fill_defaults=True, multiplier=1): + if isinstance(evaluated_obj, dict) and evaluated_obj.keys() == [root_obj]: + evaluated_obj = evaluated_obj.get(root_obj) + if not self.ref_obj: + self.ref_obj = load_yaml_to_obj(self.yaml_path) + # self.load_reference() + ref_item = self.ref_obj.get(root_obj) + if ref_item is not None: + try: + typed_obj = [False, None] # first item stores validity (multiple object "shapes"), second stored type + if "type" in evaluated_obj: + ref_item = ref_item[evaluated_obj.get("type")] + # print "lower resolution with typed object" + typed_obj = [True, evaluated_obj.get("type")] + if isinstance(ref_item, dict) and "type" not in ref_item: # this is not a terminal + result_obj = {} + if typed_obj[0]: + result_obj["type"] = typed_obj[1] + # print "processing dictionary non-terminal value" + for k, v in ref_item.items(): + # print "processing element '{0}' with value '{1}'".format(k,v) + if k in evaluated_obj: + # validate with ref obj + # print "found in evaluated object!" + tmp_type = v.get('type') + # print tmp_type + # print evaluated_obj + if tmp_type == "object": + # go deeper into nesting hierarchy + # print "This is an object type, recursion!" + result_obj[k] = self.validate_yaml(evaluated_obj.get(k), k, fill_defaults, multiplier) + else: + # validation on terminal type + # print "Validating terminal type %s" % k + res_ok, data = self.check_term_param_type(evaluated_obj.get(k), k, v, multiplier) + if res_ok: + # data field contains the value to save + result_obj[k] = data + else: + # data var contains the exception to throw + raise data + elif fill_defaults: + # complete missing values with default value, if exists + sub_obj = typed_obj[1] if typed_obj[0] else None + res_ok, data = self.get_reference_default(root_obj, sub_obj, k) + if res_ok: + # data field contains the value to save + result_obj[k] = data + else: + # data var contains the exception to throw + raise data + return result_obj + elif isinstance(ref_item, list): + # currently not handling list objects + return NotImplementedError("List object are currently unsupported") + else: + raise TypeError("Unknown parse tree object type.") + except KeyError as e: + raise + else: + raise KeyError("The given root_key '{key}' does not exists on reference object".format(key=root_obj)) + + @staticmethod + def _calc_final_value(val, multiplier, multiply): + def to_num(s): + try: + return int(s) + except ValueError: + return float(s) + if multiply: + return val * to_num(multiplier) + else: + return val + + +def load_yaml_to_obj(file_path): + try: + return yaml.load(file(file_path, 'r')) + except yaml.YAMLError as e: + raise + except Exception as e: + raise + +def yaml_exporter(file_path): + pass + +if __name__ == "__main__": + pass -- cgit 1.2.3-korg