From 995267db77f5554d5228697b8b2a862b51859fe6 Mon Sep 17 00:00:00 2001 From: imarom Date: Mon, 8 Feb 2016 06:08:14 -0500 Subject: first refactor --- .../trex_control_plane/common/rpc_defaults.yaml | 124 ----- .../trex_control_plane/common/trex_stats.py | 578 --------------------- .../common/trex_stl_exceptions.py | 53 -- .../trex_control_plane/common/trex_streams.py | 526 ------------------- .../trex_control_plane/common/trex_types.py | 95 ---- 5 files changed, 1376 deletions(-) delete mode 100755 scripts/automation/trex_control_plane/common/rpc_defaults.yaml delete mode 100755 scripts/automation/trex_control_plane/common/trex_stats.py delete mode 100644 scripts/automation/trex_control_plane/common/trex_stl_exceptions.py delete mode 100755 scripts/automation/trex_control_plane/common/trex_streams.py delete mode 100644 scripts/automation/trex_control_plane/common/trex_types.py (limited to 'scripts/automation/trex_control_plane/common') diff --git a/scripts/automation/trex_control_plane/common/rpc_defaults.yaml b/scripts/automation/trex_control_plane/common/rpc_defaults.yaml deleted file mode 100755 index 9325a0e4..00000000 --- a/scripts/automation/trex_control_plane/common/rpc_defaults.yaml +++ /dev/null @@ -1,124 +0,0 @@ -############################################################## -#### TRex RPC stream list default values #### -############################################################## - -# this document is based on TRex RPC server spec and its fields: -# http://trex-tgn.cisco.com/trex/doc/trex_rpc_server_spec.html - -### HOW TO READ THIS FILE -# 1. Each key represents an object type -# 2. Each value can be either a value field or another object -# 2.1. If a value field, read as: -# + type: type of field -# + has_default: if the value has any default -# + default: the default value (Only appears if has_default field is 'YES') -# 2.2. If an object type, jump to corresponding object key. -# 3. If an object has more than one instance type, another layer with the type shall be added. -# For example, 'mode' object has 3 types: 'continuous', 'single_burst', 'multi_burst' -# So, 3 mode objects will be defined, named: -# - mode['continuous'] -# - mode['single_burst'] -# - mode['multi_burst'] -# In this case, there's no default for the 'type' field on the object -# 4. Some values has 'multiply' property attached. -# In such case, the loaded value will be multiplied by the multiplier -# For example, if the mode's 'pps' field value is 10, and its multiplier is 5, -# the loaded pps value will be 10*5=50 -# 5. Any object type must be listed by the user, even if all its field are defaults. -# The most basic option would be to declare the object with "[]", which stands for empty object in YAML syntax. - - -stream: - enabled: - type: boolean - has_default: YES - default: True - self_start: - type: boolean - has_default: YES - default: True - isg: - type: [int, double, string] - has_default: YES - default: 0.0 - next_stream_id: - type: string # string to allow naming binding - has_default: YES - default: -1 # no next streams - packet: - type: object - mode: - type: object - vm: - type: object - rx_stats: - type: object - -packet: - binary: - type: [array,string] - has_default: NO - meta: - type: string - has_default: YES - default: "" - -mode: - continuous: - pps: - type: [int, double] - has_default: NO - multiply: YES - single_burst: - pps: - type: [int, double] - has_default: NO - multiply: YES - total_pkts: - type: int - has_default: NO - multi_burst: - pps: - type: [int, double] - has_default: NO - multiply: YES - pkts_per_burst: - type: int - has_default: NO - ibg: - type: [int, double, string] - has_default: YES - default: 100.0 - count: - type: int - has_default: YES - default: 0 # loop forever - -rx_stats: - enabled: - type: boolean - has_default: YES - default: False - stream_id: - type: string - has_default: YES - default: False # use related stream_id - seq_enabled: - type: boolean - has_default: YES - default: False - latency_enabled: - type: boolean - has_default: YES - default: False - -vm: - instructions: - type: array - has_default: YES - default: [] - split_by_var: - type: string - has_default: YES - default: "" - diff --git a/scripts/automation/trex_control_plane/common/trex_stats.py b/scripts/automation/trex_control_plane/common/trex_stats.py deleted file mode 100755 index 3bd6e0cd..00000000 --- a/scripts/automation/trex_control_plane/common/trex_stats.py +++ /dev/null @@ -1,578 +0,0 @@ -#!/router/bin/python -from collections import namedtuple, OrderedDict, deque -from client_utils import text_tables -from common.text_opts import format_text, format_threshold, format_num -from client.trex_async_client import CTRexAsyncStats -import copy -import datetime -import time -import re -import math -import copy - -GLOBAL_STATS = 'g' -PORT_STATS = 'p' -PORT_STATUS = 'ps' -ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS} -COMPACT = {GLOBAL_STATS, PORT_STATS} - -ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) - -# use to calculate diffs relative to the previous values -# for example, BW -def calculate_diff (samples): - total = 0.0 - - weight_step = 1.0 / sum(xrange(0, len(samples))) - weight = weight_step - - for i in xrange(0, len(samples) - 1): - current = samples[i] if samples[i] > 0 else 1 - next = samples[i + 1] if samples[i + 1] > 0 else 1 - - s = 100 * ((float(next) / current) - 1.0) - - # block change by 100% - total += (min(s, 100) * weight) - weight += weight_step - - return total - - -# calculate by absolute values and not relatives (useful for CPU usage in % and etc.) -def calculate_diff_raw (samples): - total = 0.0 - - weight_step = 1.0 / sum(xrange(0, len(samples))) - weight = weight_step - - for i in xrange(0, len(samples) - 1): - current = samples[i] - next = samples[i + 1] - - total += ( (next - current) * weight ) - weight += weight_step - - return total - - -class CTRexInfoGenerator(object): - """ - This object is responsible of generating stats and information from objects maintained at - STLClient and the ports. - """ - - def __init__(self, global_stats_ref, ports_dict_ref): - self._global_stats = global_stats_ref - self._ports_dict = ports_dict_ref - - def generate_single_statistic(self, port_id_list, statistic_type): - if statistic_type == GLOBAL_STATS: - return self._generate_global_stats() - elif statistic_type == PORT_STATS: - return self._generate_port_stats(port_id_list) - pass - elif statistic_type == PORT_STATUS: - return self._generate_port_status(port_id_list) - else: - # ignore by returning empty object - return {} - - def generate_streams_info(self, port_id_list, stream_id_list): - relevant_ports = self.__get_relevant_ports(port_id_list) - - return_data = OrderedDict() - - for port_obj in relevant_ports: - streams_data = self._generate_single_port_streams_info(port_obj, stream_id_list) - if not streams_data: - continue - hdr_key = "Port {port}: {yaml_file}".format(port= port_obj.port_id, - yaml_file= streams_data.raw_data.get('referring_file', '')) - - # TODO: test for other ports with same stream structure, and join them - return_data[hdr_key] = streams_data - - return return_data - - def _generate_global_stats(self): - stats_data = self._global_stats.generate_stats() - - # build table representation - stats_table = text_tables.TRexTextInfo() - stats_table.set_cols_align(["l", "l"]) - - stats_table.add_rows([[k.replace("_", " ").title(), v] - for k, v in stats_data.iteritems()], - header=False) - - return {"global_statistics": ExportableStats(stats_data, stats_table)} - - def _generate_port_stats(self, port_id_list): - relevant_ports = self.__get_relevant_ports(port_id_list) - - return_stats_data = {} - per_field_stats = OrderedDict([("owner", []), - ("state", []), - ("--", []), - ("Tx bps", []), - ("Tx pps", []), - - ("---", []), - ("Rx bps", []), - ("Rx pps", []), - - ("----", []), - ("opackets", []), - ("ipackets", []), - ("obytes", []), - ("ibytes", []), - ("tx-bytes", []), - ("rx-bytes", []), - ("tx-pkts", []), - ("rx-pkts", []), - - ("-----", []), - ("oerrors", []), - ("ierrors", []), - - ] - ) - - total_stats = CPortStats(None) - - for port_obj in relevant_ports: - # fetch port data - port_stats = port_obj.generate_port_stats() - - total_stats += port_obj.port_stats - - # populate to data structures - return_stats_data[port_obj.port_id] = port_stats - self.__update_per_field_dict(port_stats, per_field_stats) - - total_cols = len(relevant_ports) - header = ["port"] + [port.port_id for port in relevant_ports] - - if (total_cols > 1): - self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats) - header += ['total'] - total_cols += 1 - - stats_table = text_tables.TRexTextTable() - stats_table.set_cols_align(["l"] + ["r"] * total_cols) - stats_table.set_cols_width([10] + [17] * total_cols) - stats_table.set_cols_dtype(['t'] + ['t'] * total_cols) - - stats_table.add_rows([[k] + v - for k, v in per_field_stats.iteritems()], - header=False) - - stats_table.header(header) - - return {"port_statistics": ExportableStats(return_stats_data, stats_table)} - - def _generate_port_status(self, port_id_list): - relevant_ports = self.__get_relevant_ports(port_id_list) - - return_stats_data = {} - per_field_status = OrderedDict([("type", []), - ("maximum", []), - ("status", []) - ] - ) - - for port_obj in relevant_ports: - # fetch port data - # port_stats = self._async_stats.get_port_stats(port_obj.port_id) - port_status = port_obj.generate_port_status() - - # populate to data structures - return_stats_data[port_obj.port_id] = port_status - - self.__update_per_field_dict(port_status, per_field_status) - - stats_table = text_tables.TRexTextTable() - stats_table.set_cols_align(["l"] + ["c"]*len(relevant_ports)) - stats_table.set_cols_width([10] + [20] * len(relevant_ports)) - - stats_table.add_rows([[k] + v - for k, v in per_field_status.iteritems()], - header=False) - stats_table.header(["port"] + [port.port_id - for port in relevant_ports]) - - return {"port_status": ExportableStats(return_stats_data, stats_table)} - - def _generate_single_port_streams_info(self, port_obj, stream_id_list): - - return_streams_data = port_obj.generate_loaded_streams_sum(stream_id_list) - - if not return_streams_data.get("streams"): - # we got no streams available - return None - - # FORMAT VALUES ON DEMAND - - # because we mutate this - deep copy before - return_streams_data = copy.deepcopy(return_streams_data) - - for stream_id, stream_id_sum in return_streams_data['streams'].iteritems(): - stream_id_sum['rate_pps'] = format_num(stream_id_sum['rate_pps'], suffix='pps') - stream_id_sum['packet_type'] = self._trim_packet_headers(stream_id_sum['packet_type'], 20) - - info_table = text_tables.TRexTextTable() - info_table.set_cols_align(["c"] + ["l"] + ["r"] + ["c"] + ["r"] + ["c"]) - info_table.set_cols_width([10] + [20] + [8] + [16] + [10] + [12]) - info_table.set_cols_dtype(["t"] + ["t"] + ["t"] + ["t"] + ["t"] + ["t"]) - - info_table.add_rows([v.values() - for k, v in return_streams_data['streams'].iteritems()], - header=False) - info_table.header(["ID", "packet type", "length", "mode", "rate", "next stream"]) - - return ExportableStats(return_streams_data, info_table) - - - def __get_relevant_ports(self, port_id_list): - # fetch owned ports - ports = [port_obj - for _, port_obj in self._ports_dict.iteritems() - if port_obj.port_id in port_id_list] - - # display only the first FOUR options, by design - if len(ports) > 4: - print format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta') - ports = ports[:4] - return ports - - def __update_per_field_dict(self, dict_src_data, dict_dest_ref): - for key, val in dict_src_data.iteritems(): - if key in dict_dest_ref: - dict_dest_ref[key].append(val) - - @staticmethod - def _trim_packet_headers(headers_str, trim_limit): - if len(headers_str) < trim_limit: - # do nothing - return headers_str - else: - return (headers_str[:trim_limit-3] + "...") - - - -class CTRexStats(object): - """ This is an abstract class to represent a stats object """ - - def __init__(self): - self.reference_stats = {} - self.latest_stats = {} - self.last_update_ts = time.time() - self.history = deque(maxlen = 10) - - def __getitem__(self, item): - # override this to allow quick and clean access to fields - if not item in self.latest_stats: - return "N/A" - - # item must exist - m = re.search('_(([a-z])ps)$', item) - if m: - # this is a non-relative item - unit = m.group(2) - if unit == "b": - return self.get(item, format=True, suffix="b/sec") - elif unit == "p": - return self.get(item, format=True, suffix="pkt/sec") - else: - return self.get(item, format=True, suffix=m.group(1)) - - m = re.search('^[i|o](a-z+)$', item) - if m: - # this is a non-relative item - type = m.group(1) - if type == "bytes": - return self.get_rel(item, format=True, suffix="B") - elif type == "packets": - return self.get_rel(item, format=True, suffix="pkts") - else: - # do not format with suffix - return self.get_rel(item, format=True) - - # can't match to any known pattern, return N/A - return "N/A" - - - def generate_stats(self): - # must be implemented by designated classes (such as port/ global stats) - raise NotImplementedError() - - def update(self, snapshot): - # update - self.latest_stats = snapshot - self.history.append(snapshot) - - diff_time = time.time() - self.last_update_ts - - # 3 seconds is too much - this is the new reference - if (not self.reference_stats) or (diff_time > 3): - self.reference_stats = self.latest_stats - - self.last_update_ts = time.time() - - - def clear_stats(self): - self.reference_stats = self.latest_stats - - - def invalidate (self): - self.latest_stats = {} - - def get(self, field, format=False, suffix=""): - if not field in self.latest_stats: - return "N/A" - if not format: - return self.latest_stats[field] - else: - return format_num(self.latest_stats[field], suffix) - - def get_rel(self, field, format=False, suffix=""): - if not field in self.latest_stats: - return "N/A" - - if not format: - if not field in self.reference_stats: - print "REF: " + str(self.reference_stats) - print "BASE: " + str(self.latest_stats) - - return (self.latest_stats[field] - self.reference_stats[field]) - else: - return format_num(self.latest_stats[field] - self.reference_stats[field], suffix) - - # get trend for a field - def get_trend (self, field, use_raw = False, percision = 10.0): - if not field in self.latest_stats: - return 0 - - # not enough history - no trend - if len(self.history) < 5: - return 0 - - # absolute value is too low 0 considered noise - if self.latest_stats[field] < percision: - return 0 - - field_samples = [sample[field] for sample in self.history] - - if use_raw: - return calculate_diff_raw(field_samples) - else: - return calculate_diff(field_samples) - - - def get_trend_gui (self, field, show_value = False, use_raw = False, up_color = 'red', down_color = 'green'): - v = self.get_trend(field, use_raw) - - value = abs(v) - arrow = u'\u25b2' if v > 0 else u'\u25bc' - color = up_color if v > 0 else down_color - - # change in 1% is not meaningful - if value < 1: - return "" - - elif value > 5: - - if show_value: - return format_text(u"{0}{0}{0} {1:.2f}%".format(arrow,v), color) - else: - return format_text(u"{0}{0}{0}".format(arrow), color) - - elif value > 2: - - if show_value: - return format_text(u"{0}{0} {1:.2f}%".format(arrow,v), color) - else: - return format_text(u"{0}{0}".format(arrow), color) - - else: - if show_value: - return format_text(u"{0} {1:.2f}%".format(arrow,v), color) - else: - return format_text(u"{0}".format(arrow), color) - - - -class CGlobalStats(CTRexStats): - - def __init__(self, connection_info, server_version, ports_dict_ref): - super(CGlobalStats, self).__init__() - self.connection_info = connection_info - self.server_version = server_version - self._ports_dict = ports_dict_ref - - def get_stats (self): - stats = {} - - # absolute - stats['cpu_util'] = self.get("m_cpu_util") - stats['tx_bps'] = self.get("m_tx_bps") - stats['tx_pps'] = self.get("m_tx_pps") - - stats['rx_bps'] = self.get("m_rx_bps") - stats['rx_pps'] = self.get("m_rx_pps") - stats['rx_drop_bps'] = self.get("m_rx_drop_bps") - - # relatives - stats['queue_full'] = self.get_rel("m_total_queue_full") - - return stats - - - def generate_stats(self): - return OrderedDict([("connection", "{host}, Port {port}".format(host=self.connection_info.get("server"), - port=self.connection_info.get("sync_port"))), - ("version", "{ver}, UUID: {uuid}".format(ver=self.server_version.get("version", "N/A"), - uuid="N/A")), - - ("cpu_util", u"{0}% {1}".format( format_threshold(self.get("m_cpu_util"), [85, 100], [0, 85]), - self.get_trend_gui("m_cpu_util", use_raw = True))), - - (" ", ""), - - ("total_tx", u"{0} {1}".format( self.get("m_tx_bps", format=True, suffix="b/sec"), - self.get_trend_gui("m_tx_bps"))), - - ("total_rx", u"{0} {1}".format( self.get("m_rx_bps", format=True, suffix="b/sec"), - self.get_trend_gui("m_rx_bps"))), - - ("total_pps", u"{0} {1}".format( self.get("m_tx_pps", format=True, suffix="pkt/sec"), - self.get_trend_gui("m_tx_pps"))), - - (" ", ""), - - ("drop_rate", "{0}".format( format_num(self.get("m_rx_drop_bps"), - suffix = 'b/sec', - opts = 'green' if (self.get("m_rx_drop_bps")== 0) else 'red'))), - - ("queue_full", "{0}".format( format_num(self.get_rel("m_total_queue_full"), - suffix = 'pkts', - compact = False, - opts = 'green' if (self.get_rel("m_total_queue_full")== 0) else 'red'))), - - ] - ) - -class CPortStats(CTRexStats): - - def __init__(self, port_obj): - super(CPortStats, self).__init__() - self._port_obj = port_obj - - @staticmethod - def __merge_dicts (target, src): - for k, v in src.iteritems(): - if k in target: - target[k] += v - else: - target[k] = v - - - def __add__ (self, x): - if not isinstance(x, CPortStats): - raise TypeError("cannot add non stats object to stats") - - # main stats - if not self.latest_stats: - self.latest_stats = {} - - self.__merge_dicts(self.latest_stats, x.latest_stats) - - # reference stats - if x.reference_stats: - if not self.reference_stats: - self.reference_stats = x.reference_stats.copy() - else: - self.__merge_dicts(self.reference_stats, x.reference_stats) - - # history - if not self.history: - self.history = copy.deepcopy(x.history) - else: - for h1, h2 in zip(self.history, x.history): - self.__merge_dicts(h1, h2) - - return self - - # for port we need to do something smarter - def get_stats (self): - stats = {} - - stats['opackets'] = self.get_rel("opackets") - stats['ipackets'] = self.get_rel("ipackets") - stats['obytes'] = self.get_rel("obytes") - stats['ibytes'] = self.get_rel("ibytes") - stats['oerrors'] = self.get_rel("oerrors") - stats['ierrors'] = self.get_rel("ierrors") - stats['tx_bps'] = self.get("m_total_tx_bps") - stats['tx_pps'] = self.get("m_total_tx_pps") - stats['rx_bps'] = self.get("m_total_rx_bps") - stats['rx_pps'] = self.get("m_total_rx_pps") - - return stats - - - def generate_stats(self): - - state = self._port_obj.get_port_state_name() if self._port_obj else "" - if state == "ACTIVE": - state = format_text(state, 'green', 'bold') - elif state == "PAUSE": - state = format_text(state, 'magenta', 'bold') - else: - state = format_text(state, 'bold') - - return {"owner": self._port_obj.user if self._port_obj else "", - "state": "{0}".format(state), - - "--": " ", - "---": " ", - "----": " ", - "-----": " ", - - "Tx bps": u"{0} {1}".format(self.get_trend_gui("m_total_tx_bps", show_value = False), - self.get("m_total_tx_bps", format = True, suffix = "bps")), - - "Rx bps": u"{0} {1}".format(self.get_trend_gui("m_total_rx_bps", show_value = False), - self.get("m_total_rx_bps", format = True, suffix = "bps")), - - "Tx pps": u"{0} {1}".format(self.get_trend_gui("m_total_tx_pps", show_value = False), - self.get("m_total_tx_pps", format = True, suffix = "pps")), - - "Rx pps": u"{0} {1}".format(self.get_trend_gui("m_total_rx_pps", show_value = False), - self.get("m_total_rx_pps", format = True, suffix = "pps")), - - "opackets" : self.get_rel("opackets"), - "ipackets" : self.get_rel("ipackets"), - "obytes" : self.get_rel("obytes"), - "ibytes" : self.get_rel("ibytes"), - - "tx-bytes": self.get_rel("obytes", format = True, suffix = "B"), - "rx-bytes": self.get_rel("ibytes", format = True, suffix = "B"), - "tx-pkts": self.get_rel("opackets", format = True, suffix = "pkts"), - "rx-pkts": self.get_rel("ipackets", format = True, suffix = "pkts"), - - "oerrors" : format_num(self.get_rel("oerrors"), - compact = False, - opts = 'green' if (self.get_rel("oerrors")== 0) else 'red'), - - "ierrors" : format_num(self.get_rel("ierrors"), - compact = False, - opts = 'green' if (self.get_rel("ierrors")== 0) else 'red'), - - } - - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/common/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/common/trex_stl_exceptions.py deleted file mode 100644 index 9be20db9..00000000 --- a/scripts/automation/trex_control_plane/common/trex_stl_exceptions.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import sys -from common.text_opts import * - -# basic error for API -class STLError(Exception): - def __init__ (self, msg): - self.msg = str(msg) - - def __str__ (self): - exc_type, exc_obj, exc_tb = sys.exc_info() - fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - - - s = "\n******\n" - s += "Error at {0}:{1}\n\n".format(format_text(fname, 'bold'), format_text(exc_tb.tb_lineno), 'bold') - s += "specific error:\n\n{0}\n".format(format_text(self.msg, 'bold')) - - return s - - def brief (self): - return self.msg - - -# raised when the client state is invalid for operation -class STLStateError(STLError): - def __init__ (self, op, state): - self.msg = "Operation '{0}' is not valid while '{1}'".format(op, state) - - -# port state error -class STLPortStateError(STLError): - def __init__ (self, port, op, state): - self.msg = "Operation '{0}' on port(s) '{1}' is not valid while port(s) '{2}'".format(op, port, state) - - -# raised when argument is not valid for operation -class STLArgumentError(STLError): - def __init__ (self, name, got, valid_values = None, extended = None): - self.msg = "Argument: '{0}' invalid value: '{1}'".format(name, got) - if valid_values: - self.msg += " - valid values are '{0}'".format(valid_values) - - if extended: - self.msg += "\n{0}".format(extended) - -# raised when timeout occurs -class STLTimeoutError(STLError): - def __init__ (self, timeout): - self.msg = "Timeout: operation took more than '{0}' seconds".format(timeout) - - - diff --git a/scripts/automation/trex_control_plane/common/trex_streams.py b/scripts/automation/trex_control_plane/common/trex_streams.py deleted file mode 100755 index c1f1bfa6..00000000 --- a/scripts/automation/trex_control_plane/common/trex_streams.py +++ /dev/null @@ -1,526 +0,0 @@ -#!/router/bin/python - -import external_packages -from client_utils.packet_builder_interface import CTrexPktBuilderInterface -from client_utils.packet_builder import CTRexPktBuilder -from collections import OrderedDict, namedtuple -from client_utils.yaml_utils import * -import trex_stl_exceptions -import dpkt -import struct -import copy -import os -import random -import yaml -import base64 - -StreamPack = namedtuple('StreamPack', ['stream_id', 'stream']) -LoadedStreamList = namedtuple('LoadedStreamList', ['name', 'loaded', 'compiled']) - -class CStreamList(object): - - def __init__(self): - self.streams_list = OrderedDict() - self.yaml_loader = CTRexYAMLLoader(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "rpc_defaults.yaml")) - - def generate_numbered_name (self, name): - prefix = name.rstrip('01234567890') - suffix = name[len(prefix):] - if suffix == "": - n = "_1" - else: - n = int(suffix) + 1 - return prefix + str(n) - - def append_stream(self, name, stream_obj): - assert isinstance(stream_obj, CStream) - - # if name exists simply add numbered suffix to it - while name in self.streams_list: - name = self.generate_numbered_name(name) - - self.streams_list[name]=stream_obj - return name - - def remove_stream(self, name): - popped = self.streams_list.pop(name) - if popped: - for stream_name, stream in self.streams_list.items(): - if stream.next_stream_id == name: - stream.next_stream_id = -1 - try: - rx_stats_stream = getattr(stream.rx_stats, "stream_id") - if rx_stats_stream == name: - # if a referenced stream of rx_stats object deleted, revert to rx stats of current stream - setattr(stream.rx_stats, "stream_id", stream_name) - except AttributeError as e: - continue # - return popped - - def export_to_yaml(self, file_path): - raise NotImplementedError("export_to_yaml method is not implemented, yet") - - def load_yaml(self, file_path, multiplier=1): - # clear all existing streams linked to this object - self.streams_list.clear() - streams_data = load_yaml_to_obj(file_path) - assert isinstance(streams_data, list) - new_streams_data = [] - for stream in streams_data: - stream_name = stream.get("name") - raw_stream = stream.get("stream") - if not stream_name or not raw_stream: - raise ValueError("Provided stream is not according to convention." - "Each stream must be provided as two keys: 'name' and 'stream'. " - "Provided item was:\n {stream}".format(stream)) - new_stream_data = self.yaml_loader.validate_yaml(raw_stream, - "stream", - multiplier= multiplier) - new_streams_data.append(new_stream_data) - new_stream_obj = CStream() - new_stream_obj.load_data(**new_stream_data) - self.append_stream(stream_name, new_stream_obj) - return new_streams_data - - def compile_streams(self): - # first, assign an id to each stream - stream_ids = {} - for idx, stream_name in enumerate(self.streams_list): - stream_ids[stream_name] = idx - - # next, iterate over the streams and transform them from working with names to ids. - # with that build a new dict with old stream_name as the key, and StreamPack as the stored value - compiled_streams = {} - for stream_name, stream in self.streams_list.items(): - tmp_stream = CStreamList._compile_single_stream(stream_name, stream, stream_ids) - compiled_streams[stream_name] = StreamPack(stream_ids.get(stream_name), - tmp_stream) - return compiled_streams - - @staticmethod - def _compile_single_stream(stream_name, stream, id_dict): - # copy the old stream to temporary one, no change to class attributes - tmp_stream = copy.copy(stream) - next_stream_id = id_dict.get(getattr(tmp_stream, "next_stream_id"), -1) - try: - rx_stats_stream_id = id_dict.get(getattr(tmp_stream.rx_stats, "stream_id"), - id_dict.get(stream_name)) - except AttributeError as e: - rx_stats_stream_id = id_dict.get(stream_name) - # assign resolved values to stream object - tmp_stream.next_stream_id = next_stream_id - tmp_stream.rx_stats.stream_id = rx_stats_stream_id - return tmp_stream - - -class CRxStats(object): - - FIELDS = ["seq_enabled", "latency_enabled", "stream_id"] - def __init__(self, enabled=False, **kwargs): - self.enabled = bool(enabled) - for field in CRxStats.FIELDS: - setattr(self, field, kwargs.get(field, False)) - - def dump(self): - if self.enabled: - dump = {"enabled": True} - dump.update({k: getattr(self, k) - for k in CRxStats.FIELDS} - ) - return dump - else: - return {"enabled": False} - - - -class CTxMode(object): - """docstring for CTxMode""" - GENERAL_FIELDS = ["type", "pps"] - FIELDS = {"continuous": [], - "single_burst": ["total_pkts"], - "multi_burst": ["pkts_per_burst", "ibg", "count"]} - - def __init__(self, type, pps=0, **kwargs): - self._MODES = CTxMode.FIELDS.keys() - self.type = type - self.pps = pps - for field in CTxMode.FIELDS.get(self.type): - setattr(self, field, kwargs.get(field, 0)) - - @property - def type(self): - return self._type - - @type.setter - def type(self, type): - if type not in self._MODES: - raise ValueError("Unknown TX mode ('{0}')has been initialized.".format(type)) - self._type = type - self._reset_fields() - - def dump(self): - dump = ({k: getattr(self, k) - for k in CTxMode.GENERAL_FIELDS - }) - dump.update({k: getattr(self, k) - for k in CTxMode.FIELDS.get(self.type) - }) - return dump - - def _reset_fields(self): - for field in CTxMode.FIELDS.get(self.type): - setattr(self, field, 0) - - -class CStream(object): - """docstring for CStream""" - - FIELDS = ["enabled", "self_start", "next_stream_id", "isg", "mode", "rx_stats", "packet", "vm"] - - def __init__(self): - self.is_loaded = False - self._is_compiled = False - self._pkt_bld_obj = CTRexPktBuilder() - for field in CStream.FIELDS: - setattr(self, field, None) - - - def load_data(self, **kwargs): - try: - for k in CStream.FIELDS: - if k == "rx_stats": - rx_stats_data = kwargs[k] - if isinstance(rx_stats_data, dict): - setattr(self, k, CRxStats(**rx_stats_data)) - elif isinstance(rx_stats_data, CRxStats): - setattr(self, k, rx_stats_data) - elif k == "mode": - tx_mode = kwargs[k] - if isinstance(tx_mode, dict): - setattr(self, k, CTxMode(**tx_mode)) - elif isinstance(tx_mode, CTxMode): - setattr(self, k, tx_mode) - elif k == "packet": - if isinstance(kwargs[k], CTRexPktBuilder): - if "vm" not in kwargs: - self.load_packet_obj(kwargs[k]) - break # vm field check is skipped - else: - raise ValueError("When providing packet object with a CTRexPktBuilder, vm parameter " - "should not be supplied") - else: - binary = kwargs[k]["binary"] - if isinstance(binary, str): - - # TODO: load to _pkt_bld_obj also when passed as byte array! - if binary.endswith(".pcap"): - self._pkt_bld_obj.load_packet_from_pcap(binary) - self._pkt_bld_obj.metadata = kwargs[k]["meta"] - self.packet = self._pkt_bld_obj.dump_pkt() - else: - self.packet = {} - self.packet['binary'] = binary - self.packet['meta'] = "" - - else: - raise ValueError("Packet binary attribute has been loaded with unsupported value." - "Supported values are reference to pcap file with SINGLE packet, " - "or a list of unsigned-byte integers") - else: - setattr(self, k, kwargs[k]) - self.is_loaded = True - except KeyError as e: - cause = e.args[0] - raise KeyError("The attribute '{0}' is missing as a field of the CStream object.\n" - "Loaded data must contain all of the following fields: {1}".format(cause, CStream.FIELDS)) - - def load_packet_obj(self, packet_obj): - assert isinstance(packet_obj, CTRexPktBuilder) - self.packet = packet_obj.dump_pkt() - self.vm = packet_obj.get_vm_data() - - def load_packet_from_pcap(self, pcap_path, metadata=''): - with open(pcap_path, 'r') as f: - pcap = dpkt.pcap.Reader(f) - first_packet = True - for _, buf in pcap: - # this is an iterator, can't evaluate the number of files in advance - if first_packet: - self.packet = {"binary": [struct.unpack('B', buf[i:i+1])[0] # represent data as list of 0-255 ints - for i in range(0, len(buf))], - "meta": metadata} # meta data continues without a change. - first_packet = False - else: - raise ValueError("Provided pcap file contains more than single packet.") - # arrive here ONLY if pcap contained SINGLE packet - return - - - def dump(self): - if self.is_loaded: - dump = {} - for key in CStream.FIELDS: - try: - dump[key] = getattr(self, key).dump() # use dump() method of compound object, such TxMode - except AttributeError: - dump[key] = getattr(self, key) - return dump - else: - raise RuntimeError("CStream object isn't loaded with data. Use 'load_data' method.") - - def get_stream_layers(self, depth_limit=Ellipsis): - stream_layers = self._pkt_bld_obj.get_packet_layers(depth_limit) - return "/".join(stream_layers) - - - -# describes a stream DB -class CStreamsDB(object): - - def __init__(self): - self.stream_packs = {} - - def load_yaml_file(self, filename): - - stream_pack_name = filename - if stream_pack_name in self.get_loaded_streams_names(): - self.remove_stream_packs(stream_pack_name) - - stream_list = CStreamList() - loaded_obj = stream_list.load_yaml(filename) - - try: - compiled_streams = stream_list.compile_streams() - rc = self.load_streams(LoadedStreamList(stream_pack_name, - loaded_obj, - [StreamPack(v.stream_id, v.stream.dump()) - for k, v in compiled_streams.items()])) - except Exception as e: - return None - - - return self.get_stream_pack(stream_pack_name) - - def load_streams(self, LoadedStreamList_obj): - if LoadedStreamList_obj.name in self.stream_packs: - return False - else: - self.stream_packs[LoadedStreamList_obj.name] = LoadedStreamList_obj - return True - - def remove_stream_packs(self, *names): - removed_streams = [] - for name in names: - removed = self.stream_packs.pop(name) - if removed: - removed_streams.append(name) - return removed_streams - - def clear(self): - self.stream_packs.clear() - - def get_loaded_streams_names(self): - return self.stream_packs.keys() - - def stream_pack_exists (self, name): - return name in self.get_loaded_streams_names() - - def get_stream_pack(self, name): - if not self.stream_pack_exists(name): - return None - else: - return self.stream_packs.get(name) - - -########################### Simple Streams ########################### -from trex_stl_exceptions import * - -# base class for TX mode -class STLTXMode(object): - def __init__ (self): - self.fields = {} - - def to_json (self): - return self.fields - - -# continuous mode -class STLTXCont(STLTXMode): - - def __init__ (self, pps = 1): - - if not isinstance(pps, (int, float)): - raise STLArgumentError('pps', pps) - - super(STLTXCont, self).__init__() - - self.fields['type'] = 'continuous' - self.fields['pps'] = pps - - -# single burst mode -class STLTXSingleBurst(STLTXMode): - - def __init__ (self, pps = 1, total_pkts = 1): - - if not isinstance(pps, (int, float)): - raise STLArgumentError('pps', pps) - - if not isinstance(total_pkts, int): - raise STLArgumentError('total_pkts', total_pkts) - - super(STLTXSingleBurst, self).__init__() - - self.fields['type'] = 'single_burst' - self.fields['pps'] = pps - self.fields['total_pkts'] = total_pkts - - -# multi burst mode -class STLTXMultiBurst(STLTXMode): - - def __init__ (self, - pps = 1, - pkts_per_burst = 1, - ibg = 0.0, - count = 1): - - if not isinstance(pps, (int, float)): - raise STLArgumentError('pps', pps) - - if not isinstance(pkts_per_burst, int): - raise STLArgumentError('pkts_per_burst', pkts_per_burst) - - if not isinstance(ibg, (int, float)): - raise STLArgumentError('ibg', ibg) - - if not isinstance(count, int): - raise STLArgumentError('count', count) - - super(STLTXMultiBurst, self).__init__() - - self.fields['type'] = 'multi_burst' - self.fields['pps'] = pps - self.fields['pkts_per_burst'] = pkts_per_burst - self.fields['ibg'] = ibg - self.fields['count'] = count - - -class STLStream(object): - - def __init__ (self, - packet, - mode = STLTXCont(1), - enabled = True, - self_start = True, - isg = 0.0, - rx_stats = None, - next_stream_id = -1, - stream_id = None): - - # type checking - if not isinstance(mode, STLTXMode): - raise STLArgumentError('mode', mode) - - if not isinstance(packet, CTrexPktBuilderInterface): - raise STLArgumentError('packet', packet) - - if not isinstance(enabled, bool): - raise STLArgumentError('enabled', enabled) - - if not isinstance(self_start, bool): - raise STLArgumentError('self_start', self_start) - - if not isinstance(isg, (int, float)): - raise STLArgumentError('isg', isg) - - if (type(mode) == STLTXCont) and (next_stream_id != -1): - raise STLError("continuous stream cannot have a next stream ID") - - # use a random 31 bit for ID - self.stream_id = stream_id if stream_id is not None else random.getrandbits(31) - - self.fields = {} - - # basic fields - self.fields['enabled'] = enabled - self.fields['self_start'] = self_start - self.fields['isg'] = isg - - self.fields['next_stream_id'] = next_stream_id - - # mode - self.fields['mode'] = mode.to_json() - - packet.compile() - - # packet and VM - self.fields['packet'] = packet.dump_pkt() - self.fields['vm'] = packet.get_vm_data() - - self.fields['rx_stats'] = {} - if not rx_stats: - self.fields['rx_stats']['enabled'] = False - - - def __str__ (self): - return json.dumps(self.fields, indent = 4, separators=(',', ': '), sort_keys = True) - - def to_json (self): - return self.fields - - def get_id (self): - return self.stream_id - - @staticmethod - def dump_to_yaml (yaml_file, stream_list): - - # type check - if isinstance(stream_list, STLStream): - stream_list = [stream_list] - - if not all([isinstance(stream, STLStream) for stream in stream_list]): - raise STLArgumentError('stream_list', stream_list) - - - names = {} - for i, stream in enumerate(stream_list): - names[stream.get_id()] = "stream-{0}".format(i) - - yaml_lst = [] - for stream in stream_list: - - fields = dict(stream.fields) - - # handle the next stream id - if fields['next_stream_id'] == -1: - del fields['next_stream_id'] - - else: - if not stream.get_id() in names: - raise STLError('broken dependencies in stream list') - - fields['next_stream'] = names[stream.get_id()] - - # add to list - yaml_lst.append({'name': names[stream.get_id()], 'stream': fields}) - - # write to file - x = yaml.dump(yaml_lst, default_flow_style=False) - with open(yaml_file, 'w') as f: - f.write(x) - return x - - -# REMOVE ME when can - convert from stream pack to a simple stream -class HACKSTLStream(STLStream): - def __init__ (self, stream_pack): - if not isinstance(stream_pack, StreamPack): - raise Exception("internal error") - - packet = CTRexPktBuilder() - packet.load_from_stream_obj(stream_pack.stream) - super(HACKSTLStream, self).__init__(packet, stream_id = stream_pack.stream_id) - - self.fields = stream_pack.stream diff --git a/scripts/automation/trex_control_plane/common/trex_types.py b/scripts/automation/trex_control_plane/common/trex_types.py deleted file mode 100644 index a7ddacea..00000000 --- a/scripts/automation/trex_control_plane/common/trex_types.py +++ /dev/null @@ -1,95 +0,0 @@ - -from collections import namedtuple -from common.text_opts import * - -RpcCmdData = namedtuple('RpcCmdData', ['method', 'params']) - -class RpcResponseStatus(namedtuple('RpcResponseStatus', ['success', 'id', 'msg'])): - __slots__ = () - def __str__(self): - return "{id:^3} - {msg} ({stat})".format(id=self.id, - msg=self.msg, - stat="success" if self.success else "fail") - -# simple class to represent complex return value -class RC(): - - def __init__ (self, rc = None, data = None, is_warn = False): - self.rc_list = [] - - if (rc != None): - tuple_rc = namedtuple('RC', ['rc', 'data', 'is_warn']) - self.rc_list.append(tuple_rc(rc, data, is_warn)) - - def __nonzero__ (self): - return self.good() - - - def add (self, rc): - self.rc_list += rc.rc_list - - def good (self): - return all([x.rc for x in self.rc_list]) - - def bad (self): - return not self.good() - - def warn (self): - return any([x.is_warn for x in self.rc_list]) - - def data (self): - d = [x.data if x.rc else "" for x in self.rc_list] - return (d if len(d) != 1 else d[0]) - - def err (self): - e = [x.data if not x.rc else "" for x in self.rc_list] - return (e if len(e) != 1 else e[0]) - - def __str__ (self): - s = "" - for x in self.rc_list: - if x.data: - s += format_text("\n{0}".format(x.data), 'bold') - return s - - def prn_func (self, msg, newline = True): - if newline: - print msg - else: - print msg, - - def annotate (self, log_func = None, desc = None, show_status = True): - - if not log_func: - log_func = self.prn_func - - if desc: - log_func(format_text('\n{:<60}'.format(desc), 'bold'), newline = False) - else: - log_func("") - - if self.bad(): - # print all the errors - print "" - for x in self.rc_list: - if not x.rc: - log_func(format_text("\n{0}".format(x.data), 'bold')) - - print "" - if show_status: - log_func(format_text("[FAILED]\n", 'red', 'bold')) - - - else: - if show_status: - log_func(format_text("[SUCCESS]\n", 'green', 'bold')) - - -def RC_OK(data = ""): - return RC(True, data) - -def RC_ERR (err): - return RC(False, err) - -def RC_WARN (warn): - return RC(True, warn, is_warn = True) -- cgit 1.2.3-korg