From afefddfa387dad83bbcb15812bd279cf3197f583 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Tue, 8 Mar 2016 10:28:20 +0200 Subject: regression: add stateless support (WIP); Scapy builder: fix remove MACs from binary, do not build by default; --- .../trex_stl_lib/trex_stl_packet_builder_scapy.py | 95 ++++++++++++---------- 1 file changed, 50 insertions(+), 45 deletions(-) (limited to 'scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py') diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py index 6a2e268f..bb210226 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py @@ -1,7 +1,7 @@ import random import string import struct -import socket +import socket import json import yaml import binascii @@ -50,13 +50,13 @@ def mac_str_to_num (mac_buffer): def is_valid_ipv4(ip_addr): """ - return buffer in network order + return buffer in network order """ if type(ip_addr)==str and len(ip_addr) == 4: return ip_addr if type(ip_addr)==int : - ip_addr = socket.inet_ntoa(struct.pack("!I", ip_addr)) + ip_addr = socket.inet_ntoa(struct.pack("!I", ip_addr)) try: return socket.inet_pton(socket.AF_INET, ip_addr) @@ -81,7 +81,7 @@ def is_valid_ipv6(ipv6_addr): class CTRexScriptsBase(object): """ - VM Script base class + VM Script base class """ def clone (self): return copy.deepcopy(self) @@ -105,7 +105,7 @@ class CTRexScFieldRangeValue(CTRexScFieldRangeBase): """ range of field value """ - def __init__(self, field_name, + def __init__(self, field_name, field_type, min_value, max_value @@ -135,7 +135,7 @@ class CTRexScIpv4SimpleRange(CTRexScFieldRangeBase): class CTRexScIpv4TupleGen(CTRexScriptsBase): """ - range tuple + range tuple """ FLAGS_ULIMIT_FLOWS =1 @@ -157,7 +157,7 @@ class CTRexScIpv4TupleGen(CTRexScriptsBase): class CTRexScTrimPacketSize(CTRexScriptsBase): """ - trim packet size. field type is CTRexScFieldRangeBase.FILED_TYPES = ["inc","dec","rand"] + trim packet size. field type is CTRexScFieldRangeBase.FILED_TYPES = ["inc","dec","rand"] """ def __init__(self,field_type="rand",min_pkt_size=None, max_pkt_size=None): super(CTRexScTrimPacketSize, self).__init__() @@ -174,7 +174,7 @@ class CTRexScTrimPacketSize(CTRexScriptsBase): class CTRexScRaw(CTRexScriptsBase): """ - raw instructions + raw instructions """ def __init__(self,list_of_commands=None,split_by_field=None): super(CTRexScRaw, self).__init__() @@ -190,7 +190,7 @@ class CTRexScRaw(CTRexScriptsBase): ################################################################################################ -# VM raw instructions +# VM raw instructions ################################################################################################ class CTRexVmInsBase(object): @@ -283,7 +283,7 @@ class CTRexVmInsTupleGen(CTRexVmInsBase): ################################################################################################ -# +# class CTRexVmEngine(object): def __init__(self): @@ -294,7 +294,7 @@ class CTRexVmEngine(object): self.ins=[] self.split_by_var = '' - # return as json + # return as json def get_json (self): inst_array = []; # dump it as dict @@ -352,7 +352,7 @@ class CTRexScapyPktUtl(object): def _layer_offset(self, name, cnt = 0): """ - return offset of layer e.g 'IP',1 will return offfset of layer ip:1 + return offset of layer e.g 'IP',1 will return offfset of layer ip:1 """ save_cnt=cnt for pkt in self.pkt_iter (): @@ -367,7 +367,7 @@ class CTRexScapyPktUtl(object): def layer_offset(self, name, cnt = 0): """ - return offset of layer e.g 'IP',1 will return offfset of layer ip:1 + return offset of layer e.g 'IP',1 will return offfset of layer ip:1 """ save_cnt=cnt for pkt in self.pkt_iter (): @@ -381,7 +381,7 @@ class CTRexScapyPktUtl(object): def get_field_offet(self, layer, layer_cnt, field_name): """ - return offset of layer e.g 'IP',1 will return offfset of layer ip:1 + return offset of layer e.g 'IP',1 will return offfset of layer ip:1 """ t=self._layer_offset(layer,layer_cnt); l_offset=t[1]; @@ -397,7 +397,7 @@ class CTRexScapyPktUtl(object): def get_layer_offet_by_str(self, layer_des): """ - return layer offset by string + return layer offset by string :parameters: @@ -423,14 +423,14 @@ class CTRexScapyPktUtl(object): def get_field_offet_by_str(self, field_des): """ - return field_des (offset,size) layer:cnt.field - for example + return field_des (offset,size) layer:cnt.field + for example 802|1Q.vlan get 802.1Q->valn replace | with . IP.src IP:0.src (first IP.src like IP.src) for example IP:1.src for internal IP - return (offset, size) as tuple + return (offset, size) as tuple """ @@ -489,19 +489,19 @@ class CTRexVmDescBase(object): def get_var_ref (self): ''' virtual function return a ref var name - ''' + ''' return None def get_var_name(self): ''' virtual function return the varible name if exists - ''' + ''' return None - def compile(self,parent): + def compile(self,parent): ''' virtual function to take parent than has function name_to_offset - ''' + ''' pass; @@ -558,12 +558,12 @@ class CTRexVmDescFlowVar(CTRexVmDescBase): class CTRexVmDescFixIpv4(CTRexVmDescBase): def __init__(self, offset): super(CTRexVmDescFixIpv4, self).__init__() - self.offset = offset; # could be a name of offset + self.offset = offset; # could be a name of offset def get_obj (self): return CTRexVmInsFixIpv4(self.offset); - def compile(self,parent): + def compile(self,parent): if type(self.offset)==str: self.offset = parent._pkt_layer_offset(self.offset); @@ -586,7 +586,7 @@ class CTRexVmDescWrFlowVar(CTRexVmDescBase): def get_obj (self): return CTRexVmInsWrFlowVar(self.name,self.pkt_offset+self.offset_fixup,self.add_val,self.is_big) - def compile(self,parent): + def compile(self,parent): if type(self.pkt_offset)==str: t=parent._name_to_offset(self.pkt_offset) self.pkt_offset = t[0] @@ -620,7 +620,7 @@ class CTRexVmDescWrMaskFlowVar(CTRexVmDescBase): def get_obj (self): return CTRexVmInsWrMaskFlowVar(self.name,self.pkt_offset+self.offset_fixup,self.pkt_cast_size,self.mask,self.shift,self.add_value,self.is_big) - def compile(self,parent): + def compile(self,parent): if type(self.pkt_offset)==str: t=parent._name_to_offset(self.pkt_offset) self.pkt_offset = t[0] @@ -670,11 +670,11 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): """ This class defines the TRex API of building a packet using scapy package. Using this class the user can also define how TRex will handle the packet by specifying the VM setting. - pkt could be Scapy pkt or pcap file name + pkt could be Scapy pkt or pcap file name When path_relative_to_profile is True load pcap file from path relative to the profile """ - def __init__(self, pkt = None, pkt_buffer = None, vm = None, path_relative_to_profile = False, build_raw = True, remove_fcs = True): + def __init__(self, pkt = None, pkt_buffer = None, vm = None, path_relative_to_profile = False, build_raw = False, remove_fcs = True): """ Instantiate a CTRexPktBuilder object @@ -684,7 +684,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): """ super(CScapyTRexPktBuilder, self).__init__() - self.pkt = None # as input + self.pkt = None # as input self.pkt_raw = None # from raw pcap file self.vm_scripts = [] # list of high level instructions self.vm_low_level = None @@ -692,7 +692,8 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): self.metadata="" self.path_relative_to_profile = path_relative_to_profile self.remove_fcs = remove_fcs - + self.is_binary_source = pkt_buffer != None + if pkt != None and pkt_buffer != None: raise CTRexPacketBuildException(-15, "packet builder cannot be provided with both pkt and pkt_buffer") @@ -725,7 +726,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): def get_vm_data(self): """ - Dumps the instructions + Dumps the instructions :parameters: None @@ -739,7 +740,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): assert self.vm_low_level is not None, 'vm_low_level is None, please use compile()' - return self.vm_low_level.get_json() + return self.vm_low_level.get_json() def dump_pkt(self, encode = True): """ @@ -763,7 +764,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): return {'binary': base64.b64encode(pkt_buf) if encode else pkt_buf, 'meta': self.metadata} - + def dump_pkt_to_pcap(self, file_path): wrpcap(file_path, self._get_pkt_as_str()) @@ -799,7 +800,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): def set_pcap_file (self, pcap_file): """ - load raw pcap file into a buffer. load only the first packet + load raw pcap file into a buffer. load only the first packet :parameters: pcap_file : file_name @@ -845,7 +846,9 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): else: raise CTRexPacketBuildException(-14, "bad packet" ) - def is_def_src_mac (self): + def is_default_src_mac (self): + if self.is_binary_source: + return True p = self.pkt if isinstance(p, Packet): if isinstance(p,Ether): @@ -853,7 +856,9 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): return False return True - def is_def_dst_mac (self): + def is_default_dst_mac (self): + if self.is_binary_source: + return True p = self.pkt if isinstance(p, Packet): if isinstance(p,Ether): @@ -865,7 +870,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): if self.pkt == None and self.pkt_raw == None: raise CTRexPacketBuildException(-14, "Packet is empty") - + self.vm_low_level = CTRexVmEngine() # compile the VM @@ -882,7 +887,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): raise CTRexPacketBuildException(-14, "Packet is empty") #################################################### - # private + # private def _get_pcap_file_path (self,pcap_file_name): @@ -891,7 +896,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): f_path = pcap_file_name else: if self.path_relative_to_profile: - p = self._get_path_relative_to_profile () # loader + p = self._get_path_relative_to_profile () # loader if p : f_path=os.path.abspath(os.path.join(os.path.dirname(p),pcap_file_name)) @@ -907,7 +912,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): def _compile_raw (self,obj): - # make sure we have varibles once + # make sure we have varibles once vars={}; # add it add var to dit @@ -926,17 +931,17 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): var_name = desc.get_var_ref() if var_name : if not vars.has_key(var_name): - raise CTRexPacketBuildException(-11,("variable %s does not exists ") % (var_name) ); + raise CTRexPacketBuildException(-11,("variable %s does not exists ") % (var_name) ); desc.compile(self); for desc in obj.commands: self.vm_low_level.add_ins(desc.get_obj()); # set split_by_var - if obj.split_by_field : + if obj.split_by_field : assert type(obj.split_by_field)==str, "type of split by var should be string" #if not vars.has_key(obj.split_by_field): - # raise CTRexPacketBuildException(-11,("variable %s does not exists. change split_by_var args ") % (var_name) ); + # raise CTRexPacketBuildException(-11,("variable %s does not exists. change split_by_var args ") % (var_name) ); self.vm_low_level.split_by_var = obj.split_by_field @@ -955,7 +960,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): # regular scapy packet elif not self.pkt: # should not reach here - raise CTRexPacketBuildException(-11, 'empty packet') + raise CTRexPacketBuildException(-11, 'empty packet') if self.remove_fcs and self.pkt.lastlayer().name == 'Padding': self.pkt.lastlayer().underlayer.remove_payload() @@ -983,7 +988,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): return str(self.pkt) if self.pkt_raw: return self.pkt_raw - raise CTRexPacketBuildException(-11, 'empty packet'); + raise CTRexPacketBuildException(-11, 'empty packet'); def _add_tuple_gen(self,tuple_gen): -- cgit 1.2.3-korg From 60fbd456c4d804adc903839f916c9c2bbe272d94 Mon Sep 17 00:00:00 2001 From: imarom Date: Tue, 8 Mar 2016 09:20:02 +0200 Subject: RX stats - major refactor --- .../trex_control_plane/stl/console/trex_tui.py | 38 ++- .../stl/examples/stl_flow_stats.py | 1 - .../stl/trex_stl_lib/trex_stl_client.py | 15 +- .../trex_stl_lib/trex_stl_packet_builder_scapy.py | 3 +- .../stl/trex_stl_lib/trex_stl_stats.py | 353 ++++++++++++++++----- .../stl/trex_stl_lib/utils/parsing_opts.py | 10 +- scripts/stl/flow_stats.py | 8 +- src/internal_api/trex_platform_api.h | 2 +- src/sim/trex_sim.h | 2 + src/sim/trex_sim_stateless.cpp | 33 ++ src/stateless/cp/trex_dp_port_events.cpp | 6 + src/stateless/cp/trex_stateless_port.cpp | 9 +- 12 files changed, 380 insertions(+), 100 deletions(-) (limited to 'scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py') diff --git a/scripts/automation/trex_control_plane/stl/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py index f972b905..02b00b78 100644 --- a/scripts/automation/trex_control_plane/stl/console/trex_tui.py +++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py @@ -8,6 +8,7 @@ from cStringIO import StringIO from trex_stl_lib.utils.text_opts import * from trex_stl_lib.utils import text_tables +from trex_stl_lib import trex_stl_stats # for STL exceptions from trex_stl_lib.api import * @@ -217,6 +218,35 @@ class TrexTUIPort(TrexTUIPanel): self.stateless_client.clear_stats([self.port_id]) return "port {0}: cleared stats".format(self.port_id) + + +# streams stats +class TrexTUIStreamsStats(TrexTUIPanel): + def __init__ (self, mng): + super(TrexTUIStreamsStats, self).__init__(mng, "sstats") + + self.key_actions = OrderedDict() + + self.key_actions['c'] = {'action': self.action_clear, 'legend': 'clear', 'show': True} + + + def show (self): + stats = self.stateless_client._get_formatted_stats(port_id_list = None, stats_mask = trex_stl_stats.SS_COMPAT) + # print stats to screen + for stat_type, stat_data in stats.iteritems(): + text_tables.print_table_with_header(stat_data.text_table, stat_type) + pass + + + def get_key_actions (self): + return self.key_actions + + def action_clear (self): + self.stateless_client.flow_stats.clear_stats() + + return "" + + # log class TrexTUILog(): def __init__ (self): @@ -247,10 +277,12 @@ class TrexTUIPanelManager(): self.panels = {} self.panels['dashboard'] = TrexTUIDashBoard(self) + self.panels['sstats'] = TrexTUIStreamsStats(self) self.key_actions = OrderedDict() self.key_actions['q'] = {'action': self.action_quit, 'legend': 'quit', 'show': True} self.key_actions['g'] = {'action': self.action_show_dash, 'legend': 'dashboard', 'show': True} + self.key_actions['s'] = {'action': self.action_show_sstats, 'legend': 'streams stats', 'show': True} for port_id in self.ports: self.key_actions[str(port_id)] = {'action': self.action_show_port(port_id), 'legend': 'port {0}'.format(port_id), 'show': False} @@ -352,6 +384,10 @@ class TrexTUIPanelManager(): return action_show_port_x + def action_show_sstats (self): + self.main_panel = self.panels['sstats'] + self.init(self.show_log) + return "" # shows a textual top style window class TrexTUI(): @@ -427,7 +463,7 @@ class TrexTUI(): elif self.state == self.STATE_RECONNECT: try: - self.stateless_client.connect("RO") + self.stateless_client.connect() self.state = self.STATE_ACTIVE except STLError: self.state = self.STATE_LOST_CONT diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py index 3708834e..657350f0 100644 --- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py +++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py @@ -14,7 +14,6 @@ def rx_example (tx_port, rx_port, burst_size): try: pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP()/'a_payload_example') - total_pkts = burst_size s1 = STLStream(name = 'rx', packet = pkt, diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py index a13e2793..7dc7ff32 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py @@ -436,10 +436,12 @@ class STLClient(object): self.server_version, self.ports) + self.flow_stats = trex_stl_stats.CRxStats() + self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats, - self.ports) + self.ports, + self.flow_stats) - self.flow_stats = trex_stl_stats.CRxStats() ############# private functions - used by the class itself ########### @@ -714,7 +716,7 @@ class STLClient(object): # clear stats - def __clear_stats(self, port_id_list, clear_global): + def __clear_stats(self, port_id_list, clear_global, clear_flow_stats): for port_id in port_id_list: self.ports[port_id].clear_stats() @@ -722,6 +724,9 @@ class STLClient(object): if clear_global: self.global_stats.clear_stats() + if clear_flow_stats: + self.flow_stats.clear_stats() + self.logger.log_cmd("clearing stats on port(s) {0}:".format(port_id_list)) return RC @@ -1546,7 +1551,7 @@ class STLClient(object): """ @__api_check(False) - def clear_stats (self, ports = None, clear_global = True): + def clear_stats (self, ports = None, clear_global = True, clear_flow_stats = True): ports = ports if ports is not None else self.get_all_ports() ports = self._validate_port_list(ports) @@ -1556,7 +1561,7 @@ class STLClient(object): raise STLArgumentError('clear_global', clear_global) - rc = self.__clear_stats(ports, clear_global) + rc = self.__clear_stats(ports, clear_global, clear_flow_stats) if not rc: raise STLError(rc) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py index bb210226..14c04a52 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py @@ -964,8 +964,7 @@ class CScapyTRexPktBuilder(CTrexPktBuilderInterface): if self.remove_fcs and self.pkt.lastlayer().name == 'Padding': self.pkt.lastlayer().underlayer.remove_payload() - if len(self.pkt) < 60: # simulator can write padding with non-zeros, set it explicit - self.pkt /= Padding('\x00' * (60 - len(self.pkt))) + self.pkt.build() self.is_pkt_built = True diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py index c2e318bc..78bd2358 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py @@ -17,11 +17,24 @@ import threading GLOBAL_STATS = 'g' PORT_STATS = 'p' PORT_STATUS = 'ps' -ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS} +STREAMS_STATS = 's' + +ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS} COMPACT = {GLOBAL_STATS, PORT_STATS} +SS_COMPAT = {GLOBAL_STATS, STREAMS_STATS} ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) +# deep mrege of dicts dst = src + dst +def deep_merge_dicts (dst, src): + for k, v in src.iteritems(): + # if not exists - deep copy it + if not k in dst: + dst[k] = copy.deepcopy(v) + else: + if isinstance(v, dict): + deep_merge_dicts(dst[k], v) + # use to calculate diffs relative to the previous values # for example, BW def calculate_diff (samples): @@ -66,18 +79,23 @@ class CTRexInfoGenerator(object): STLClient and the ports. """ - def __init__(self, global_stats_ref, ports_dict_ref): + def __init__(self, global_stats_ref, ports_dict_ref, rx_stats_ref): self._global_stats = global_stats_ref self._ports_dict = ports_dict_ref + self._rx_stats_ref = rx_stats_ref def generate_single_statistic(self, port_id_list, statistic_type): if statistic_type == GLOBAL_STATS: return self._generate_global_stats() + elif statistic_type == PORT_STATS: return self._generate_port_stats(port_id_list) - pass + elif statistic_type == PORT_STATUS: return self._generate_port_status(port_id_list) + + elif statistic_type == STREAMS_STATS: + return self._generate_streams_stats() else: # ignore by returning empty object return {} @@ -110,6 +128,92 @@ class CTRexInfoGenerator(object): return {"global_statistics": ExportableStats(stats_data, stats_table)} + def _generate_streams_stats (self): + + sstats_data = self._rx_stats_ref.generate_stats() + streams_keys = self._rx_stats_ref.get_streams_keys() + stream_count = len(streams_keys) + + + stats_table = text_tables.TRexTextTable() + stats_table.set_cols_align(["l"] + ["r"] * stream_count) + stats_table.set_cols_width([20] + [17] * stream_count) + stats_table.set_cols_dtype(['t'] + ['t'] * stream_count) + + stats_table.add_rows([[k] + v + for k, v in sstats_data.iteritems()], + header=False) + + header = ["PG ID"] + [key for key in streams_keys] + stats_table.header(header) + + return {"streams_statistics": ExportableStats(sstats_data, stats_table)} + + + + per_stream_stats = OrderedDict([("owner", []), + ("state", []), + ("--", []), + ("Tx bps L2", []), + ("Tx bps L1", []), + ("Tx pps", []), + ("Line Util.", []), + + ("---", []), + ("Rx bps", []), + ("Rx pps", []), + + ("----", []), + ("opackets", []), + ("ipackets", []), + ("obytes", []), + ("ibytes", []), + ("tx-bytes", []), + ("rx-bytes", []), + ("tx-pkts", []), + ("rx-pkts", []), + + ("-----", []), + ("oerrors", []), + ("ierrors", []), + + ] + ) + + total_stats = CPortStats(None) + + for port_obj in relevant_ports: + # fetch port data + port_stats = port_obj.generate_port_stats() + + total_stats += port_obj.port_stats + + # populate to data structures + return_stats_data[port_obj.port_id] = port_stats + self.__update_per_field_dict(port_stats, per_field_stats) + + total_cols = len(relevant_ports) + header = ["port"] + [port.port_id for port in relevant_ports] + + if (total_cols > 1): + self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats) + header += ['total'] + total_cols += 1 + + stats_table = text_tables.TRexTextTable() + stats_table.set_cols_align(["l"] + ["r"] * total_cols) + stats_table.set_cols_width([10] + [17] * total_cols) + stats_table.set_cols_dtype(['t'] + ['t'] * total_cols) + + stats_table.add_rows([[k] + v + for k, v in per_field_stats.iteritems()], + header=False) + + stats_table.header(header) + + return {"streams_statistics": ExportableStats(return_stats_data, stats_table)} + + def _generate_port_stats(self, port_id_list): relevant_ports = self.__get_relevant_ports(port_id_list) @@ -131,10 +235,10 @@ class CTRexInfoGenerator(object): ("ipackets", []), ("obytes", []), ("ibytes", []), - ("tx_bytes", []), - ("rx_bytes", []), - ("tx_pkts", []), - ("rx_pkts", []), + ("tx-bytes", []), + ("rx-bytes", []), + ("tx-pkts", []), + ("rx-pkts", []), ("-----", []), ("oerrors", []), @@ -278,58 +382,40 @@ class CTRexInfoGenerator(object): class CTRexStats(object): """ This is an abstract class to represent a stats object """ - def __init__(self): + def __init__(self, flowing = True): self.reference_stats = {} self.latest_stats = {} self.last_update_ts = time.time() self.history = deque(maxlen = 10) self.lock = threading.Lock() - def __getitem__(self, item): - # override this to allow quick and clean access to fields - if not item in self.latest_stats: - return "N/A" - - # item must exist - m = re.search('_(([a-z])ps)$', item) - if m: - # this is a non-relative item - unit = m.group(2) - if unit == "b": - return self.get(item, format=True, suffix="b/sec") - elif unit == "p": - return self.get(item, format=True, suffix="pkt/sec") - else: - return self.get(item, format=True, suffix=m.group(1)) - - m = re.search('^[i|o](a-z+)$', item) - if m: - # this is a non-relative item - type = m.group(1) - if type == "bytes": - return self.get_rel(item, format=True, suffix="B") - elif type == "packets": - return self.get_rel(item, format=True, suffix="pkts") - else: - # do not format with suffix - return self.get_rel(item, format=True) + # does the object gets a constant flow of data ? + self.flowing = flowing - # can't match to any known pattern, return N/A - return "N/A" + ######## abstract methods ########## + # get stats for user / API + def get_stats (self): + raise NotImplementedError() + # generate format stats (for TUI) def generate_stats(self): - # must be implemented by designated classes (such as port/ global stats) raise NotImplementedError() - def generate_extended_values (self, snapshot): + # called when a snapshot arrives - add more fields + def preprocess_snapshot (self, snapshot): raise NotImplementedError() + ######## END abstract methods ########## + + def __update_ref (self): + deep_merge_dicts(self.reference_stats, self.latest_stats) + def update(self, snapshot): # some extended generated values (from base values) - self.generate_extended_values(snapshot) + self.preprocess_snapshot(snapshot) # update self.latest_stats = snapshot @@ -339,42 +425,64 @@ class CTRexStats(object): diff_time = time.time() - self.last_update_ts - # 3 seconds is too much - this is the new reference - if (not self.reference_stats) or (diff_time > 3): - self.reference_stats = self.latest_stats + # handle the reference (base) + self.__update_ref() + + # for flowing objects 3 seconds is too much + if self.flowing and (diff_time > 3): + self.clear_stats() - self.last_update_ts = time.time() def clear_stats(self): - self.reference_stats = self.latest_stats + self.reference_stats = copy.deepcopy(self.latest_stats) def invalidate (self): self.latest_stats = {} + + def __get (self, src, field): + if isinstance(field, list): + # deep + value = src + for level in field: + if not level in value: + return None + value = value[level] + else: + # flat + if not field in src: + return None + value = src[field] + + return value + def get(self, field, format=False, suffix=""): - if not field in self.latest_stats: + value = self.__get(self.latest_stats, field) + if value == None: return "N/A" - if not format: - return self.latest_stats[field] - else: - return format_num(self.latest_stats[field], suffix) + + return value if not format else format_num(value, suffix) + def get_rel(self, field, format=False, suffix=""): - if not field in self.latest_stats: + ref_value = self.__get(self.reference_stats, field) + if ref_value == None: return "N/A" - if not format: - if not field in self.reference_stats: - print "REF: " + str(self.reference_stats) - print "BASE: " + str(self.latest_stats) + # if the latest does not have the value - its like the ref + latest_value = self.__get(self.latest_stats, field) + if latest_value == None: + latest_value = ref_value + + + value = latest_value - ref_value + + return value if not format else format_num(value, suffix) - return (self.latest_stats[field] - self.reference_stats[field]) - else: - return format_num(self.latest_stats[field] - self.reference_stats[field], suffix) # get trend for a field def get_trend (self, field, use_raw = False, percision = 10.0): @@ -458,7 +566,7 @@ class CGlobalStats(CTRexStats): return stats - def generate_extended_values (self, snapshot): + def preprocess_snapshot (self, snapshot): # L1 bps bps = snapshot.get("m_tx_bps") pps = snapshot.get("m_tx_pps") @@ -568,7 +676,7 @@ class CPortStats(CTRexStats): return stats - def generate_extended_values (self, snapshot): + def preprocess_snapshot (self, snapshot): # L1 bps bps = snapshot.get("m_total_tx_bps") pps = snapshot.get("m_total_tx_pps") @@ -627,10 +735,10 @@ class CPortStats(CTRexStats): "obytes" : self.get_rel("obytes"), "ibytes" : self.get_rel("ibytes"), - "tx_bytes": self.get_rel("obytes", format = True, suffix = "B"), - "rx_bytes": self.get_rel("ibytes", format = True, suffix = "B"), - "tx_pkts": self.get_rel("opackets", format = True, suffix = "pkts"), - "rx_pkts": self.get_rel("ipackets", format = True, suffix = "pkts"), + "tx-bytes": self.get_rel("obytes", format = True, suffix = "B"), + "rx-bytes": self.get_rel("ibytes", format = True, suffix = "B"), + "tx-pkts": self.get_rel("opackets", format = True, suffix = "pkts"), + "rx-pkts": self.get_rel("ipackets", format = True, suffix = "pkts"), "oerrors" : format_num(self.get_rel("oerrors"), compact = False, @@ -643,33 +751,120 @@ class CPortStats(CTRexStats): } -class CRxStats(object): +class CRxStats(CTRexStats): def __init__(self): - self.flow_stats = {} + super(CRxStats, self).__init__(flowing = False) + + + + def preprocess_snapshot (self, snapshot): + # heavy pre-processing here... + new_snapshot = {} + + if not 'timestamp' in snapshot: + raise ValueError("INTERNAL ERROR: RX stats snapshot MUST contain 'timestamp' field") + + for key, value in snapshot.iteritems(): + # skip non int values (simply copy) + if key == 'timestamp': + new_snapshot[key] = value + continue + + # all the rest MUST be ints + try: + pg_id = int(key) + except ValueError: + assert(0) + # handle PG ID + new_snapshot[pg_id] = {} + for field in ['tx_pkts', 'tx_bytes', 'rx_pkts']: + new_snapshot[pg_id][field] = {'total': 0} + if field in value: + for port, pv in value[field].iteritems(): + new_snapshot[pg_id][field][int(port)] = pv + new_snapshot[pg_id][field]['total'] += pv - def update (self, snapshot): - self.flow_stats = snapshot + snapshot.clear() + snapshot.update(new_snapshot) + def get_stats (self): stats = {} - for pg_id, pg_id_data in self.flow_stats.iteritems(): - # ignore non pg ID keys + + for pg_id in self.get_streams_keys(): + stats[pg_id] = {} + for field in ['tx_pkts', 'tx_bytes', 'rx_pkts']: + stats[pg_id][field] = {'total': self.get_rel([pg_id, field, 'total'])} + + for port, pv in self.reference_stats[pg_id][field].iteritems(): + stats[pg_id][field][port] = self.get_rel([pg_id, field, port]) + + return stats + + + + def get_streams_keys (self): + keys = [] + for user_id, user_id_data in self.reference_stats.iteritems(): + # ignore non user ID keys try: - pg_id = int(pg_id) + keys.append(int(user_id)) except ValueError: continue - # handle pg id - stats[pg_id] = {} - for field, per_port_data in pg_id_data.iteritems(): - stats[pg_id][field] = {} - for port, value in per_port_data.iteritems(): - stats[pg_id][field][int(port)] = value + return keys - return stats + def generate_stats (self): + + stats = self.get_stats() + pg_ids = stats.keys()[:4] + cnt = len(pg_ids) + + + formatted_stats = OrderedDict([ ('Tx bps L2', []), + ('Tx bps L1', []), + ('Tx pps', []), + ('---', [''] * cnt), + ('Rx pps', []), + ('----', [''] * cnt), + ('opackets', []), + ('ipackets', []), + ('obytes', []), + ('ibytes', []), + ('-----', [''] * cnt), + ('tx_pkts', []), + ('rx_pkts', []), + ('tx_bytes', []), + ('rx_bytes', []) + ]) + + + + # maximum 4 + for pg_id in pg_ids: + + #formatted_stats['TX packet count'].append(stats[key]['tx-pkts']['total']) + #formatted_stats['TX byte count'].append(stats[key]['tx-bytes']['total']) + #formatted_stats['RX packet count'].append(stats[key]['rx-pkts']['total']) + formatted_stats['Tx bps L2'].append(0) + formatted_stats['Tx bps L1'].append(0) + formatted_stats['Tx pps'].append(0) + formatted_stats['Rx pps'].append(0) + formatted_stats['opackets'].append(self.get_rel([pg_id, 'tx_pkts', 'total'])) + formatted_stats['ipackets'].append(self.get_rel([pg_id, 'rx_pkts', 'total'])) + formatted_stats['obytes'].append(self.get_rel([pg_id, 'tx_bytes', 'total'])) + formatted_stats['ibytes'].append(self.get_rel([pg_id, 'rx_bytes', 'total'])) + formatted_stats['tx_bytes'].append(self.get_rel([pg_id, 'tx_bytes', 'total'], format = True, suffix = "B")) + formatted_stats['rx_bytes'].append(self.get_rel([pg_id, 'rx_bytes', 'total'], format = True, suffix = "B")) + formatted_stats['tx_pkts'].append(self.get_rel([pg_id, 'tx_pkts', 'total'], format = True, suffix = "pkts")) + formatted_stats['rx_pkts'].append(self.get_rel([pg_id, 'rx_pkts', 'total'], format = True, suffix = "pkts")) + + + + return formatted_stats if __name__ == "__main__": pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py index 649c192a..0390ac9c 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py @@ -34,7 +34,8 @@ PROMISCUOUS_SWITCH = 21 GLOBAL_STATS = 50 PORT_STATS = 51 PORT_STATUS = 52 -STATS_MASK = 53 +STREAMS_STATS = 53 +STATS_MASK = 54 STREAMS_MASK = 60 # ALL_STREAMS = 61 @@ -312,6 +313,10 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'], {'action': 'store_true', 'help': "Fetch only port status data"}), + STREAMS_STATS: ArgumentPack(['-s'], + {'action': 'store_true', + 'help': "Fetch only streams stats"}), + STREAMS_MASK: ArgumentPack(['--streams'], {"nargs": '+', 'dest':'streams', @@ -336,7 +341,8 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'], {'required': True}), STATS_MASK: ArgumentGroup(MUTEX, [GLOBAL_STATS, PORT_STATS, - PORT_STATUS], + PORT_STATUS, + STREAMS_STATS], {}) } diff --git a/scripts/stl/flow_stats.py b/scripts/stl/flow_stats.py index 05d7a9f7..e2701d23 100644 --- a/scripts/stl/flow_stats.py +++ b/scripts/stl/flow_stats.py @@ -6,8 +6,12 @@ class STLS1(object): def get_streams (self, direction = 0): return [STLStream(packet = STLPktBuilder(pkt ="stl/yaml/udp_64B_no_crc.pcap"), # path relative to pwd - mode = STLTXCont(pps=10), - flow_stats = STLFlowStats(pg_id = 7)) + mode = STLTXCont(pps=1000), + flow_stats = STLFlowStats(pg_id = 7)), + + STLStream(packet = STLPktBuilder(pkt ="stl/yaml/udp_594B_no_crc.pcap"), # path relative to pwd + mode = STLTXCont(pps=5000), + flow_stats = STLFlowStats(pg_id = 12)) ] diff --git a/src/internal_api/trex_platform_api.h b/src/internal_api/trex_platform_api.h index b1cf2fb7..e9cf56d7 100644 --- a/src/internal_api/trex_platform_api.h +++ b/src/internal_api/trex_platform_api.h @@ -214,7 +214,7 @@ public: virtual void get_interface_stats(uint8_t interface_id, TrexPlatformInterfaceStats &stats) const { } - virtual void get_interface_stat_info(uint8_t interface_id, uint16_t &num_counters, uint16_t &capabilities) const {num_counters=128; capabilities=0; } + virtual void get_interface_stat_info(uint8_t interface_id, uint16_t &num_counters, uint16_t &capabilities) const {num_counters=128; capabilities=TrexPlatformApi::IF_STAT_IPV4_ID; } virtual void port_id_to_cores(uint8_t port_id, std::vector> &cores_id_list) const { for (int i = 0; i < m_dp_core_count; i++) { diff --git a/src/sim/trex_sim.h b/src/sim/trex_sim.h index 59184b75..5aeeb226 100644 --- a/src/sim/trex_sim.h +++ b/src/sim/trex_sim.h @@ -149,7 +149,9 @@ private: uint64_t &simulated_pkts, uint64_t &written_pkts); + void cleanup(); void flush_dp_to_cp_messages_core(int core_index); + void flush_cp_to_dp_messages_core(int core_index); void validate_response(const Json::Value &resp); diff --git a/src/sim/trex_sim_stateless.cpp b/src/sim/trex_sim_stateless.cpp index 87c61ae2..ffe377f4 100644 --- a/src/sim/trex_sim_stateless.cpp +++ b/src/sim/trex_sim_stateless.cpp @@ -347,6 +347,10 @@ SimStateless::run_dp(const std::string &out_filename) { } } + /* cleanup */ + cleanup(); + + std::cout << "\n\nSimulation summary:\n"; std::cout << "-------------------\n\n"; std::cout << "simulated " << simulated_pkts_cnt << " packets\n"; @@ -360,6 +364,18 @@ SimStateless::run_dp(const std::string &out_filename) { std::cout << "\n"; } +void +SimStateless::cleanup() { + + for (int port_id = 0; port_id < get_stateless_obj()->get_port_count(); port_id++) { + get_stateless_obj()->get_port_by_id(port_id)->stop_traffic(); + get_stateless_obj()->get_port_by_id(port_id)->remove_and_delete_all_streams(); + } + for (int i = 0; i < m_dp_core_count; i++) { + flush_cp_to_dp_messages_core(i); + flush_dp_to_cp_messages_core(i); + } +} uint64_t SimStateless::get_limit_per_core(int core_index) { @@ -418,6 +434,23 @@ SimStateless::flush_dp_to_cp_messages_core(int core_index) { } } +void +SimStateless::flush_cp_to_dp_messages_core(int core_index) { + + CNodeRing *ring = CMsgIns::Ins()->getCpDp()->getRingCpToDp(core_index); + + while ( true ) { + CGenNode * node = NULL; + if (ring->Dequeue(node) != 0) { + break; + } + assert(node); + + TrexStatelessCpToDpMsgBase * msg = (TrexStatelessCpToDpMsgBase *)node; + delete msg; + } +} + bool SimStateless::should_capture_core(int i) { diff --git a/src/stateless/cp/trex_dp_port_events.cpp b/src/stateless/cp/trex_dp_port_events.cpp index 8e098adf..1321a362 100644 --- a/src/stateless/cp/trex_dp_port_events.cpp +++ b/src/stateless/cp/trex_dp_port_events.cpp @@ -82,6 +82,12 @@ protected: void TrexDpPortEvents::barrier() { + + /* simulator will be stuck here forever */ + #ifdef TREX_SIM + return; + #endif + int barrier_id = create_event(new DPBarrier()); TrexStatelessCpToDpMsgBase *barrier_msg = new TrexStatelessDpBarrier(m_port->m_port_id, barrier_id); diff --git a/src/stateless/cp/trex_stateless_port.cpp b/src/stateless/cp/trex_stateless_port.cpp index 7302e05d..5947aaf7 100644 --- a/src/stateless/cp/trex_stateless_port.cpp +++ b/src/stateless/cp/trex_stateless_port.cpp @@ -103,14 +103,9 @@ TrexStatelessPort::TrexStatelessPort(uint8_t port_id, const TrexPlatformApi *api } TrexStatelessPort::~TrexStatelessPort() { - if (m_graph_obj) { - delete m_graph_obj; - } - if (m_pending_async_stop_event != TrexDpPortEvents::INVALID_ID) { - m_dp_events.destroy_event(m_pending_async_stop_event); - m_pending_async_stop_event = TrexDpPortEvents::INVALID_ID; - } + stop_traffic(); + remove_and_delete_all_streams(); } /** -- cgit 1.2.3-korg