diff options
Diffstat (limited to 'scripts/automation/trex_control_plane/stl/trex_stl_lib')
12 files changed, 1133 insertions, 368 deletions
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py index 0f0fe83e..022077a9 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py @@ -275,18 +275,18 @@ class CTRexAsyncClient(): # stats if name == "trex-global": - self.event_handler.handle_async_stats_update(data, baseline) + self.event_handler.on_async_stats_update(data, baseline) # events elif name == "trex-event": - self.event_handler.handle_async_event(type, data) + self.event_handler.on_async_event(type, data) # barriers elif name == "trex-barrier": self.handle_async_barrier(type, data) elif name == "flow_stats": - self.event_handler.handle_async_rx_stats_event(data, baseline) + self.event_handler.on_async_rx_stats_event(data, baseline) else: pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py index bddc4ad0..77fa40bb 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py @@ -12,6 +12,7 @@ from .trex_stl_types import * from .trex_stl_async_client import CTRexAsyncClient from .utils import parsing_opts, text_tables, common +from .utils.common import list_intersect, list_difference, is_sub_list from .utils.text_opts import * from functools import wraps @@ -125,8 +126,26 @@ class DefaultLogger(LoggerApi): ############################ ############################# ############################ ############################# +# an event +class Event(object): + + def __init__ (self, origin, ev_type, msg): + self.origin = origin + self.ev_type = ev_type + self.msg = msg + + self.ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + + def __str__ (self): + + prefix = "[{:^}][{:^}]".format(self.origin, self.ev_type) + + return "{:<10} - {:18} - {:}".format(self.ts, prefix, format_text(self.msg, 'bold')) + + # handles different async events given to the client -class AsyncEventHandler(object): +class EventsHandler(object): + def __init__ (self, client): self.client = client @@ -136,31 +155,41 @@ class AsyncEventHandler(object): # public functions - def get_events (self): - return self.events + def get_events (self, ev_type_filter = None): + if ev_type_filter: + return [ev for ev in self.events if ev.ev_type in listify(ev_type_filter)] + else: + return [ev for ev in self.events] def clear_events (self): self.events = [] + def log_warning (self, msg, show = True): + self.__add_event_log('local', 'warning', msg, show) + + + # events called internally + def on_async_dead (self): if self.client.connected: msg = 'Lost connection to server' - self.__add_event_log(msg, 'local', True) + self.__add_event_log('local', 'info', msg, True) self.client.connected = False def on_async_alive (self): pass + - def handle_async_rx_stats_event (self, data, baseline): + def on_async_rx_stats_event (self, data, baseline): self.client.flow_stats.update(data, baseline) # handles an async stats update from the subscriber - def handle_async_stats_update(self, dump_data, baseline): + def on_async_stats_update(self, dump_data, baseline): global_stats = {} port_stats = {} @@ -189,8 +218,9 @@ class AsyncEventHandler(object): self.client.ports[port_id].port_stats.update(data, baseline) + # dispatcher for server async events (port started, port stopped and etc.) - def handle_async_event (self, type, data): + def on_async_event (self, type, data): # DP stopped show_event = False @@ -234,22 +264,55 @@ class AsyncEventHandler(object): self.__async_event_port_job_done(port_id) show_event = True - # port was stolen... + # port was acquired - maybe stolen... elif (type == 5): session_id = data['session_id'] - # false alarm, its us + port_id = int(data['port_id']) + who = data['who'] + force = data['force'] + + # if we hold the port and it was not taken by this session - show it + if port_id in self.client.get_acquired_ports() and session_id != self.client.session_id: + show_event = True + + # format the thief/us... if session_id == self.client.session_id: - return + user = 'you' + elif who == self.client.username: + user = 'another session of you' + else: + user = "'{0}'".format(who) - port_id = int(data['port_id']) - who = data['who'] + if force: + ev = "Port {0} was forcely taken by {1}".format(port_id, user) + else: + ev = "Port {0} was taken by {1}".format(port_id, user) - ev = "Port {0} was forcely taken by '{1}'".format(port_id, who) + # call the handler in case its not this session + if session_id != self.client.session_id: + self.__async_event_port_acquired(port_id, who) + + + # port was released + elif (type == 6): + port_id = int(data['port_id']) + who = data['who'] + session_id = data['session_id'] + + if session_id == self.client.session_id: + user = 'you' + elif who == self.client.username: + user = 'another session of you' + else: + user = "'{0}'".format(who) + + ev = "Port {0} was released by {1}".format(port_id, user) + + # call the handler in case its not this session + if session_id != self.client.session_id: + self.__async_event_port_released(port_id) - # call the handler - self.__async_event_port_forced_acquired(port_id) - show_event = True # server stopped elif (type == 100): @@ -263,7 +326,7 @@ class AsyncEventHandler(object): return - self.__add_event_log(ev, 'server', show_event) + self.__add_event_log('server', 'info', ev, show_event) # private functions @@ -287,28 +350,23 @@ class AsyncEventHandler(object): self.client.ports[port_id].async_event_port_resumed() - def __async_event_port_forced_acquired (self, port_id): - self.client.ports[port_id].async_event_forced_acquired() + def __async_event_port_acquired (self, port_id, who): + self.client.ports[port_id].async_event_acquired(who) + def __async_event_port_released (self, port_id): + self.client.ports[port_id].async_event_released() def __async_event_server_stopped (self): self.client.connected = False # add event to log - def __add_event_log (self, msg, ev_type, show = False): - - if ev_type == "server": - prefix = "[server]" - elif ev_type == "local": - prefix = "[local]" - - ts = time.time() - st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') - self.events.append("{:<10} - {:^8} - {:}".format(st, prefix, format_text(msg, 'bold'))) + def __add_event_log (self, origin, ev_type, msg, show = False): + event = Event(origin, ev_type, msg) + self.events.append(event) if show: - self.logger.async_log(format_text("\n\n{:^8} - {:}".format(prefix, format_text(msg, 'bold')))) + self.logger.async_log("\n\n{0}".format(str(event))) @@ -452,7 +510,7 @@ class STLClient(object): self) # async event handler manager - self.event_handler = AsyncEventHandler(self) + self.event_handler = EventsHandler(self) # async subscriber level self.async_client = CTRexAsyncClient(server, @@ -472,9 +530,10 @@ class STLClient(object): self.global_stats = trex_stl_stats.CGlobalStats(self.connection_info, self.server_version, - self.ports) + self.ports, + self.event_handler) - self.flow_stats = trex_stl_stats.CRxStats() + self.flow_stats = trex_stl_stats.CRxStats(self.ports) self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats, self.ports, @@ -482,7 +541,7 @@ class STLClient(object): # API classes - self.api_vers = [ {'type': 'core', 'major': 1, 'minor':0 } + self.api_vers = [ {'type': 'core', 'major': 1, 'minor':2 } ] self.api_h = {'core': None} @@ -977,7 +1036,8 @@ class STLClient(object): """ - return not (self.get_all_ports() == self.get_acquired_ports()) + return (self.get_all_ports() == self.get_acquired_ports()) + # is the client connected ? def is_connected (self): @@ -1117,23 +1177,39 @@ class STLClient(object): if port_obj.is_acquired()] # get all active ports (TX or pause) - def get_active_ports(self): - return [port_id - for port_id, port_obj in self.ports.items() - if port_obj.is_active()] + def get_active_ports(self, owned = True): + if owned: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_active() and port_obj.is_acquired()] + else: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_active()] # get paused ports - def get_paused_ports (self): - return [port_id - for port_id, port_obj in self.ports.items() - if port_obj.is_paused()] + def get_paused_ports (self, owned = True): + if owned: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_paused() and port_obj.is_acquired()] + else: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_paused()] + # get all TX ports - def get_transmitting_ports (self): - return [port_id - for port_id, port_obj in self.ports.items() - if port_obj.is_transmitting()] + def get_transmitting_ports (self, owned = True): + if owned: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_transmitting() and port_obj.is_acquired()] + else: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_transmitting()] # get stats @@ -1155,9 +1231,58 @@ class STLClient(object): return self.__get_stats(ports) - # return all async events - def get_events (self): - return self.event_handler.get_events() + + def get_events (self, ev_type_filter = None): + """ + returns all the logged events + + :parameters: + ev_type_filter - 'info', 'warning' or a list of those + default is no filter + + :return: + logged events + + :raises: + None + + """ + return self.event_handler.get_events(ev_type_filter) + + + def get_warnings (self): + """ + returns all the warnings logged events + + :parameters: + None + + :return: + warning logged events + + :raises: + None + + """ + return self.get_events(ev_type_filter = 'warning') + + + def get_info (self): + """ + returns all the info logged events + + :parameters: + None + + :return: + warning logged events + + :raises: + None + + """ + return self.get_events(ev_type_filter = 'info') + # get port(s) info as a list of dicts @__api_check(True) @@ -1473,8 +1598,8 @@ class STLClient(object): stream_id_list = [stream_id_list] # check streams - if not all([isinstance(stream_id, long) for stream_id in stream_id_list]): - raise STLArgumentError('stream_id_list', stream_id_list) + for stream_id in stream_id_list: + validate_type('stream_id', stream_id, int) # remove streams self.logger.pre_cmd("Removing {0} streams from port(s) {1}:".format(len(stream_id_list), ports)) @@ -1948,28 +2073,111 @@ class STLClient(object): return wrap + @__console + def ping_line (self, line): + '''pings the server''' + self.ping() + return True @__console def connect_line (self, line): - '''Connects to the TRex server''' - # define a parser + '''Connects to the TRex server and acquire ports''' parser = parsing_opts.gen_parser(self, "connect", self.connect_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, parsing_opts.FORCE) - opts = parser.parse_args(line.split()) - + opts = parser.parse_args(line.split(), default_ports = self.get_all_ports()) if opts is None: return - # call the API self.connect() - self.acquire(force = opts.force) + self.acquire(ports = opts.ports, force = opts.force) # true means print time return True + + @__console + def acquire_line (self, line): + '''Acquire ports\n''' + + # define a parser + parser = parsing_opts.gen_parser(self, + "acquire", + self.acquire_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, + parsing_opts.FORCE) + + opts = parser.parse_args(line.split(), default_ports = self.get_all_ports()) + if opts is None: + return + + # filter out all the already owned ports + ports = list_difference(opts.ports, self.get_acquired_ports()) + if not ports: + self.logger.log("acquire - all port(s) {0} are already acquired".format(opts.ports)) + return + + self.acquire(ports = ports, force = opts.force) + + # true means print time + return True + + + # + @__console + def release_line (self, line): + '''Release ports\n''' + + parser = parsing_opts.gen_parser(self, + "release", + self.release_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports()) + if opts is None: + return + + ports = list_intersect(opts.ports, self.get_acquired_ports()) + if not ports: + if not opts.ports: + self.logger.log("release - no acquired ports") + return + else: + self.logger.log("release - none of port(s) {0} are acquired".format(opts.ports)) + return + + + self.release(ports = ports) + + # true means print time + return True + + + @__console + def reacquire_line (self, line): + '''reacquire all the ports under your username which are not acquired by your session''' + + parser = parsing_opts.gen_parser(self, + "reacquire", + self.reacquire_line.__doc__) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + # find all the on-owned ports under your name + my_unowned_ports = list_difference([k for k, v in self.ports.items() if v.get_owner() == self.username], self.get_acquired_ports()) + if not my_unowned_ports: + self.logger.log("reacquire - no unowned ports under '{0}'".format(self.username)) + return + + self.acquire(ports = my_unowned_ports, force = True) + return True + + @__console def disconnect_line (self, line): self.disconnect() @@ -1978,12 +2186,24 @@ class STLClient(object): @__console def reset_line (self, line): - self.reset() + '''Reset ports - if no ports are provided all acquired ports will be reset''' + + parser = parsing_opts.gen_parser(self, + "reset", + self.reset_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True) + if opts is None: + return + + self.reset(ports = opts.ports) # true means print time return True + @__console def start_line (self, line): '''Start selected traffic on specified ports on TRex\n''' @@ -2000,15 +2220,11 @@ class STLClient(object): parsing_opts.MULTIPLIER_STRICT, parsing_opts.DRY_RUN) - opts = parser.parse_args(line.split()) - - + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True) if opts is None: return - - active_ports = list(set(self.get_active_ports()).intersection(opts.ports)) - + active_ports = list_intersect(self.get_active_ports(), opts.ports) if active_ports: if not opts.force: msg = "Port(s) {0} are active - please stop them or add '--force'\n".format(active_ports) @@ -2079,17 +2295,21 @@ class STLClient(object): self.stop_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_active_ports(), verify_acquired = True) if opts is None: return - # find the relevant ports - ports = list(set(self.get_active_ports()).intersection(opts.ports)) + # find the relevant ports + ports = list_intersect(opts.ports, self.get_active_ports()) if not ports: - self.logger.log(format_text("No active traffic on provided ports\n", 'bold')) + if not opts.ports: + self.logger.log('stop - no active ports') + else: + self.logger.log('stop - no active traffic on ports {0}'.format(opts.ports)) return + # call API self.stop(ports) # true means print time @@ -2107,15 +2327,18 @@ class STLClient(object): parsing_opts.TOTAL, parsing_opts.FORCE) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_active_ports(), verify_acquired = True) if opts is None: return - # find the relevant ports - ports = list(set(self.get_active_ports()).intersection(opts.ports)) + # find the relevant ports + ports = list_intersect(opts.ports, self.get_active_ports()) if not ports: - self.logger.log(format_text("No ports in valid state to update\n", 'bold')) + if not opts.ports: + self.logger.log('update - no active ports') + else: + self.logger.log('update - no active traffic on ports {0}'.format(opts.ports)) return self.update(ports, opts.mult, opts.total, opts.force) @@ -2132,15 +2355,22 @@ class STLClient(object): self.pause_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_transmitting_ports(), verify_acquired = True) if opts is None: return - # find the relevant ports - ports = list(set(self.get_transmitting_ports()).intersection(opts.ports)) + # check for already paused case + if opts.ports and is_sub_list(opts.ports, self.get_paused_ports()): + self.logger.log('pause - all of port(s) {0} are already paused'.format(opts.ports)) + return + # find the relevant ports + ports = list_intersect(opts.ports, self.get_transmitting_ports()) if not ports: - self.logger.log(format_text("No ports in valid state to pause\n", 'bold')) + if not opts.ports: + self.logger.log('pause - no transmitting ports') + else: + self.logger.log('pause - none of ports {0} are transmitting'.format(opts.ports)) return self.pause(ports) @@ -2157,18 +2387,21 @@ class STLClient(object): self.resume_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_paused_ports(), verify_acquired = True) if opts is None: return # find the relevant ports - ports = list(set(self.get_paused_ports()).intersection(opts.ports)) - + ports = list_intersect(opts.ports, self.get_paused_ports()) if not ports: - self.logger.log(format_text("No ports in valid state to resume\n", 'bold')) + if not opts.ports: + self.logger.log('resume - no paused ports') + else: + self.logger.log('resume - none of ports {0} are paused'.format(opts.ports)) return - return self.resume(ports) + + self.resume(ports) # true means print time return True @@ -2212,7 +2445,7 @@ class STLClient(object): mask = self.__get_mask_keys(**self.__filter_namespace_args(opts, trex_stl_stats.ALL_STATS_OPTS)) if not mask: # set to show all stats if no filter was given - mask = trex_stl_stats.ALL_STATS_OPTS + mask = trex_stl_stats.COMPACT stats_opts = common.list_intersect(trex_stl_stats.ALL_STATS_OPTS, mask) @@ -2320,15 +2553,20 @@ class STLClient(object): '''Sets port attributes ''' parser = parsing_opts.gen_parser(self, - "port", + "port_attr", self.set_port_attr_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL, parsing_opts.PROMISCUOUS_SWITCH) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True) if opts is None: return + # if no attributes - fall back to printing the status + if opts.prom is None: + self.show_stats_line("--ps --port {0}".format(' '.join(str(port) for port in opts.ports))) + return + self.set_port_attr(opts.ports, opts.prom) @@ -2371,3 +2609,54 @@ class STLClient(object): self.logger.log("") + + @__console + def get_events_line (self, line): + '''shows events recieved from server\n''' + + x = [parsing_opts.ArgumentPack(['-c','--clear'], + {'action' : "store_true", + 'default': False, + 'help': "clear the events log"}), + + parsing_opts.ArgumentPack(['-i','--info'], + {'action' : "store_true", + 'default': False, + 'help': "show info events"}), + + parsing_opts.ArgumentPack(['-w','--warn'], + {'action' : "store_true", + 'default': False, + 'help': "show warning events"}), + + ] + + + parser = parsing_opts.gen_parser(self, + "events", + self.get_events_line.__doc__, + *x) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + + ev_type_filter = [] + + if opts.info: + ev_type_filter.append('info') + + if opts.warn: + ev_type_filter.append('warning') + + if not ev_type_filter: + ev_type_filter = None + + events = self.get_events(ev_type_filter) + for ev in events: + self.logger.log(ev) + + if opts.clear: + self.clear_events() +
\ No newline at end of file diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py index b506137b..0afeff20 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py @@ -178,7 +178,6 @@ from .api import * from .trex_stl_types import * from .utils.common import get_number - class HLT_ERR(dict): def __init__(self, log = 'Unknown error', **kwargs): dict.__init__(self, {'status': 0}) @@ -201,7 +200,7 @@ def merge_kwargs(default_kwargs, user_kwargs): for key, value in user_kwargs.items(): if key in kwargs: kwargs[key] = value - elif key in ('save_to_yaml', 'save_to_pcap'): # internal debug arguments + elif key in ('save_to_yaml', 'save_to_pcap', 'pg_id'): # internal arguments kwargs[key] = value else: print("Warning: provided parameter '%s' is not supported" % key) @@ -305,7 +304,7 @@ class CStreamsPerPort(defaultdict): # save HLT args to modify streams later def save_stream_args(self, ports_list, stream_id, stream_hlt_args): - if stream_id is None: return # no stream_id, can't save TODO: remove this check ASAP + if stream_id is None: raise STLError('CStreamsPerPort: no stream_id in stream') if stream_hlt_args.get('load_profile'): return # can't modify profiles, don't save if not self.hlt_history: raise STLError('CStreamsPerPort: this object works only with HLT history, try init with hlt_history = True') if not is_integer(stream_id): raise STLError('CStreamsPerPort: stream_id should be number') @@ -313,10 +312,9 @@ class CStreamsPerPort(defaultdict): if not isinstance(ports_list, list): ports_list = [ports_list] for port in ports_list: - if stream_id in self[port]: - self[port][stream_id].update(stream_hlt_args) - else: - self[port][stream_id] = stream_hlt_args + if stream_id not in self[port]: + self[port][stream_id] = {} + self[port][stream_id].update(stream_hlt_args) def remove_stream(self, ports_list, stream_id): if not isinstance(ports_list, list): @@ -338,8 +336,11 @@ class CTRexHltApi(object): def __init__(self, verbose = 0): self.trex_client = None self.verbose = verbose - self._streams_history = {} # streams per stream_id per port in format of HLT arguments for modify later - + self._last_pg_id = 0 # pg_id acts as stream_handle + self._streams_history = {} # streams in format of HLT arguments for modify later + self._native_handle_by_pg_id = {} # pg_id -> native handle + port + self._pg_id_by_id = {} # stream_id -> pg_id + self._pg_id_by_name = {} # name -> pg_id ########################### # Session functions # @@ -371,13 +372,12 @@ class CTRexHltApi(object): try: port_list = self._parse_port_list(kwargs['port_list']) self.trex_client.acquire(ports = port_list, force = kwargs['break_locks']) + for port in port_list: + self._native_handle_by_pg_id[port] = {} except Exception as e: self.trex_client = None return HLT_ERR('Could not acquire ports %s: %s' % (port_list, e if isinstance(e, STLError) else traceback.format_exc())) - # since only supporting single TRex at the moment, 1:1 map - port_handle = self.trex_client.get_acquired_ports() - # arrived here, all desired ports were successfully acquired if kwargs['reset']: # remove all port traffic configuration from TRex @@ -389,7 +389,7 @@ class CTRexHltApi(object): return HLT_ERR('Error in reset traffic: %s' % e if isinstance(e, STLError) else traceback.format_exc()) self._streams_history = CStreamsPerPort(hlt_history = True) - return HLT_OK(port_handle = port_handle) + return HLT_OK(port_handle = dict([(port_id, port_id) for port_id in port_list])) def cleanup_session(self, **user_kwargs): kwargs = merge_kwargs(cleanup_session_kwargs, user_kwargs) @@ -446,23 +446,9 @@ class CTRexHltApi(object): kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs) stream_id = kwargs['stream_id'] mode = kwargs['mode'] - if type(stream_id) is list: - if len(stream_id) > 1: - streams_per_port = CStreamsPerPort() - for each_stream_id in stream_id: - user_kwargs[stream_id] = each_stream_id - res = self.traffic_config(**user_kwargs) - if type(res) is HLT_ERR: - return res - streams_per_port.add_streams_from_res(res) - if mode == 'create': - return HLT_OK(stream_id = streams_per_port) - else: - return HLT_OK() - else: - stream_id = stream_id[0] - + pg_id = None port_handle = port_list = self._parse_port_list(kwargs['port_handle']) + ALLOWED_MODES = ['create', 'modify', 'remove', 'enable', 'disable', 'reset'] if mode not in ALLOWED_MODES: return HLT_ERR('Mode must be one of the following values: %s' % ALLOWED_MODES) @@ -480,7 +466,7 @@ class CTRexHltApi(object): if mode == 'remove': if stream_id is None: return HLT_ERR('Please specify stream_id to remove.') - if type(stream_id) is str and stream_id == 'all': + if stream_id == 'all': try: self.trex_client.remove_all_streams(port_handle) for port in port_handle: @@ -504,14 +490,20 @@ class CTRexHltApi(object): # self._streams_history[stream_id].update(kwargs) # <- the modification if mode == 'modify': # we remove stream and create new one with same stream_id - stream_id = kwargs.get('stream_id') - if stream_id is None: + pg_id = kwargs.get('stream_id') + if pg_id is None: return HLT_ERR('Please specify stream_id to modify.') if len(port_handle) > 1: for port in port_handle: - user_kwargs[port_handle] = port - res = self.traffic_config(**user_kwargs) # recurse per port, each port can have different stream with such id + try: + user_kwargs['port_handle'] = port + res = self.traffic_config(**user_kwargs) + if res['status'] == 0: + return HLT_ERR('Error during modify of stream: %s' % res['log']) + except Exception as e: + return HLT_ERR('Could not remove stream(s) %s from port(s) %s: %s' % (stream_id, port_handle, e if isinstance(e, STLError) else traceback.format_exc())) + return HLT_OK() else: if type(port_handle) is list: port = port_handle[0] @@ -519,35 +511,37 @@ class CTRexHltApi(object): port = port_handle if port not in self._streams_history: return HLT_ERR('Port %s was not used/acquired' % port) - if stream_id not in self._streams_history[port]: + if pg_id not in self._streams_history[port]: return HLT_ERR('This stream_id (%s) was not used before at port %s, please create new.' % (stream_id, port)) - kwargs.update(self._streams_history[port][stream_id]) - kwargs.update(user_kwargs) + new_kwargs = {} + new_kwargs.update(self._streams_history[port][pg_id]) + new_kwargs.update(user_kwargs) + user_kwargs = new_kwargs try: - self.trex_client.remove_streams(stream_id, port_handle) + self._remove_stream(pg_id, [port]) except Exception as e: return HLT_ERR('Could not remove stream(s) %s from port(s) %s: %s' % (stream_id, port_handle, e if isinstance(e, STLError) else traceback.format_exc())) if mode == 'create' or mode == 'modify': # create a new stream with desired attributes, starting by creating packet - streams_per_port = CStreamsPerPort() if is_true(kwargs['bidirectional']): # two streams with opposite directions del user_kwargs['bidirectional'] + stream_per_port = {} save_to_yaml = user_kwargs.get('save_to_yaml') bidirect_err = 'When using bidirectional flag, ' if len(port_handle) != 1: - return HLT_ERR(bidirect_err + 'port_handle1 should be single port handle.') + return HLT_ERR(bidirect_err + 'port_handle should be single port handle.') + port_handle = port_handle[0] port_handle2 = kwargs['port_handle2'] if (type(port_handle2) is list and len(port_handle2) > 1) or port_handle2 is None: return HLT_ERR(bidirect_err + 'port_handle2 should be single port handle.') try: if save_to_yaml and type(save_to_yaml) is str: user_kwargs['save_to_yaml'] = save_to_yaml.replace('.yaml', '_bi1.yaml') - user_kwargs['port_handle'] = port_handle[0] res1 = self.traffic_config(**user_kwargs) if res1['status'] == 0: raise STLError('Could not create bidirectional stream 1: %s' % res1['log']) - streams_per_port.add_streams_from_res(res1) + stream_per_port[port_handle] = res1['stream_id'] kwargs['direction'] = 1 - kwargs['direction'] # not correct_direction(user_kwargs, kwargs) if save_to_yaml and type(save_to_yaml) is str: @@ -556,37 +550,38 @@ class CTRexHltApi(object): res2 = self.traffic_config(**user_kwargs) if res2['status'] == 0: raise STLError('Could not create bidirectional stream 2: %s' % res2['log']) - streams_per_port.add_streams_from_res(res2) + stream_per_port[port_handle2] = res2['stream_id'] except Exception as e: return HLT_ERR('Could not generate bidirectional traffic: %s' % e if isinstance(e, STLError) else traceback.format_exc()) if mode == 'create': - return HLT_OK(stream_id = streams_per_port) + return HLT_OK(stream_id = stream_per_port) else: return HLT_OK() try: + if not pg_id: + pg_id = self._get_available_pg_id() if kwargs['load_profile']: stream_obj = STLProfile.load_py(kwargs['load_profile'], direction = kwargs['direction']) else: + user_kwargs['pg_id'] = pg_id stream_obj = STLHltStream(**user_kwargs) except Exception as e: return HLT_ERR('Could not create stream: %s' % e if isinstance(e, STLError) else traceback.format_exc()) # try adding the stream per ports try: - stream_id_arr = self.trex_client.add_streams(streams = stream_obj, - ports = port_handle) - if type(stream_id_arr) is not list: - stream_id_arr = [stream_id_arr] for port in port_handle: - self._streams_history.save_stream_args(port_handle, stream_id_arr[0], user_kwargs) + stream_id_arr = self.trex_client.add_streams(streams = stream_obj, + ports = port) + self._streams_history.save_stream_args(port, pg_id, user_kwargs) + if type(stream_id_arr) is not list: + stream_id_arr = [stream_id_arr] + self._native_handle_by_pg_id[port][pg_id] = stream_id_arr except Exception as e: return HLT_ERR('Could not add stream to ports: %s' % e if isinstance(e, STLError) else traceback.format_exc()) if mode == 'create': - if len(stream_id_arr) == 1: - return HLT_OK(stream_id = dict((port, stream_id_arr[0]) for port in port_handle)) - else: - return HLT_OK(stream_id = dict((port, stream_id_arr) for port in port_handle)) + return HLT_OK(stream_id = pg_id) else: return HLT_OK() @@ -652,42 +647,64 @@ class CTRexHltApi(object): kwargs = merge_kwargs(traffic_stats_kwargs, user_kwargs) mode = kwargs['mode'] port_handle = kwargs['port_handle'] + if type(port_handle) is not list: + port_handle = [port_handle] ALLOWED_MODES = ['aggregate', 'streams', 'all'] if mode not in ALLOWED_MODES: return HLT_ERR("'mode' must be one of the following values: %s" % ALLOWED_MODES) - if mode == 'streams': - return HLT_ERR("mode 'streams' not implemented'") - if mode in ('all', 'aggregate'): - hlt_stats_dict = {} - try: - stats = self.trex_client.get_stats(port_handle) - except Exception as e: - return HLT_ERR('Could not retrieve stats: %s' % e if isinstance(e, STLError) else traceback.format_exc()) - for port_id, stat_dict in stats.items(): - if is_integer(port_id): - hlt_stats_dict[port_id] = { - 'aggregate': { - 'tx': { - 'pkt_bit_rate': stat_dict.get('tx_bps'), - 'pkt_byte_count': stat_dict.get('obytes'), - 'pkt_count': stat_dict.get('opackets'), - 'pkt_rate': stat_dict.get('tx_pps'), - 'total_pkt_bytes': stat_dict.get('obytes'), - 'total_pkt_rate': stat_dict.get('tx_pps'), - 'total_pkts': stat_dict.get('opackets'), - }, - 'rx': { - 'pkt_bit_rate': stat_dict.get('rx_bps'), - 'pkt_byte_count': stat_dict.get('ibytes'), - 'pkt_count': stat_dict.get('ipackets'), - 'pkt_rate': stat_dict.get('rx_pps'), - 'total_pkt_bytes': stat_dict.get('ibytes'), - 'total_pkt_rate': stat_dict.get('rx_pps'), - 'total_pkts': stat_dict.get('ipackets'), + hlt_stats_dict = dict([(port, {}) for port in port_handle]) + try: + stats = self.trex_client.get_stats(port_handle) + if mode in ('all', 'aggregate'): + for port_id in port_handle: + port_stats = stats[port_id] + if is_integer(port_id): + hlt_stats_dict[port_id]['aggregate'] = { + 'tx': { + 'pkt_bit_rate': port_stats.get('tx_bps', 0), + 'pkt_byte_count': port_stats.get('obytes', 0), + 'pkt_count': port_stats.get('opackets', 0), + 'pkt_rate': port_stats.get('tx_pps', 0), + 'total_pkt_bytes': port_stats.get('obytes', 0), + 'total_pkt_rate': port_stats.get('tx_pps', 0), + 'total_pkts': port_stats.get('opackets', 0), + }, + 'rx': { + 'pkt_bit_rate': port_stats.get('rx_bps', 0), + 'pkt_byte_count': port_stats.get('ibytes', 0), + 'pkt_count': port_stats.get('ipackets', 0), + 'pkt_rate': port_stats.get('rx_pps', 0), + 'total_pkt_bytes': port_stats.get('ibytes', 0), + 'total_pkt_rate': port_stats.get('rx_pps', 0), + 'total_pkts': port_stats.get('ipackets', 0), + } } - } - } - return HLT_OK(hlt_stats_dict) + if mode in ('all', 'streams'): + for pg_id, pg_stats in stats['flow_stats'].items(): + for port_id in port_handle: + if 'stream' not in hlt_stats_dict[port_id]: + hlt_stats_dict[port_id]['stream'] = {} + hlt_stats_dict[port_id]['stream'][pg_id] = { + 'tx': { + 'total_pkts': pg_stats['tx_pkts'].get(port_id, 0), + 'total_pkt_bytes': pg_stats['tx_bytes'].get(port_id, 0), + 'total_pkts_bytes': pg_stats['tx_bytes'].get(port_id, 0), + 'total_pkt_bit_rate': pg_stats['tx_bps'].get(port_id, 0), + 'total_pkt_rate': pg_stats['tx_pps'].get(port_id, 0), + 'line_rate_percentage': pg_stats['tx_line_util'].get(port_id, 0), + }, + 'rx': { + 'total_pkts': pg_stats['rx_pkts'].get(port_id, 0), + 'total_pkt_bytes': pg_stats['rx_bytes'].get(port_id, 0), + 'total_pkts_bytes': pg_stats['rx_bytes'].get(port_id, 0), + 'total_pkt_bit_rate': pg_stats['rx_bps'].get(port_id, 0), + 'total_pkt_rate': pg_stats['rx_pps'].get(port_id, 0), + 'line_rate_percentage': pg_stats['rx_line_util'].get(port_id, 0), + }, + } + except Exception as e: + return HLT_ERR('Could not retrieve stats: %s' % e if isinstance(e, STLError) else traceback.format_exc()) + return HLT_OK(hlt_stats_dict) # timeout = maximal time to wait def wait_on_traffic(self, port_handle = None, timeout = 60): @@ -700,16 +717,31 @@ class CTRexHltApi(object): # Private functions # ########################### + def _get_available_pg_id(self): + pg_id = self._last_pg_id + used_pg_ids = self.trex_client.get_stats()['flow_stats'].keys() + for i in range(65535): + pg_id += 1 + if pg_id not in used_pg_ids: + self._last_pg_id = pg_id + return pg_id + if pg_id == 65535: + pg_id = 0 + raise STLError('Could not find free pg_id in range [1, 65535].') + # remove streams from given port(s). # stream_id can be: # * int - exact stream_id value # * list - list of stream_id values or strings (see below) # * string - exact stream_id value, mix of ranges/list separated by comma: 2, 4-13 def _remove_stream(self, stream_id, port_handle): - if get_number(stream_id) is not None: # exact value of int or str - self.trex_client.remove_streams(get_number(stream_id), port_handle) # actual remove + stream_num = get_number(stream_id) + if stream_num is not None: # exact value of int or str for port in port_handle: - del self._streams_history[port][get_number(stream_id)] + native_handles = self._native_handle_by_pg_id[port][stream_num] + self.trex_client.remove_streams(native_handles, port) # actual remove + del self._native_handle_by_pg_id[port][stream_num] + del self._streams_history[port][stream_num] return if type(stream_id) is list: # list of values/strings for each_stream_id in stream_id: @@ -733,7 +765,7 @@ class CTRexHltApi(object): for each_stream_id in xrange(stream_id_min, stream_id_max + 1): self._remove_stream(each_stream_id, port_handle) # recurse return - raise STLError('_remove_stream: wrong param %s' % stream_id) + raise STLError('_remove_stream: wrong stream_id param %s' % stream_id) @staticmethod def _parse_port_list(port_list): @@ -812,32 +844,33 @@ def STLHltStream(**user_kwargs): # packet generation packet = generate_packet(**user_kwargs) + + # stream generation try: rate_types_dict = {'rate_pps': 'pps', 'rate_bps': 'bps_L2', 'rate_percent': 'percentage'} rate_stateless = {rate_types_dict[rate_key]: float(kwargs[rate_key])} transmit_mode = kwargs['transmit_mode'] pkts_per_burst = kwargs['pkts_per_burst'] if transmit_mode == 'continuous': - transmit_mode_class = STLTXCont(**rate_stateless) + transmit_mode_obj = STLTXCont(**rate_stateless) elif transmit_mode == 'single_burst': - transmit_mode_class = STLTXSingleBurst(total_pkts = pkts_per_burst, **rate_stateless) + transmit_mode_obj = STLTXSingleBurst(total_pkts = pkts_per_burst, **rate_stateless) elif transmit_mode == 'multi_burst': - transmit_mode_class = STLTXMultiBurst(total_pkts = pkts_per_burst, count = int(kwargs['burst_loop_count']), + transmit_mode_obj = STLTXMultiBurst(total_pkts = pkts_per_burst, count = int(kwargs['burst_loop_count']), ibg = kwargs['inter_burst_gap'], **rate_stateless) else: raise STLError('transmit_mode %s not supported/implemented') except Exception as e: - raise STLError('Could not create transmit_mode class %s: %s' % (transmit_mode, e if isinstance(e, STLError) else traceback.format_exc())) + raise STLError('Could not create transmit_mode object %s: %s' % (transmit_mode, e if isinstance(e, STLError) else traceback.format_exc())) - # stream generation try: + pg_id = kwargs.get('pg_id') stream = STLStream(packet = packet, random_seed = 1 if is_true(kwargs['consistent_random']) else 0, #enabled = True, #self_start = True, - mode = transmit_mode_class, - stream_id = kwargs['stream_id'], - name = kwargs['name'], + flow_stats = STLFlowStats(pg_id) if pg_id else None, + mode = transmit_mode_obj, ) except Exception as e: raise STLError('Could not create stream: %s' % e if isinstance(e, STLError) else traceback.format_exc()) @@ -848,8 +881,12 @@ def STLHltStream(**user_kwargs): stream.dump_to_yaml(debug_filename) return stream +packet_cache = LRU_cache(maxlen = 20) + def generate_packet(**user_kwargs): correct_macs(user_kwargs) + if repr(user_kwargs) in packet_cache: + return packet_cache[repr(user_kwargs)] kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs) correct_sizes(kwargs) # we are producing the packet - 4 bytes fcs correct_direction(kwargs, kwargs) @@ -868,8 +905,12 @@ def generate_packet(**user_kwargs): kwargs['mac_dst'] = None kwargs['mac_src_mode'] = 'fixed' kwargs['mac_dst_mode'] = 'fixed' - - l2_layer = Ether(src = kwargs['mac_src'], dst = kwargs['mac_dst']) + ethernet_kwargs = {} + if kwargs['mac_src']: + ethernet_kwargs['src'] = kwargs['mac_src'] + if kwargs['mac_dst']: + ethernet_kwargs['dst'] = kwargs['mac_dst'] + l2_layer = Ether(**ethernet_kwargs) # Eth VM, change only 32 lsb if kwargs['mac_src_mode'] != 'fixed': @@ -1026,7 +1067,7 @@ def generate_packet(**user_kwargs): if ip_tos < 0 or ip_tos > 255: raise STLError('TOS %s is not in range 0-255' % ip_tos) l3_layer = IP(tos = ip_tos, - len = kwargs['l3_length'], + #len = kwargs['l3_length'], don't let user create corrupt packets id = kwargs['ip_id'], frag = kwargs['ip_fragment_offset'], ttl = kwargs['ip_ttl'], @@ -1475,6 +1516,7 @@ def generate_packet(**user_kwargs): debug_filename = kwargs.get('save_to_pcap') if type(debug_filename) is str: pkt.dump_pkt_to_pcap(debug_filename) + packet_cache[repr(user_kwargs)] = pkt return pkt def get_TOS(user_kwargs, kwargs): @@ -1542,7 +1584,7 @@ def correct_direction(user_kwargs, kwargs): # we produce packets without fcs, so need to reduce produced sizes def correct_sizes(kwargs): - for arg in kwargs.keys(): - if is_integer(arg): + for arg, value in kwargs.items(): + if is_integer(value): if arg.endswith(('_length', '_size', '_size_min', '_size_max', '_length_min', '_length_max')): kwargs[arg] -= 4 diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py index 89ad2663..5cf94bda 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py @@ -64,6 +64,8 @@ class Port(object): self.tx_stopped_ts = None self.has_rx_streams = False + self.owner = '' + def err(self, msg): return RC_ERR("port {0} : {1}\n".format(self.port_id, msg)) @@ -74,6 +76,9 @@ class Port(object): def get_speed_bps (self): return (self.info['speed'] * 1000 * 1000 * 1000) + def get_formatted_speed (self): + return "{0} Gbps".format(self.info['speed']) + # take the port def acquire(self, force = False): params = {"port_id": self.port_id, @@ -94,9 +99,12 @@ class Port(object): "handler": self.handler} rc = self.transmit("release", params) - self.handler = None - + if rc.good(): + + self.handler = None + self.owner = '' + return self.ok() else: return self.err(rc.err()) @@ -113,6 +121,11 @@ class Port(object): def is_paused (self): return (self.state == self.STATE_PAUSE) + def get_owner (self): + if self.is_acquired(): + return self.user + else: + return self.owner def sync(self): params = {"port_id": self.port_id} @@ -137,6 +150,7 @@ class Port(object): else: raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state)) + self.owner = rc.data()['owner'] self.next_available_id = int(rc.data()['max_stream_id']) + 1 @@ -231,10 +245,11 @@ class Port(object): if single_rc.rc: stream_id = batch[i].params['stream_id'] next_id = batch[i].params['stream']['next_stream_id'] - self.streams[stream_id] = {'next_id' : next_id, - 'pkt' : streams_list[i].get_pkt(), - 'mode' : streams_list[i].get_mode(), - 'rate' : streams_list[i].get_rate()} + self.streams[stream_id] = {'next_id' : next_id, + 'pkt' : streams_list[i].get_pkt(), + 'mode' : streams_list[i].get_mode(), + 'rate' : streams_list[i].get_rate(), + 'has_flow_stats' : streams_list[i].has_flow_stats()} ret.add(RC_OK(data = stream_id)) @@ -280,12 +295,12 @@ class Port(object): for i, single_rc in enumerate(rc): if single_rc: id = batch[i].params['stream_id'] - del self.streams[stream_id] + del self.streams[id] self.state = self.STATE_STREAMS if (len(self.streams) > 0) else self.STATE_IDLE # recheck if any RX stats streams present on the port - self.has_rx_streams = any([stream.has_flow_stats() for stream in self.streams]) + self.has_rx_streams = any([stream['has_flow_stats'] for stream in self.streams.values()]) return self.ok() if rc else self.err(rc.err()) @@ -670,6 +685,10 @@ class Port(object): if not self.is_acquired(): self.state = self.STATE_TX - def async_event_forced_acquired (self): + def async_event_acquired (self, who): self.handler = None + self.owner = who + + def async_event_released (self): + self.owner = '' diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py index 1d89a599..11e80b9a 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py @@ -22,6 +22,7 @@ from .trex_stl_streams import * from .utils import parsing_opts from .trex_stl_client import STLClient from .utils import pcap +from trex_stl_lib.trex_stl_packet_builder_scapy import RawPcapReader, RawPcapWriter, hexdump from yaml import YAMLError @@ -291,13 +292,13 @@ class STLSim(object): return - print("Mering cores output to a single pcap file...\n") + if not self.silent: + print("Mering cores output to a single pcap file...\n") inputs = ["{0}-{1}".format(self.outfile, index) for index in range(0, self.dp_core_count)] pcap.merge_cap_files(inputs, self.outfile, delete_src = True) - def is_valid_file(filename): if not os.path.isfile(filename): raise argparse.ArgumentTypeError("The file '%s' does not exist" % filename) @@ -421,6 +422,11 @@ def setParserOptions(): action = "store_true", default = False) + group.add_argument("--test_multi_core", + help = "runs the profile with c=1-8", + action = "store_true", + default = False) + return parser @@ -435,6 +441,110 @@ def validate_args (parser, options): parser.error("limit cannot be lower than number of DP cores") +# a more flexible check +def compare_caps (cap1, cap2, max_diff_sec = (5 * 1e-6)): + pkts1 = list(RawPcapReader(cap1)) + pkts2 = list(RawPcapReader(cap2)) + + if len(pkts1) != len(pkts2): + print('{0} contains {1} packets vs. {2} contains {3} packets'.format(cap1, len(pkts1), cap2, len(pkts2))) + return False + + # to be less strict we define equality if all packets from cap1 exists and in cap2 + # and vice versa + # 'exists' means the same packet with abs(TS1-TS2) < 5nsec + # its O(n^2) but who cares, right ? + for i, pkt1 in enumerate(pkts1): + ts1 = float(pkt1[1][0]) + (float(pkt1[1][1]) / 1e6) + found = None + for j, pkt2 in enumerate(pkts2): + ts2 = float(pkt2[1][0]) + (float(pkt2[1][1]) / 1e6) + + if abs(ts1-ts2) > max_diff_sec: + break + + if pkt1[0] == pkt2[0]: + found = j + break + + + if found is None: + print(format_text("cannot find packet #{0} from {1} in {2}\n".format(i, cap1, cap2), 'bold')) + return False + else: + del pkts2[found] + + return True + + + + +# a more strict comparsion 1 <--> 1 +def compare_caps_strict (cap1, cap2, max_diff_sec = (5 * 1e-6)): + pkts1 = list(RawPcapReader(cap1)) + pkts2 = list(RawPcapReader(cap2)) + + if len(pkts1) != len(pkts2): + print('{0} contains {1} packets vs. {1} contains {2} packets'.format(cap1, len(pkts1), cap2, len(pkts2))) + return False + + # a strict check + for pkt1, pkt2, i in zip(pkts1, pkts2, range(1, len(pkts1))): + ts1 = float(pkt1[1][0]) + (float(pkt1[1][1]) / 1e6) + ts2 = float(pkt2[1][0]) + (float(pkt2[1][1]) / 1e6) + + if abs(ts1-ts2) > 0.000005: # 5 nsec + print(format_text("TS error: cap files '{0}', '{1}' differ in cap #{2} - '{3}' vs. '{4}'\n".format(cap1, cap2, i, ts1, ts2), 'bold')) + return False + + if pkt1[0] != pkt2[0]: + print(format_text("RAW error: cap files '{0}', '{1}' differ in cap #{2}\n".format(cap1, cap2, i), 'bold')) + print(hexdump(pkt1[0])) + print("") + print(hexdump(pkt2[0])) + print("") + return False + + return True + +# +def test_multi_core (r, options): + + for core_count in [1, 2, 4, 6, 8]: + r.run(input_list = options.input_file, + outfile = '{0}.cap'.format(core_count), + dp_core_count = core_count, + is_debug = (not options.release), + pkt_limit = options.limit, + mult = options.mult, + duration = options.duration, + mode = 'none', + silent = True, + tunables = options.tunables) + + print("") + + print(format_text("comparing 2 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '2.cap') + if rc: + print("[Passed]\n") + + print(format_text("comparing 4 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '4.cap') + if rc: + print("[Passed]\n") + + print(format_text("comparing 6 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '6.cap') + if rc: + print("[Passed]\n") + + print(format_text("comparing 8 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '8.cap') + if rc: + print("[Passed]\n") + + def main (args = None): parser = setParserOptions() options = parser.parse_args(args = args) @@ -455,23 +565,28 @@ def main (args = None): mode = 'native' elif options.pkt: mode = 'pkt' + elif options.test_multi_core: + mode = 'test_multi_core' else: mode = 'none' try: r = STLSim(bp_sim_path = options.bp_sim_path, port_id = options.port_id) - r.run(input_list = options.input_file, - outfile = options.output_file, - dp_core_count = options.dp_core_count, - dp_core_index = options.dp_core_index, - is_debug = (not options.release), - pkt_limit = options.limit, - mult = options.mult, - duration = options.duration, - mode = mode, - silent = options.silent, - tunables = options.tunables) + if mode == 'test_multi_core': + test_multi_core(r, options) + else: + r.run(input_list = options.input_file, + outfile = options.output_file, + dp_core_count = options.dp_core_count, + dp_core_index = options.dp_core_index, + is_debug = (not options.release), + pkt_limit = options.limit, + mult = options.mult, + duration = options.duration, + mode = mode, + silent = options.silent, + tunables = options.tunables) except KeyboardInterrupt as e: print("\n\n*** Caught Ctrl + C... Exiting...\n\n") diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py index a4bb64db..42bef360 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py @@ -2,6 +2,7 @@ from .utils import text_tables from .utils.text_opts import format_text, format_threshold, format_num +from .trex_stl_types import StatNotAvailable from collections import namedtuple, OrderedDict, deque import sys @@ -16,11 +17,13 @@ import pprint GLOBAL_STATS = 'g' PORT_STATS = 'p' +PORT_GRAPH = 'pg' PORT_STATUS = 'ps' STREAMS_STATS = 's' -ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS] +ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS, PORT_GRAPH] COMPACT = [GLOBAL_STATS, PORT_STATS] +GRAPH_PORT_COMPACT = [GLOBAL_STATS, PORT_GRAPH] SS_COMPAT = [GLOBAL_STATS, STREAMS_STATS] ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) @@ -88,6 +91,33 @@ def calculate_diff_raw (samples): return total +# a simple object to keep a watch over a field +class WatchedField(object): + + def __init__ (self, name, suffix, high_th, low_th, events_handler): + self.name = name + self.suffix = suffix + self.high_th = high_th + self.low_th = low_th + self.events_handler = events_handler + + self.hot = False + self.current = None + + def update (self, value): + if value is None: + return + + if value > self.high_th and not self.hot: + self.events_handler.log_warning("{0} is high: {1}{2}".format(self.name, value, self.suffix)) + self.hot = True + + if value < self.low_th and self.hot: + self.hot = False + + self.current = value + + class CTRexInfoGenerator(object): """ @@ -107,11 +137,15 @@ class CTRexInfoGenerator(object): elif statistic_type == PORT_STATS: return self._generate_port_stats(port_id_list) + elif statistic_type == PORT_GRAPH: + return self._generate_port_graph(port_id_list) + elif statistic_type == PORT_STATUS: return self._generate_port_status(port_id_list) elif statistic_type == STREAMS_STATS: return self._generate_streams_stats() + else: # ignore by returning empty object return {} @@ -163,70 +197,53 @@ class CTRexInfoGenerator(object): return {"streams_statistics": ExportableStats(sstats_data, stats_table)} - - - per_stream_stats = OrderedDict([("owner", []), - ("state", []), - ("--", []), - ("Tx bps L2", []), - ("Tx bps L1", []), - ("Tx pps", []), - ("Line Util.", []), - - ("---", []), - ("Rx bps", []), - ("Rx pps", []), - - ("----", []), - ("opackets", []), - ("ipackets", []), - ("obytes", []), - ("ibytes", []), - ("tx-bytes", []), - ("rx-bytes", []), - ("tx-pkts", []), - ("rx-pkts", []), - - ("-----", []), - ("oerrors", []), - ("ierrors", []), - - ] - ) - - total_stats = CPortStats(None) - - for port_obj in relevant_ports: - # fetch port data - port_stats = port_obj.generate_port_stats() - - total_stats += port_obj.port_stats - - # populate to data structures - return_stats_data[port_obj.port_id] = port_stats - self.__update_per_field_dict(port_stats, per_field_stats) - - total_cols = len(relevant_ports) - header = ["port"] + [port.port_id for port in relevant_ports] + @staticmethod + def _get_rational_block_char(value, range_start, interval): + # in Konsole, utf-8 is sometimes printed with artifacts, return ascii for now + #return 'X' if value >= range_start + float(interval) / 2 else ' ' + + if sys.__stdout__.encoding != 'UTF-8': + return 'X' if value >= range_start + float(interval) / 2 else ' ' + + value -= range_start + ratio = float(value) / interval + if ratio <= 0.0625: + return u' ' # empty block + if ratio <= 0.1875: + return u'\u2581' # 1/8 + if ratio <= 0.3125: + return u'\u2582' # 2/4 + if ratio <= 0.4375: + return u'\u2583' # 3/8 + if ratio <= 0.5625: + return u'\u2584' # 4/8 + if ratio <= 0.6875: + return u'\u2585' # 5/8 + if ratio <= 0.8125: + return u'\u2586' # 6/8 + if ratio <= 0.9375: + return u'\u2587' # 7/8 + return u'\u2588' # full block + + def _generate_port_graph(self, port_id_list): + relevant_port = self.__get_relevant_ports(port_id_list)[0] + hist_len = len(relevant_port.port_stats.history) + hist_maxlen = relevant_port.port_stats.history.maxlen + util_tx_hist = [0] * (hist_maxlen - hist_len) + [round(relevant_port.port_stats.history[i]['tx_percentage']) for i in range(hist_len)] + util_rx_hist = [0] * (hist_maxlen - hist_len) + [round(relevant_port.port_stats.history[i]['rx_percentage']) for i in range(hist_len)] - if (total_cols > 1): - self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats) - header += ['total'] - total_cols += 1 stats_table = text_tables.TRexTextTable() - stats_table.set_cols_align(["l"] + ["r"] * total_cols) - stats_table.set_cols_width([10] + [17] * total_cols) - stats_table.set_cols_dtype(['t'] + ['t'] * total_cols) - - stats_table.add_rows([[k] + v - for k, v in per_field_stats.items()], - header=False) + stats_table.header([' Util(%)', 'TX', 'RX']) + stats_table.set_cols_align(['c', 'c', 'c']) + stats_table.set_cols_width([8, hist_maxlen, hist_maxlen]) + stats_table.set_cols_dtype(['t', 't', 't']) - stats_table.header(header) - - return {"streams_statistics": ExportableStats(return_stats_data, stats_table)} + for y in range(95, -1, -5): + stats_table.add_row([y, ''.join([self._get_rational_block_char(util_tx, y, 5) for util_tx in util_tx_hist]), + ''.join([self._get_rational_block_char(util_rx, y, 5) for util_rx in util_rx_hist])]) + return {"port_graph": ExportableStats({}, stats_table)} def _generate_port_stats(self, port_id_list): relevant_ports = self.__get_relevant_ports(port_id_list) @@ -234,6 +251,7 @@ class CTRexInfoGenerator(object): return_stats_data = {} per_field_stats = OrderedDict([("owner", []), ("state", []), + ("speed", []), ("--", []), ("Tx bps L2", []), ("Tx bps L1", []), @@ -374,7 +392,8 @@ class CTRexInfoGenerator(object): # display only the first FOUR options, by design if len(ports) > 4: - self.logger.log(format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta')) + #self.logger is not defined + #self.logger.log(format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta')) ports = ports[:4] return ports @@ -400,7 +419,7 @@ class CTRexStats(object): self.reference_stats = {} self.latest_stats = {} self.last_update_ts = time.time() - self.history = deque(maxlen = 10) + self.history = deque(maxlen = 47) self.lock = threading.Lock() self.has_baseline = False @@ -444,6 +463,7 @@ class CTRexStats(object): def clear_stats(self): self.reference_stats = copy.deepcopy(self.latest_stats) + self.history.clear() def invalidate (self): @@ -469,19 +489,18 @@ class CTRexStats(object): def get(self, field, format=False, suffix=""): value = self._get(self.latest_stats, field) if value == None: - return "N/A" + return 'N/A' return value if not format else format_num(value, suffix) def get_rel(self, field, format=False, suffix=""): - ref_value = self._get(self.reference_stats, field) latest_value = self._get(self.latest_stats, field) # latest value is an aggregation - must contain the value if latest_value == None: - return "N/A" + return 'N/A' if ref_value == None: ref_value = 0 @@ -493,7 +512,7 @@ class CTRexStats(object): # get trend for a field def get_trend (self, field, use_raw = False, percision = 10.0): - if not field in self.latest_stats: + if field not in self.latest_stats: return 0 # not enough history - no trend @@ -506,7 +525,7 @@ class CTRexStats(object): # must lock, deque is not thread-safe for iteration with self.lock: - field_samples = [sample[field] for sample in self.history] + field_samples = [sample[field] for sample in list(self.history)[-5:]] if use_raw: return calculate_diff_raw(field_samples) @@ -518,7 +537,12 @@ class CTRexStats(object): v = self.get_trend(field, use_raw) value = abs(v) - arrow = u'\u25b2' if v > 0 else u'\u25bc' + + # use arrows if utf-8 is supported + if sys.__stdout__.encoding == 'UTF-8': + arrow = u'\u25b2' if v > 0 else u'\u25bc' + else: + arrow = '' if sys.version_info < (3,0): arrow = arrow.encode('utf-8') @@ -553,17 +577,24 @@ class CTRexStats(object): class CGlobalStats(CTRexStats): - def __init__(self, connection_info, server_version, ports_dict_ref): + def __init__(self, connection_info, server_version, ports_dict_ref, events_handler): super(CGlobalStats, self).__init__() + self.connection_info = connection_info - self.server_version = server_version - self._ports_dict = ports_dict_ref + self.server_version = server_version + self._ports_dict = ports_dict_ref + self.events_handler = events_handler + + self.watched_cpu_util = WatchedField('CPU util.', '%', 85, 60, events_handler) + self.watched_rx_cpu_util = WatchedField('RX core util.', '%', 85, 60, events_handler) def get_stats (self): stats = {} # absolute - stats['cpu_util'] = self.get("m_cpu_util") + stats['cpu_util'] = self.get("m_cpu_util") + stats['rx_cpu_util'] = self.get("m_rx_cpu_util") + stats['tx_bps'] = self.get("m_tx_bps") stats['tx_pps'] = self.get("m_tx_pps") @@ -589,6 +620,9 @@ class CGlobalStats(CTRexStats): # simple... self.latest_stats = snapshot + self.watched_cpu_util.update(snapshot.get('m_cpu_util')) + self.watched_rx_cpu_util.update(snapshot.get('m_rx_cpu_util')) + return True @@ -601,6 +635,9 @@ class CGlobalStats(CTRexStats): ("cpu_util", "{0}% {1}".format( format_threshold(self.get("m_cpu_util"), [85, 100], [0, 85]), self.get_trend_gui("m_cpu_util", use_raw = True))), + ("rx_cpu_util", "{0}% {1}".format( format_threshold(self.get("m_rx_cpu_util"), [85, 100], [0, 85]), + self.get_trend_gui("m_rx_cpu_util", use_raw = True))), + (" ", ""), ("total_tx_L2", "{0} {1}".format( self.get("m_tx_bps", format=True, suffix="b/sec"), @@ -694,11 +731,27 @@ class CPortStats(CTRexStats): # L1 bps bps = snapshot.get("m_total_tx_bps") pps = snapshot.get("m_total_tx_pps") + rx_bps = snapshot.get("m_total_rx_bps") + rx_pps = snapshot.get("m_total_rx_pps") + ts_diff = 0.5 # TODO: change this to real ts diff from server bps_L1 = calc_bps_L1(bps, pps) + bps_rx_L1 = calc_bps_L1(rx_bps, rx_pps) snapshot['m_total_tx_bps_L1'] = bps_L1 snapshot['m_percentage'] = (bps_L1 / self._port_obj.get_speed_bps()) * 100 + # TX line util not smoothed + diff_tx_pkts = snapshot.get('opackets', 0) - self.latest_stats.get('opackets', 0) + diff_tx_bytes = snapshot.get('obytes', 0) - self.latest_stats.get('obytes', 0) + tx_bps_L1 = calc_bps_L1(8.0 * diff_tx_bytes / ts_diff, float(diff_tx_pkts) / ts_diff) + snapshot['tx_percentage'] = 100.0 * tx_bps_L1 / self._port_obj.get_speed_bps() + + # RX line util not smoothed + diff_rx_pkts = snapshot.get('ipackets', 0) - self.latest_stats.get('ipackets', 0) + diff_rx_bytes = snapshot.get('ibytes', 0) - self.latest_stats.get('ibytes', 0) + rx_bps_L1 = calc_bps_L1(8.0 * diff_rx_bytes / ts_diff, float(diff_rx_pkts) / ts_diff) + snapshot['rx_percentage'] = 100.0 * rx_bps_L1 / self._port_obj.get_speed_bps() + # simple... self.latest_stats = snapshot @@ -715,9 +768,17 @@ class CPortStats(CTRexStats): else: state = format_text(state, 'bold') + # mark owned ports by color + if self._port_obj: + owner = self._port_obj.get_owner() + if self._port_obj.is_acquired(): + owner = format_text(owner, 'green') + else: + owner = '' - return {"owner": self._port_obj.user if self._port_obj else "", + return {"owner": owner, "state": "{0}".format(state), + "speed": self._port_obj.get_formatted_speed() if self._port_obj else '', "--": " ", "---": " ", @@ -769,9 +830,15 @@ class CPortStats(CTRexStats): # RX stats objects - COMPLEX :-( class CRxStats(CTRexStats): - def __init__(self): + def __init__(self, ports): super(CRxStats, self).__init__() + self.ports = ports + self.ports_speed = {} + def get_ports_speed(self): + for port in self.ports: + self.ports_speed[str(port)] = self.ports[port].get_speed_bps() + self.ports_speed['total'] = sum(self.ports_speed.values()) # calculates a diff between previous snapshot # and current one @@ -797,7 +864,7 @@ class CRxStats(CTRexStats): for field in ['tx_pkts', 'tx_bytes', 'rx_pkts', 'rx_bytes']: # is in the first time ? (nothing in prev) - if not field in output: + if field not in output: output[field] = {} # does the current snapshot has this field ? @@ -832,7 +899,7 @@ class CRxStats(CTRexStats): output['ts'] = current['ts'] # we care only about the current active keys - pg_ids = filter(is_intable, current.keys()) + pg_ids = list(filter(is_intable, current.keys())) for pg_id in pg_ids: @@ -858,7 +925,7 @@ class CRxStats(CTRexStats): # cleanp old reference values - they are dead - ref_pg_ids = filter(is_intable, self.reference_stats.keys()) + ref_pg_ids = list(filter(is_intable, self.reference_stats.keys())) deleted_pg_ids = set(ref_pg_ids).difference(pg_ids) for d_pg_id in deleted_pg_ids: @@ -869,46 +936,66 @@ class CRxStats(CTRexStats): def calculate_bw_for_pg (self, pg_current, pg_prev = None, diff_sec = 0.0): - - # if no previous values - its None + # no previous values if (pg_prev == None) or not (diff_sec > 0): - pg_current['tx_pps'] = None - pg_current['tx_bps'] = None - pg_current['tx_bps_L1'] = None - pg_current['rx_pps'] = None - pg_current['rx_bps'] = None + pg_current['tx_pps'] = {} + pg_current['tx_bps'] = {} + pg_current['tx_bps_L1'] = {} + pg_current['tx_line_util'] = {} + pg_current['rx_pps'] = {} + pg_current['rx_bps'] = {} + pg_current['rx_bps_L1'] = {} + pg_current['rx_line_util'] = {} + + pg_current['tx_pps_lpf'] = {} + pg_current['tx_bps_lpf'] = {} + pg_current['tx_bps_L1_lpf'] = {} + pg_current['rx_pps_lpf'] = {} + pg_current['rx_bps_lpf'] = {} + pg_current['rx_bps_L1_lpf'] = {} return - - # read the current values - now_tx_pkts = pg_current['tx_pkts']['total'] - now_tx_bytes = pg_current['tx_bytes']['total'] - now_rx_pkts = pg_current['rx_pkts']['total'] - now_rx_bytes = pg_current['rx_bytes']['total'] - - # prev values - prev_tx_pkts = pg_prev['tx_pkts']['total'] - prev_tx_bytes = pg_prev['tx_bytes']['total'] - prev_rx_pkts = pg_prev['rx_pkts']['total'] - prev_rx_bytes = pg_prev['rx_bytes']['total'] - - # prev B/W - prev_tx_pps = pg_prev['tx_pps'] - prev_tx_bps = pg_prev['tx_bps'] - prev_rx_pps = pg_prev['rx_pps'] - prev_rx_bps = pg_prev['rx_bps'] - - - #assert(now_tx_pkts >= prev_tx_pkts) - pg_current['tx_pps'] = self.calc_pps(prev_tx_pps, now_tx_pkts, prev_tx_pkts, diff_sec) - pg_current['tx_bps'] = self.calc_bps(prev_tx_bps, now_tx_bytes, prev_tx_bytes, diff_sec) - pg_current['rx_pps'] = self.calc_pps(prev_rx_pps, now_rx_pkts, prev_rx_pkts, diff_sec) - pg_current['rx_bps'] = self.calc_bps(prev_rx_bps, now_rx_bytes, prev_rx_bytes, diff_sec) - - if pg_current['tx_bps'] != None and pg_current['tx_pps'] != None: - pg_current['tx_bps_L1'] = calc_bps_L1(pg_current['tx_bps'], pg_current['tx_pps']) - else: - pg_current['tx_bps_L1'] = None + # TX + self.get_ports_speed() + for port in pg_current['tx_pkts'].keys(): + prev_tx_pps = pg_prev['tx_pps'].get(port) + now_tx_pkts = pg_current['tx_pkts'].get(port) + prev_tx_pkts = pg_prev['tx_pkts'].get(port) + pg_current['tx_pps'][port], pg_current['tx_pps_lpf'][port] = self.calc_pps(prev_tx_pps, now_tx_pkts, prev_tx_pkts, diff_sec) + + prev_tx_bps = pg_prev['tx_bps'].get(port) + now_tx_bytes = pg_current['tx_bytes'].get(port) + prev_tx_bytes = pg_prev['tx_bytes'].get(port) + pg_current['tx_bps'][port], pg_current['tx_bps_lpf'][port] = self.calc_bps(prev_tx_bps, now_tx_bytes, prev_tx_bytes, diff_sec) + + if pg_current['tx_bps'].get(port) != None and pg_current['tx_pps'].get(port) != None: + pg_current['tx_bps_L1'][port] = calc_bps_L1(pg_current['tx_bps'][port], pg_current['tx_pps'][port]) + pg_current['tx_bps_L1_lpf'][port] = calc_bps_L1(pg_current['tx_bps_lpf'][port], pg_current['tx_pps_lpf'][port]) + pg_current['tx_line_util'][port] = 100.0 * pg_current['tx_bps_L1'][port] / self.ports_speed[port] + else: + pg_current['tx_bps_L1'][port] = None + pg_current['tx_bps_L1_lpf'][port] = None + pg_current['tx_line_util'][port] = None + + # RX + for port in pg_current['rx_pkts'].keys(): + prev_rx_pps = pg_prev['rx_pps'].get(port) + now_rx_pkts = pg_current['rx_pkts'].get(port) + prev_rx_pkts = pg_prev['rx_pkts'].get(port) + pg_current['rx_pps'][port], pg_current['rx_pps_lpf'][port] = self.calc_pps(prev_rx_pps, now_rx_pkts, prev_rx_pkts, diff_sec) + + prev_rx_bps = pg_prev['rx_bps'].get(port) + now_rx_bytes = pg_current['rx_bytes'].get(port) + prev_rx_bytes = pg_prev['rx_bytes'].get(port) + pg_current['rx_bps'][port], pg_current['rx_bps_lpf'][port] = self.calc_bps(prev_rx_bps, now_rx_bytes, prev_rx_bytes, diff_sec) + if pg_current['rx_bps'].get(port) != None and pg_current['rx_pps'].get(port) != None: + pg_current['rx_bps_L1'][port] = calc_bps_L1(pg_current['rx_bps'][port], pg_current['rx_pps'][port]) + pg_current['rx_bps_L1_lpf'][port] = calc_bps_L1(pg_current['rx_bps_lpf'][port], pg_current['rx_pps_lpf'][port]) + pg_current['rx_line_util'][port] = 100.0 * pg_current['rx_bps_L1'][port] / self.ports_speed[port] + else: + pg_current['rx_bps_L1'][port] = None + pg_current['rx_bps_L1_lpf'][port] = None + pg_current['rx_line_util'][port] = None def calc_pps (self, prev_bw, now, prev, diff_sec): @@ -918,11 +1005,11 @@ class CRxStats(CTRexStats): def calc_bps (self, prev_bw, now, prev, diff_sec): return self.calc_bw(prev_bw, now, prev, diff_sec, True) - + # returns tuple - first value is real, second is low pass filtered def calc_bw (self, prev_bw, now, prev, diff_sec, is_bps): # B/W is not valid when the values are None if (now is None) or (prev is None): - return None + return (None, None) # calculate the B/W for current snapshot current_bw = (now - prev) / diff_sec @@ -933,7 +1020,7 @@ class CRxStats(CTRexStats): if prev_bw is None: prev_bw = 0 - return ( (0.5 * prev_bw) + (0.5 * current_bw) ) + return (current_bw, 0.5 * prev_bw + 0.5 * current_bw) @@ -960,22 +1047,29 @@ class CRxStats(CTRexStats): # skip non ints if not is_intable(pg_id): continue - + # bare counters stats[int(pg_id)] = {} - for field in ['tx_pkts', 'tx_bytes', 'rx_pkts']: - stats[int(pg_id)][field] = {'total': self.get_rel([pg_id, field, 'total'])} - - for port, pv in value[field].items(): - try: - int(port) - except ValueError: - continue - stats[int(pg_id)][field][int(port)] = self.get_rel([pg_id, field, port]) + for field in ['tx_pkts', 'tx_bytes', 'rx_pkts', 'rx_bytes']: + val = self.get_rel([pg_id, field, 'total']) + stats[int(pg_id)][field] = {'total': val if val != 'N/A' else StatNotAvailable(field)} + for port in value[field].keys(): + if is_intable(port): + val = self.get_rel([pg_id, field, port]) + stats[int(pg_id)][field][int(port)] = val if val != 'N/A' else StatNotAvailable(field) + + # BW values + for field in ['tx_pps', 'tx_bps', 'tx_bps_L1', 'rx_pps', 'rx_bps', 'rx_bps_L1', 'tx_line_util', 'rx_line_util']: + val = self.get([pg_id, field, 'total']) + stats[int(pg_id)][field] = {'total': val if val != 'N/A' else StatNotAvailable(field)} + for port in value[field].keys(): + if is_intable(port): + val = self.get([pg_id, field, port]) + stats[int(pg_id)][field][int(port)] = val if val != 'N/A' else StatNotAvailable(field) return stats - + # for Console def generate_stats (self): # for TUI - maximum 4 @@ -1005,13 +1099,13 @@ class CRxStats(CTRexStats): # maximum 4 for pg_id in pg_ids: - formatted_stats['Tx pps'].append(self.get([pg_id, 'tx_pps'], format = True, suffix = "pps")) - formatted_stats['Tx bps L2'].append(self.get([pg_id, 'tx_bps'], format = True, suffix = "bps")) + formatted_stats['Tx pps'].append(self.get([pg_id, 'tx_pps_lpf', 'total'], format = True, suffix = "pps")) + formatted_stats['Tx bps L2'].append(self.get([pg_id, 'tx_bps_lpf', 'total'], format = True, suffix = "bps")) - formatted_stats['Tx bps L1'].append(self.get([pg_id, 'tx_bps_L1'], format = True, suffix = "bps")) + formatted_stats['Tx bps L1'].append(self.get([pg_id, 'tx_bps_L1_lpf', 'total'], format = True, suffix = "bps")) - formatted_stats['Rx pps'].append(self.get([pg_id, 'rx_pps'], format = True, suffix = "pps")) - formatted_stats['Rx bps'].append(self.get([pg_id, 'rx_bps'], format = True, suffix = "bps")) + formatted_stats['Rx pps'].append(self.get([pg_id, 'rx_pps_lpf', 'total'], format = True, suffix = "pps")) + formatted_stats['Rx bps'].append(self.get([pg_id, 'rx_bps_lpf', 'total'], format = True, suffix = "bps")) formatted_stats['opackets'].append(self.get_rel([pg_id, 'tx_pkts', 'total'])) formatted_stats['ipackets'].append(self.get_rel([pg_id, 'rx_pkts', 'total'])) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py index 3ce876ad..165942d8 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py @@ -361,6 +361,8 @@ class STLStream(object): int_mac_dst_override_mode = int(mac_dst_override_mode); + self.is_default_mac = not (int_mac_src_override_by_pkt or int_mac_dst_override_mode) + self.fields['flags'] = (int_mac_src_override_by_pkt&1) + ((int_mac_dst_override_mode&3)<<1) self.fields['action_count'] = action_count @@ -421,6 +423,10 @@ class STLStream(object): return self.id + def has_custom_mac_addr (self): + """ Return True if src or dst MAC were set as custom """ + return not self.is_default_mac + def get_name (self): """ Get the stream name """ return self.name @@ -835,6 +841,9 @@ class STLProfile(object): def is_pauseable (self): return all([x.get_mode() == "Continuous" for x in self.get_streams()]) + def has_custom_mac_addr (self): + return any([x.has_custom_mac_addr() for x in self.get_streams()]) + def has_flow_stats (self): return any([x.has_flow_stats() for x in self.get_streams()]) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py index cd15b831..d84af22f 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py @@ -1,5 +1,5 @@ -from collections import namedtuple +from collections import namedtuple, OrderedDict from .utils.text_opts import * from .trex_stl_exceptions import * import types @@ -139,4 +139,30 @@ def validate_type(arg_name, arg, valid_types): def verify_exclusive_arg (args_list): if not (len(list(filter(lambda x: x is not None, args_list))) == 1): raise STLError('exactly one parameter from {0} should be provided'.format(args_list)) - + +def listify (x): + if isinstance(x, list): + return x + else: + return [x] + +# shows as 'N/A', but does not let any compares for user to not mistake in automation +class StatNotAvailable(object): + def __init__(self, stat_name): + self.stat_name = stat_name + + def __repr__(self, *args, **kwargs): + return 'N/A' + + def __cmp__(self, *args, **kwargs): + raise Exception("Stat '%s' not available at this setup" % self.stat_name) + +class LRU_cache(OrderedDict): + def __init__(self, maxlen = 20, *args, **kwargs): + OrderedDict.__init__(self, *args, **kwargs) + self.maxlen = maxlen + + def __setitem__(self, *args, **kwargs): + OrderedDict.__setitem__(self, *args, **kwargs) + if len(self) > self.maxlen: + self.popitem(last = False) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py index ae74e932..b4903e81 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py @@ -49,12 +49,18 @@ def random_id_gen(length=8): # try to get number from input, return None in case of fail def get_number(input): try: - if type(input) in (int, long): - return input - return int(input) + return long(input) except: - return None + try: + return int(input) + except: + return None def list_intersect(l1, l2): return list(filter(lambda x: x in l2, l1)) +def list_difference (l1, l2): + return list(filter(lambda x: x not in l2, l1)) + +def is_sub_list (l1, l2): + return set(l1) <= set(l2) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/filters.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/filters.py new file mode 100644 index 00000000..714f7807 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/filters.py @@ -0,0 +1,144 @@ + +def shallow_copy(x): + return type(x)(x) + + +class ToggleFilter(object): + """ + This class provides a "sticky" filter, that works by "toggling" items of the original database on and off. + """ + def __init__(self, db_ref, show_by_default=True): + """ + Instantiate a ToggleFilter object + + :parameters: + db_ref : iterable + an iterable object (i.e. list, set etc) that would serve as the reference db of the instance. + Changes in that object will affect the output of ToggleFilter instance. + + show_by_default: bool + decide if by default all the items are "on", i.e. these items will be presented if no other + toggling occurred. + + default value : **True** + + """ + self._data = db_ref + self._toggle_db = set() + self._filter_method = filter + self.__set_initial_state(show_by_default) + + def reset (self): + """ + Toggles off all the items + """ + self._toggle_db = set() + + + def toggle_item(self, item_key): + """ + Toggle a single item in/out. + + :parameters: + item_key : + an item the by its value the filter can decide to toggle or not. + Example: int, str and so on. + + :return: + + **True** if item toggled **into** the filtered items + + **False** if item toggled **out from** the filtered items + + :raises: + + KeyError, in case if item key is not part of the toggled list and not part of the referenced db. + + """ + if item_key in self._toggle_db: + self._toggle_db.remove(item_key) + return False + elif item_key in self._data: + self._toggle_db.add(item_key) + return True + else: + raise KeyError("Provided item key isn't a key of the referenced data structure.") + + def toggle_items(self, *args): + """ + Toggle multiple items in/out with a single call. Each item will be ha. + + :parameters: + args : iterable + an iterable object containing all item keys to be toggled in/out + + :return: + + **True** if all toggled items were toggled **into** the filtered items + + **False** if at least one of the items was toggled **out from** the filtered items + + :raises: + + KeyError, in case if ont of the item keys was not part of the toggled list and not part of the referenced db. + + """ + # in python 3, 'map' returns an iterator, so wrapping with 'list' call creates same effect for both python 2 and 3 + return all(list(map(self.toggle_item, args))) + + def filter_items(self): + """ + Filters the pointed database by showing only the items mapped at toggle_db set. + + :returns: + Filtered data of the original object. + + """ + return self._filter_method(self.__toggle_filter, self._data) + + # private methods + + def __set_initial_state(self, show_by_default): + try: + _ = (x for x in self._data) + if isinstance(self._data, dict): + self._filter_method = ToggleFilter.dict_filter + if show_by_default: + self._toggle_db = set(self._data.keys()) + return + elif isinstance(self._data, list): + self._filter_method = ToggleFilter.list_filter + elif isinstance(self._data, set): + self._filter_method = ToggleFilter.set_filter + elif isinstance(self._data, tuple): + self._filter_method = ToggleFilter.tuple_filter + if show_by_default: + self._toggle_db = set(shallow_copy(self._data)) # assuming all relevant data with unique identifier + return + except TypeError: + raise TypeError("provided data object is not iterable") + + def __toggle_filter(self, x): + return (x in self._toggle_db) + + # static utility methods + + @staticmethod + def dict_filter(function, iterable): + assert isinstance(iterable, dict) + return {k: v + for k,v in iterable.items() + if function(k)} + + @staticmethod + def list_filter(function, iterable): + # in python 3, filter returns an iterator, so wrapping with list creates same effect for both python 2 and 3 + return list(filter(function, iterable)) + + @staticmethod + def set_filter(function, iterable): + return {x + for x in iterable + if function(x)} + + @staticmethod + def tuple_filter(function, iterable): + return tuple(filter(function, iterable)) + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py index c4f2b358..ad46625b 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py @@ -1,5 +1,6 @@ import argparse from collections import namedtuple +from .common import list_intersect, list_difference import sys import re import os @@ -262,7 +263,7 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'], 'action': "store_false"}), - PORT_LIST: ArgumentPack(['--port'], + PORT_LIST: ArgumentPack(['--port', '-p'], {"nargs": '+', 'dest':'ports', 'metavar': 'PORTS', @@ -374,22 +375,49 @@ class CCmdArgParser(argparse.ArgumentParser): def __init__(self, stateless_client, *args, **kwargs): super(CCmdArgParser, self).__init__(*args, **kwargs) self.stateless_client = stateless_client + self.cmd_name = kwargs.get('prog') - def parse_args(self, args=None, namespace=None): + + def has_ports_cfg (self, opts): + return hasattr(opts, "all_ports") or hasattr(opts, "ports") + + def parse_args(self, args=None, namespace=None, default_ports=None, verify_acquired=False): try: opts = super(CCmdArgParser, self).parse_args(args, namespace) if opts is None: return None + if not self.has_ports_cfg(opts): + return opts + # if all ports are marked or if (getattr(opts, "all_ports", None) == True) or (getattr(opts, "ports", None) == []): - opts.ports = self.stateless_client.get_all_ports() + if default_ports is None: + opts.ports = self.stateless_client.get_acquired_ports() + else: + opts.ports = default_ports # so maybe we have ports configured - elif getattr(opts, "ports", None): - for port in opts.ports: - if not self.stateless_client._validate_port_list(port): - self.error("port id '{0}' is not a valid port id\n".format(port)) + invalid_ports = list_difference(opts.ports, self.stateless_client.get_all_ports()) + if invalid_ports: + self.stateless_client.logger.log("{0}: port(s) {1} are not valid port IDs".format(self.cmd_name, invalid_ports)) + return None + + # verify acquired ports + if verify_acquired: + acquired_ports = self.stateless_client.get_acquired_ports() + + diff = list_difference(opts.ports, acquired_ports) + if diff: + self.stateless_client.logger.log("{0} - port(s) {1} are not acquired".format(self.cmd_name, diff)) + return None + + # no acquire ports at all + if not acquired_ports: + self.stateless_client.logger.log("{0} - no acquired ports".format(self.cmd_name)) + return None + + return opts diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py index bc2d44f4..5c0dfb14 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py @@ -124,16 +124,9 @@ def underline(text): def text_attribute(text, attribute): - if isinstance(text, str): - return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], - txt=text, - stop=TEXT_CODES[attribute]['end']) - elif isinstance(text, unicode): - return u"{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], - txt=text, - stop=TEXT_CODES[attribute]['end']) - else: - raise Exception("not a string") + return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], + txt=text, + stop=TEXT_CODES[attribute]['end']) FUNC_DICT = {'blue': blue, |