diff options
Diffstat (limited to 'scripts/automation/trex_control_plane')
23 files changed, 1321 insertions, 713 deletions
diff --git a/scripts/automation/trex_control_plane/doc/index.rst b/scripts/automation/trex_control_plane/doc/index.rst index 62fd9975..041a17c2 100755 --- a/scripts/automation/trex_control_plane/doc/index.rst +++ b/scripts/automation/trex_control_plane/doc/index.rst @@ -18,6 +18,49 @@ To understand the entirely how the API works and how to set up the server side, **Use the table of contents below or the menu to your left to navigate through the site** +Client Package +============== + +| Starting from version v1.99 TRex has separated client package included in main directory. +| Put it at any place you like, preferably same place as your scripts. +| (If it's not at same place as your scripts, you will need to ensure trex_client directory is in sys.path) + +Un-pack it using command::: + + tar -xzf trex_client_<TRex version>.tar.gz + +| The client assumes stateful daemon is running. +| After un-tarring the client package, you can verify basic tests in examples directory out of the box: + +.. code-block:: bash + + cd trex_client/stf/examples + python stf_example.py -s <server address> + +You can verify that the traffic was sent and arrives properly if you see something like this::: + + Connecting to 127.0.0.1 + Connected, start TRex + Sample until end + Test results: + Is valid history? True + Done warmup? True + Expected tx rate: {u'm_tx_expected_pps': 71898.4, u'm_tx_expected_bps': 535157280.0, u'm_tx_expected_cps': 1943.2} + Current tx rate: {u'm_tx_bps': 542338368.0, u'm_tx_cps': 1945.4, u'm_tx_pps': 79993.4} + Maximum latency: {u'max-4': 55, u'max-5': 30, u'max-6': 50, u'max-7': 30, u'max-0': 55, u'max-1': 40, u'max-2': 55, u'max-3': 30} + Average latency: {'all': 32.913, u'port6': 35.9, u'port7': 30.0, u'port4': 35.8, u'port5': 30.0, u'port2': 35.8, u'port3': 30.0, u'port0': 35.8, u'port1': 30.0} + Average window latency: {'all': 31.543, u'port6': 32.871, u'port7': 28.929, u'port4': 33.886, u'port5': 28.929, u'port2': 33.843, u'port3': 28.929, u'port0': 33.871, u'port1': 31.086} + Total drops: -3 + Drop rate: 0.0 + History size so far: 7 + + TX by ports: + 0: 230579 | 1: 359435 | 2: 230578 | 3: 359430 | 4: 230570 | 5: 359415 | 6: 230564 | 7: 359410 + RX by ports: + 0: 359434 | 1: 230580 | 2: 359415 | 3: 230571 | 4: 359429 | 5: 230579 | 6: 359411 | 7: 230565 + CPU utilization: + [0.0, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8] + API Reference ============= .. toctree:: @@ -50,4 +93,4 @@ Indices and tables .. rubric:: Footnotes -.. [#f1] For more information on JSON-RPC, check out the `official site <http://www.jsonrpc.org/>`_
\ No newline at end of file +.. [#f1] For more information on JSON-RPC, check out the `official site <http://www.jsonrpc.org/>`_ diff --git a/scripts/automation/trex_control_plane/doc_stl/api/client_code.rst b/scripts/automation/trex_control_plane/doc_stl/api/client_code.rst index 4ae2b9fd..d3e48dab 100755 --- a/scripts/automation/trex_control_plane/doc_stl/api/client_code.rst +++ b/scripts/automation/trex_control_plane/doc_stl/api/client_code.rst @@ -82,6 +82,11 @@ STLClient snippet # block until done c.wait_on_traffic(ports = [0, 1]) + # check for any warnings + if c.get_warnings(): + # handle warnings here + pass + finally: c.disconnect() diff --git a/scripts/automation/trex_control_plane/doc_stl/index.rst b/scripts/automation/trex_control_plane/doc_stl/index.rst index aa2abd75..8a2fc4b0 100644 --- a/scripts/automation/trex_control_plane/doc_stl/index.rst +++ b/scripts/automation/trex_control_plane/doc_stl/index.rst @@ -24,6 +24,32 @@ Un-pack it using command:: tar -xzf trex_client.tar.gz +How to use +================== + +| The client assumes server is running. +| After un-tarring the client package, you can verify basic tests in examples directory out of the box: + +.. code-block:: bash + + cd trex_client/stl/examples + python stl_imix.py -s <server address> + +You can verify that the traffic was sent and arrives properly if you see something like this::: + + Mapped ports to sides [0, 2] <--> [1, 3] + Injecting [0, 2] <--> [1, 3] on total rate of '30%' for 10 seconds + + Packets injected from [0, 2]: 473,856 + Packets injected from [1, 3]: 473,856 + + packets lost from [0, 2] --> [0, 2]: 0 pkts + packets lost from [1, 3] --> [1, 3]: 0 pkts + + Test has passed :-) + + +Also, in the stl folder there are directories with profiles that define streams and the console (with which you can easily send the profiles) How to pyATS ================== diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py index 074d9060..4fd1e4c7 100755 --- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py +++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py @@ -592,7 +592,10 @@ class CTRexClient(object): """ try: - return binascii.a2b_base64(self.server.get_trex_daemon_log()) + res = binascii.a2b_base64(self.server.get_trex_daemon_log()) + if type(res) is bytes: + return res.decode() + return res except AppError as err: self._handle_AppError_exception(err.args[0]) except ProtocolError: @@ -613,7 +616,10 @@ class CTRexClient(object): """ try: - return binascii.a2b_base64(self.server.get_trex_log()) + res = binascii.a2b_base64(self.server.get_trex_log()) + if type(res) is bytes: + return res.decode() + return res except AppError as err: self._handle_AppError_exception(err.args[0]) except ProtocolError: @@ -636,7 +642,10 @@ class CTRexClient(object): try: version_dict = OrderedDict() - result_lines = binascii.a2b_base64(self.server.get_trex_version()).split('\n') + res = binascii.a2b_base64(self.server.get_trex_version()) + if type(res) is bytes: + res = res.decode() + result_lines = res.split('\n') for line in result_lines: if not line: continue @@ -1224,13 +1233,12 @@ class CTRexResult(object): continue hist_last_keys = deque([res['histogram'][-1]['key']], maxlen = 2) sum_high = 0.0 - - for i, elem in enumerate(reversed(res['histogram'])): + for elem in reversed(res['histogram']): sum_high += elem['val'] hist_last_keys.append(elem['key']) if sum_high / res['cnt'] >= filtered_latency_amount: break - result[max_port] = sum(hist_last_keys) / len(hist_last_keys) + result[max_port] = (hist_last_keys[0] + hist_last_keys[-1]) / 2 else: return {} return result diff --git a/scripts/automation/trex_control_plane/stl/console/trex_console.py b/scripts/automation/trex_control_plane/stl/console/trex_console.py index da4c4486..f8161dcb 100755 --- a/scripts/automation/trex_control_plane/stl/console/trex_console.py +++ b/scripts/automation/trex_control_plane/stl/console/trex_console.py @@ -47,7 +47,7 @@ except: from functools import wraps -__version__ = "1.1" +__version__ = "2.0" # console custom logger class ConsoleLogger(LoggerApi): @@ -212,29 +212,7 @@ class TRexConsole(TRexGeneralCmd): return wrap - # TODO: remove this ugly duplication - def verify_connected_and_rw (f): - @wraps(f) - def wrap(*args): - inst = args[0] - func_name = f.__name__ - if func_name.startswith("do_"): - func_name = func_name[3:] - - if not inst.stateless_client.is_connected(): - print(format_text("\n'{0}' cannot be executed on offline mode\n".format(func_name), 'bold')) - return - - if inst.stateless_client.is_all_ports_acquired(): - print(format_text("\n'{0}' cannot be executed on read only mode\n".format(func_name), 'bold')) - return - - rc = f(*args) - return rc - - return wrap - - + def get_console_identifier(self): return "{context}_{server}".format(context=get_current_user(), server=self.stateless_client.get_connection_info()['server']) @@ -267,17 +245,19 @@ class TRexConsole(TRexGeneralCmd): if not self.stateless_client.is_connected(): self.prompt = "trex(offline)>" self.supported_rpc = None - return stop - if self.stateless_client.is_all_ports_acquired(): + elif not self.stateless_client.get_acquired_ports(): self.prompt = "trex(read-only)>" - return stop + elif self.stateless_client.is_all_ports_acquired(): + self.prompt = "trex>" - self.prompt = "trex>" + else: + self.prompt = "trex {0}>".format(self.stateless_client.get_acquired_ports()) return stop + def default(self, line): print("'{0}' is an unrecognized command. type 'help' or '?' for a list\n".format(line)) @@ -309,7 +289,7 @@ class TRexConsole(TRexGeneralCmd): @verify_connected def do_ping (self, line): '''Ping the server\n''' - self.stateless_client.ping() + self.stateless_client.ping_line(line) # set verbose on / off @@ -358,6 +338,7 @@ class TRexConsole(TRexGeneralCmd): ports = self.stateless_client.get_acquired_ports() if not ports: print("No ports acquired\n") + return with self.stateless_client.logger.supress(): table = stl_map_ports(self.stateless_client, ports = ports) @@ -415,17 +396,44 @@ class TRexConsole(TRexGeneralCmd): ############### connect def do_connect (self, line): - '''Connects to the server\n''' + '''Connects to the server and acquire ports\n''' self.stateless_client.connect_line(line) + def help_connect (self): + self.do_connect("-h") def do_disconnect (self, line): '''Disconnect from the server\n''' self.stateless_client.disconnect_line(line) - + + @verify_connected + def do_acquire (self, line): + '''Acquire ports\n''' + + self.stateless_client.acquire_line(line) + + + @verify_connected + def do_release (self, line): + '''Release ports\n''' + self.stateless_client.release_line(line) + + def do_reacquire (self, line): + '''reacquire all the ports under your logged user name''' + self.stateless_client.reacquire_line(line) + + def help_acquire (self): + self.do_acquire("-h") + + def help_release (self): + self.do_release("-h") + + def help_reacquire (self): + self.do_reacquire("-h") + ############### start def complete_start(self, text, line, begidx, endidx): @@ -441,7 +449,7 @@ class TRexConsole(TRexGeneralCmd): return TRexConsole.tree_autocomplete(s[l - 1]) - @verify_connected_and_rw + @verify_connected def do_start(self, line): '''Start selected traffic in specified port(s) on TRex\n''' @@ -454,7 +462,7 @@ class TRexConsole(TRexGeneralCmd): self.do_start("-h") ############# stop - @verify_connected_and_rw + @verify_connected def do_stop(self, line): '''stops port(s) transmitting traffic\n''' @@ -464,7 +472,7 @@ class TRexConsole(TRexGeneralCmd): self.do_stop("-h") ############# update - @verify_connected_and_rw + @verify_connected def do_update(self, line): '''update speed of port(s)currently transmitting traffic\n''' @@ -474,14 +482,14 @@ class TRexConsole(TRexGeneralCmd): self.do_update("-h") ############# pause - @verify_connected_and_rw + @verify_connected def do_pause(self, line): '''pause port(s) transmitting traffic\n''' self.stateless_client.pause_line(line) ############# resume - @verify_connected_and_rw + @verify_connected def do_resume(self, line): '''resume port(s) transmitting traffic\n''' @@ -490,7 +498,7 @@ class TRexConsole(TRexGeneralCmd): ########## reset - @verify_connected_and_rw + @verify_connected def do_reset (self, line): '''force stop all ports\n''' self.stateless_client.reset_line(line) @@ -537,28 +545,7 @@ class TRexConsole(TRexGeneralCmd): def do_events (self, line): '''shows events recieved from server\n''' - - x = parsing_opts.ArgumentPack(['-c','--clear'], - {'action' : "store_true", - 'default': False, - 'help': "clear the events log"}) - - parser = parsing_opts.gen_parser(self, - "events", - self.do_events.__doc__, - x) - - opts = parser.parse_args(line.split()) - if opts is None: - return - - events = self.stateless_client.get_events() - for ev in events: - print(ev) - - if opts.clear: - self.stateless_client.clear_events() - print(format_text("\n\nEvent log was cleared\n\n")) + return self.stateless_client.get_events_line(line) def complete_profile(self, text, line, begidx, endidx): @@ -590,9 +577,10 @@ class TRexConsole(TRexGeneralCmd): info = self.stateless_client.get_connection_info() exe = './trex-console --top -t -q -s {0} -p {1} --async_port {2}'.format(info['server'], info['sync_port'], info['async_port']) - cmd = ['/usr/bin/xterm', '-geometry', '111x47', '-sl', '0', '-title', 'trex_tui', '-e', exe] + cmd = ['/usr/bin/xterm', '-geometry', '111x49', '-sl', '0', '-title', 'trex_tui', '-e', exe] - self.terminal = subprocess.Popen(cmd) + # detach child + self.terminal = subprocess.Popen(cmd, preexec_fn = os.setpgrp) return @@ -749,9 +737,24 @@ def setParserOptions(): default = False) - parser.add_argument("--no_acquire", dest="acquire", - action="store_false", help="Acquire all ports on connect. Default is: ON.", - default = True) + group = parser.add_mutually_exclusive_group() + + group.add_argument("-a", "--acquire", dest="acquire", + nargs = '+', + type = int, + help="Acquire ports on connect. default is all available ports", + default = None) + + group.add_argument("-r", "--readonly", dest="readonly", + action="store_true", + help="Starts console in a read only mode", + default = False) + + + parser.add_argument("-f", "--force", dest="force", + action="store_true", + help="Force acquire the requested ports", + default = False) parser.add_argument("--batch", dest="batch", nargs = 1, @@ -777,7 +780,29 @@ def setParserOptions(): return parser - +# a simple info printed on log on +def show_intro (logger, c): + x = c.get_server_system_info() + ver = c.get_server_version().get('version', 'N/A') + + # find out which NICs the server has + port_types = {} + for port in x['ports']: + key = (port['speed'], port['driver']) + if not key in port_types: + port_types[key] = 0 + port_types[key] += 1 + + port_line = '' + for k, v in port_types.items(): + port_line += "{0} x {1}Gbps @ {2}".format(v, k[0], k[1]) + + logger.log(format_text("\nServer Info:\n", 'underline')) + logger.log("Server version: {:>}".format(format_text(ver, 'bold'))) + logger.log("Server CPU: {:>}".format(format_text("{:>} x {:>}".format(x.get('dp_core_count'), x.get('core_type')), 'bold'))) + logger.log("Ports count: {:>}".format(format_text(port_line, 'bold'))) + + def main(): parser = setParserOptions() options = parser.parse_args() @@ -814,15 +839,22 @@ def main(): logger.log("Log:\n" + format_text(e.brief() + "\n", 'bold')) return - if not options.tui and options.acquire: + if not options.tui and not options.readonly: try: # acquire all ports - stateless_client.acquire() + stateless_client.acquire(options.acquire, force = options.force) except STLError as e: logger.log("Log:\n" + format_text(e.brief() + "\n", 'bold')) - logger.log(format_text("\nSwitching to read only mode - only few commands will be available", 'bold')) + + logger.log("\n*** Failed to acquire all required ports ***\n") + return + + if options.readonly: + logger.log(format_text("\nRead only mode - only few commands will be available", 'bold')) + + show_intro(logger, stateless_client) + - # a script mode if options.batch: cont = run_script_file(options.batch[0], stateless_client) diff --git a/scripts/automation/trex_control_plane/stl/console/trex_tui.py b/scripts/automation/trex_control_plane/stl/console/trex_tui.py index 975017a5..231eff93 100644 --- a/scripts/automation/trex_control_plane/stl/console/trex_tui.py +++ b/scripts/automation/trex_control_plane/stl/console/trex_tui.py @@ -2,7 +2,7 @@ import termios import sys import os import time -from collections import OrderedDict +from collections import OrderedDict, deque import datetime if sys.version_info > (3,0): @@ -13,8 +13,7 @@ else: from trex_stl_lib.utils.text_opts import * from trex_stl_lib.utils import text_tables from trex_stl_lib import trex_stl_stats -import trex_root_path -from common.filters import ToggleFilter +from trex_stl_lib.utils.filters import ToggleFilter # for STL exceptions from trex_stl_lib.api import * @@ -42,6 +41,7 @@ class TrexTUIPanel(object): self.mng = mng self.name = name self.stateless_client = mng.stateless_client + self.is_graph = False def show (self): raise NotImplementedError("must implement this") @@ -49,48 +49,78 @@ class TrexTUIPanel(object): def get_key_actions (self): raise NotImplementedError("must implement this") + def get_name (self): return self.name # dashboard panel class TrexTUIDashBoard(TrexTUIPanel): + + FILTER_ACQUIRED = 1 + FILTER_ALL = 2 + def __init__ (self, mng): super(TrexTUIDashBoard, self).__init__(mng, "dashboard") + self.ports = self.stateless_client.get_all_ports() + self.key_actions = OrderedDict() self.key_actions['c'] = {'action': self.action_clear, 'legend': 'clear', 'show': True} - self.key_actions['p'] = {'action': self.action_pause, 'legend': 'pause', 'show': True} - self.key_actions['r'] = {'action': self.action_resume, 'legend': 'resume', 'show': True} + self.key_actions['p'] = {'action': self.action_pause, 'legend': 'pause', 'show': True, 'color': 'red'} + self.key_actions['r'] = {'action': self.action_resume, 'legend': 'resume', 'show': True, 'color': 'blue'} self.key_actions['+'] = {'action': self.action_raise, 'legend': 'up 5%', 'show': True} self.key_actions['-'] = {'action': self.action_lower, 'legend': 'low 5%', 'show': True} - self.ports = self.stateless_client.get_all_ports() + self.key_actions['o'] = {'action': self.action_show_owned, 'legend': 'owned ports', 'show': True} + self.key_actions['a'] = {'action': self.action_show_all, 'legend': 'all ports', 'show': True} + + # register all the ports to the toggle action + for port_id in self.ports: + self.key_actions[str(port_id)] = {'action': self.action_toggle_port(port_id), 'legend': 'port {0}'.format(port_id), 'show': False} + + self.toggle_filter = ToggleFilter(self.ports) + if self.stateless_client.get_acquired_ports(): + self.action_show_owned() + else: + self.action_show_all() + + + def get_showed_ports (self): + return self.toggle_filter.filter_items() + + def show (self): - stats = self.stateless_client._get_formatted_stats(self.toggle_filter.filter_items()) + stats = self.stateless_client._get_formatted_stats(self.get_showed_ports()) # print stats to screen for stat_type, stat_data in stats.items(): text_tables.print_table_with_header(stat_data.text_table, stat_type) def get_key_actions (self): - allowed = {} + allowed = OrderedDict() allowed['c'] = self.key_actions['c'] + allowed['o'] = self.key_actions['o'] + allowed['a'] = self.key_actions['a'] + for i in self.ports: + allowed[str(i)] = self.key_actions[str(i)] - if self.stateless_client.is_all_ports_acquired(): + + # if not all ports are acquired - no operations + if not (set(self.get_showed_ports()) <= set(self.stateless_client.get_acquired_ports())): return allowed - if len(self.stateless_client.get_transmitting_ports()) > 0: + # if any/some ports are transmitting - support those actions + if set(self.get_showed_ports()) & set(self.stateless_client.get_transmitting_ports()): allowed['p'] = self.key_actions['p'] allowed['+'] = self.key_actions['+'] allowed['-'] = self.key_actions['-'] - - if len(self.stateless_client.get_paused_ports()) > 0: + if set(self.get_showed_ports()) & set(self.stateless_client.get_paused_ports()): allowed['r'] = self.key_actions['r'] return allowed @@ -99,7 +129,7 @@ class TrexTUIDashBoard(TrexTUIPanel): ######### actions def action_pause (self): try: - rc = self.stateless_client.pause(ports = self.mng.ports) + rc = self.stateless_client.pause(ports = self.get_showed_ports()) except STLError: pass @@ -109,7 +139,7 @@ class TrexTUIDashBoard(TrexTUIPanel): def action_resume (self): try: - self.stateless_client.resume(ports = self.mng.ports) + self.stateless_client.resume(ports = self.get_showed_ports()) except STLError: pass @@ -118,7 +148,7 @@ class TrexTUIDashBoard(TrexTUIPanel): def action_raise (self): try: - self.stateless_client.update(mult = "5%+", ports = self.mng.ports) + self.stateless_client.update(mult = "5%+", ports = self.get_showed_ports()) except STLError: pass @@ -127,102 +157,34 @@ class TrexTUIDashBoard(TrexTUIPanel): def action_lower (self): try: - self.stateless_client.update(mult = "5%-", ports = self.mng.ports) - except STLError: - pass - - return "" - - - def action_clear (self): - self.stateless_client.clear_stats(self.mng.ports) - return "cleared all stats" - - -# port panel -class TrexTUIPort(TrexTUIPanel): - def __init__ (self, mng, port_id): - super(TrexTUIPort, self).__init__(mng, "port {0}".format(port_id)) - - self.port_id = port_id - self.port = self.mng.stateless_client.get_port(port_id) - - self.key_actions = OrderedDict() - - self.key_actions['c'] = {'action': self.action_clear, 'legend': 'clear', 'show': True} - self.key_actions['p'] = {'action': self.action_pause, 'legend': 'pause', 'show': True} - self.key_actions['r'] = {'action': self.action_resume, 'legend': 'resume', 'show': True} - self.key_actions['+'] = {'action': self.action_raise, 'legend': 'up 5%', 'show': True} - self.key_actions['-'] = {'action': self.action_lower, 'legend': 'low 5%', 'show': True} - - - def show (self): - stats = self.stateless_client._get_formatted_stats([self.port_id]) - # print stats to screen - for stat_type, stat_data in stats.items(): - text_tables.print_table_with_header(stat_data.text_table, stat_type) - - def get_key_actions (self): - - allowed = {} - - allowed['c'] = self.key_actions['c'] - - if self.stateless_client.is_all_ports_acquired(): - return allowed - - if self.port.state == self.port.STATE_TX: - allowed['p'] = self.key_actions['p'] - allowed['+'] = self.key_actions['+'] - allowed['-'] = self.key_actions['-'] - - elif self.port.state == self.port.STATE_PAUSE: - allowed['r'] = self.key_actions['r'] - - - return allowed - - # actions - def action_pause (self): - try: - self.stateless_client.pause(ports = [self.port_id]) + self.stateless_client.update(mult = "5%-", ports = self.get_showed_ports()) except STLError: pass return "" - def action_resume (self): - try: - self.stateless_client.resume(ports = [self.port_id]) - except STLError: - pass + def action_show_owned (self): + self.toggle_filter.reset() + self.toggle_filter.toggle_items(*self.stateless_client.get_acquired_ports()) return "" - - def action_raise (self): - mult = {'type': 'percentage', 'value': 5, 'op': 'add'} - - try: - self.stateless_client.update(mult = mult, ports = [self.port_id]) - except STLError: - pass - + def action_show_all (self): + self.toggle_filter.reset() + self.toggle_filter.toggle_items(*self.stateless_client.get_all_ports()) return "" - def action_lower (self): - mult = {'type': 'percentage', 'value': 5, 'op': 'sub'} + def action_clear (self): + self.stateless_client.clear_stats(self.toggle_filter.filter_items()) + return "cleared all stats" - try: - self.stateless_client.update(mult = mult, ports = [self.port_id]) - except STLError: - pass - return "" + def action_toggle_port(self, port_id): + def action_toggle_port_x(): + self.toggle_filter.toggle_item(port_id) + return "" - def action_clear (self): - self.stateless_client.clear_stats([self.port_id]) - return "port {0}: cleared stats".format(self.port_id) + return action_toggle_port_x @@ -290,10 +252,6 @@ class TrexTUIPanelManager(): self.key_actions['g'] = {'action': self.action_show_dash, 'legend': 'dashboard', 'show': True} self.key_actions['s'] = {'action': self.action_show_sstats, 'legend': 'streams stats', 'show': True} - for port_id in self.ports: - self.key_actions[str(port_id)] = {'action': self.action_toggle_port(port_id), 'legend': 'port {0}'.format(port_id), 'show': False} - self.panels['port {0}'.format(port_id)] = TrexTUIPort(self, port_id) - # start with dashboard self.main_panel = self.panels['dashboard'] @@ -306,23 +264,30 @@ class TrexTUIPanelManager(): self.dis_bar = SimpleBar('status: ', ['X', ' ']) self.show_log = False + def generate_legend (self): + self.legend = "\n{:<12}".format("browse:") for k, v in self.key_actions.items(): if v['show']: x = "'{0}' - {1}, ".format(k, v['legend']) - self.legend += "{:}".format(x) - - self.legend += "'0-{0}' - port display".format(len(self.ports) - 1) + if v.get('color'): + self.legend += "{:}".format(format_text(x, v.get('color'))) + else: + self.legend += "{:}".format(x) self.legend += "\n{:<12}".format(self.main_panel.get_name() + ":") for k, v in self.main_panel.get_key_actions().items(): if v['show']: x = "'{0}' - {1}, ".format(k, v['legend']) - self.legend += "{:}".format(x) + + if v.get('color'): + self.legend += "{:}".format(format_text(x, v.get('color'))) + else: + self.legend += "{:}".format(x) def print_connection_status (self): @@ -389,14 +354,6 @@ class TrexTUIPanelManager(): return action_show_port_x - def action_toggle_port(self, port_id): - def action_toggle_port_x(): - self.panels['dashboard'].toggle_filter.toggle_item(port_id) - self.init() - return "" - - return action_toggle_port_x - def action_show_sstats (self): @@ -410,6 +367,7 @@ class TrexTUI(): STATE_ACTIVE = 0 STATE_LOST_CONT = 1 STATE_RECONNECT = 2 + is_graph = False def __init__ (self, stateless_client): self.stateless_client = stateless_client @@ -503,7 +461,9 @@ class TrexTUI(): sys.stdout = old_stdout self.clear_screen() + print(mystdout.getvalue()) + sys.stdout.flush() self.draw_policer = 0 diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py index ff16d397..9977fa3e 100644 --- a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py +++ b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows.py @@ -32,13 +32,13 @@ def create_pkt (size, direction): base = Ether()/IP()/UDP() - pad = max(0, len(base)) * 'x' + pad = max(0, size-len(base)) * 'x' return STLPktBuilder(pkt = base/pad, vm = vm) -def simple_burst (): +def simple_burst (port_a, port_b, pkt_size, rate): # create client @@ -50,11 +50,11 @@ def simple_burst (): #c.set_verbose("high") # create two streams - s1 = STLStream(packet = create_pkt(200, 0), + s1 = STLStream(packet = create_pkt(pkt_size, 0), mode = STLTXCont(pps = 100)) # second stream with a phase of 1ms (inter stream gap) - s2 = STLStream(packet = create_pkt(200, 1), + s2 = STLStream(packet = create_pkt(pkt_size, 1), isg = 1000, mode = STLTXCont(pps = 100)) @@ -62,36 +62,41 @@ def simple_burst (): # connect to server c.connect() - # prepare our ports (my machine has 0 <--> 1 with static route) - c.reset(ports = [0, 1]) + # prepare our ports + c.reset(ports = [port_a, port_b]) # add both streams to ports - c.add_streams(s1, ports = [0]) - c.add_streams(s2, ports = [1]) + c.add_streams(s1, ports = [port_a]) + c.add_streams(s2, ports = [port_b]) # clear the stats before injecting c.clear_stats() - # choose rate and start traffic for 10 seconds on 5 mpps - print("Running 5 Mpps on ports 0, 1 for 10 seconds...") - c.start(ports = [0, 1], mult = "5mpps", duration = 10) + # here we multiply the traffic lineaer to whatever given in rate + print("Running {:} on ports {:}, {:} for 10 seconds...".format(rate, port_a, port_b)) + c.start(ports = [port_a, port_b], mult = rate, duration = 10) # block until done - c.wait_on_traffic(ports = [0, 1]) + c.wait_on_traffic(ports = [port_a, port_b]) # read the stats after the test stats = c.get_stats() - print(json.dumps(stats[0], indent = 4, separators=(',', ': '), sort_keys = True)) - print(json.dumps(stats[1], indent = 4, separators=(',', ': '), sort_keys = True)) + print(json.dumps(stats[port_a], indent = 4, separators=(',', ': '), sort_keys = True)) + print(json.dumps(stats[port_b], indent = 4, separators=(',', ': '), sort_keys = True)) - lost_a = stats[0]["opackets"] - stats[1]["ipackets"] - lost_b = stats[1]["opackets"] - stats[0]["ipackets"] + lost_a = stats[port_a]["opackets"] - stats[port_b]["ipackets"] + lost_b = stats[port_b]["opackets"] - stats[port_a]["ipackets"] - print("\npackets lost from 0 --> 1: {0} pkts".format(lost_a)) - print("packets lost from 1 --> 0: {0} pkts".format(lost_b)) + print("\npackets lost from {0} --> {1}: {2} pkts".format(port_a, port_b, lost_a)) + print("packets lost from {0} --> {1}: {2} pkts".format(port_b, port_a, lost_b)) - if (lost_a == 0) and (lost_b == 0): + if c.get_warnings(): + print("\n\n*** test had warnings ****\n\n") + for w in c.get_warnings(): + print(w) + + if (lost_a == 0) and (lost_b == 0) and not c.get_warnings(): passed = True else: passed = False @@ -108,7 +113,6 @@ def simple_burst (): else: print("\nTest has failed :-(\n") -while True: # run the tests - simple_burst() +simple_burst(0, 3, 64, "10gbps") diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows1.py b/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows1.py deleted file mode 100644 index 6e08a0fa..00000000 --- a/scripts/automation/trex_control_plane/stl/examples/stl_bi_dir_flows1.py +++ /dev/null @@ -1,114 +0,0 @@ -import stl_path -from trex_stl_lib.api import * - -import time -import json - -# simple packet creation -def create_pkt (size, direction): - - ip_range = {'src': {'start': "10.0.0.1", 'end': "10.0.0.254"}, - 'dst': {'start': "8.0.0.1", 'end': "8.0.0.254"}} - - if (direction == 0): - src = ip_range['src'] - dst = ip_range['dst'] - else: - src = ip_range['dst'] - dst = ip_range['src'] - - vm = [ - # src - STLVmFlowVar(name="src",min_value=src['start'],max_value=src['end'],size=4,op="inc"), - STLVmWrFlowVar(fv_name="src",pkt_offset= "IP.src"), - - # dst - STLVmFlowVar(name="dst",min_value=dst['start'],max_value=dst['end'],size=4,op="inc"), - STLVmWrFlowVar(fv_name="dst",pkt_offset= "IP.dst"), - - # checksum - STLVmFixIpv4(offset = "IP") - ] - - - base = Ether()/IP()/UDP() - pad = max(0, len(base)) * 'x' - - return STLPktBuilder(pkt = base/pad, - vm = vm) - - -def simple_burst (): - - - # create client - c = STLClient() - passed = True - - try: - # turn this on for some information - #c.set_verbose("high") - - # create two streams - s1 = STLStream(packet = create_pkt(200, 0), - mode = STLTXCont(pps = 100)) - - # second stream with a phase of 1ms (inter stream gap) - s2 = STLStream(packet = create_pkt(200, 1), - isg = 1000, - mode = STLTXCont(pps = 100)) - - - # connect to server - c.connect() - - # prepare our ports (my machine has 0 <--> 1 with static route) - c.reset(ports = [2, 3]) - - # add both streams to ports - c.add_streams(s1, ports = [2]) - c.add_streams(s2, ports = [3]) - - # clear the stats before injecting - c.clear_stats() - - # choose rate and start traffic for 10 seconds on 5 mpps - print("Running 5 Mpps on ports 0, 1 for 10 seconds...") - c.start(ports = [2, 3], mult = "5mpps", duration = 10) - - # block until done - c.wait_on_traffic(ports = [2, 3]) - - # read the stats after the test - stats = c.get_stats() - - print(json.dumps(stats[2], indent = 4, separators=(',', ': '), sort_keys = True)) - print(json.dumps(stats[3], indent = 4, separators=(',', ': '), sort_keys = True)) - - lost_a = stats[2]["opackets"] - stats[3]["ipackets"] - lost_b = stats[3]["opackets"] - stats[2]["ipackets"] - - print("\npackets lost from 0 --> 1: {0} pkts".format(lost_a)) - print("packets lost from 1 --> 0: {0} pkts".format(lost_b)) - - if (lost_a == 0) and (lost_b == 0): - passed = True - else: - passed = False - - except STLError as e: - passed = False - print(e) - - finally: - c.disconnect() - - if passed: - print("\nTest has passed :-)\n") - else: - print("\nTest has failed :-(\n") - -while True : - # run the tests - simple_burst() - diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py index d938852e..ed4902fa 100644 --- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py +++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py @@ -4,7 +4,7 @@ from trex_stl_lib.api import * import time import pprint -def rx_example (tx_port, rx_port, burst_size): +def rx_example (tx_port, rx_port, burst_size, bw): print("\nGoing to inject {0} packets on port {1} - checking RX stats on port {2}\n".format(burst_size, tx_port, rx_port)) @@ -19,9 +19,7 @@ def rx_example (tx_port, rx_port, burst_size): packet = pkt, flow_stats = STLFlowStats(pg_id = 5), mode = STLTXSingleBurst(total_pkts = total_pkts, - #pps = total_pkts - percentage = 80 - )) + percentage = bw)) # connect to server c.connect() @@ -36,7 +34,7 @@ def rx_example (tx_port, rx_port, burst_size): for i in range(0, 10): print("\nStarting iteration: {0}:".format(i)) - rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len()) + rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len(), bw) if not rc: passed = False break @@ -55,7 +53,7 @@ def rx_example (tx_port, rx_port, burst_size): print("\nTest has failed :-(\n") # RX one iteration -def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len): +def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw): c.clear_stats() @@ -71,6 +69,12 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len): tx_bytes = flow_stats['tx_bytes'].get(tx_port, 0) rx_pkts = flow_stats['rx_pkts'].get(rx_port, 0) + if c.get_warnings(): + print("\n\n*** test had warnings ****\n\n") + for w in c.get_warnings(): + print(w) + return False + if tx_pkts != total_pkts: print("TX pkts mismatch - got: {0}, expected: {1}".format(tx_pkts, total_pkts)) pprint.pprint(flow_stats) @@ -95,5 +99,5 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len): return True # run the tests -rx_example(tx_port = 1, rx_port = 2, burst_size = 500000) +rx_example(tx_port = 1, rx_port = 2, burst_size = 500000, bw = 50) diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py index bc7990aa..46d86b2b 100644 --- a/scripts/automation/trex_control_plane/stl/examples/stl_imix.py +++ b/scripts/automation/trex_control_plane/stl/examples/stl_imix.py @@ -83,7 +83,12 @@ def imix_test (server, mult): print("\npackets lost from {0} --> {1}: {2:,} pkts".format(dir_0, dir_0, lost_0)) print("packets lost from {0} --> {1}: {2:,} pkts".format(dir_1, dir_1, lost_1)) - if (lost_0 <= 0) and (lost_1 <= 0): # less or equal because we might have incoming arps etc. + if c.get_warnings(): + print("\n\n*** test had warnings ****\n\n") + for w in c.get_warnings(): + print(w) + + if (lost_0 <= 0) and (lost_1 <= 0) and not c.get_warnings(): # less or equal because we might have incoming arps etc. passed = True else: passed = False diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py b/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py index 29341674..4bd9fd4c 100644 --- a/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py +++ b/scripts/automation/trex_control_plane/stl/examples/stl_simple_burst.py @@ -3,47 +3,49 @@ from trex_stl_lib.api import * import time -def simple_burst (): +def simple_burst (port_a, port_b, pkt_size, burst_size, rate): # create client c = STLClient() passed = True try: - pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP()/'a_payload_example') + pkt_base = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP() + pad = max(0, pkt_size - len(pkt_base)) * 'x' + pkt = STLPktBuilder(pkt = pkt_base / pad) # create two bursts and link them s1 = STLStream(name = 'A', packet = pkt, - mode = STLTXSingleBurst(total_pkts = 5000), + mode = STLTXSingleBurst(total_pkts = burst_size), next = 'B') s2 = STLStream(name = 'B', self_start = False, packet = pkt, - mode = STLTXSingleBurst(total_pkts = 3000)) + mode = STLTXSingleBurst(total_pkts = burst_size)) # connect to server c.connect() # prepare our ports - c.reset(ports = [0, 3]) + c.reset(ports = [port_a, port_b]) # add both streams to ports - stream_ids = c.add_streams([s1, s2], ports = [0, 3]) + stream_ids = c.add_streams([s1, s2], ports = [port_a, port_b]) # run 5 times for i in range(1, 6): c.clear_stats() - c.start(ports = [0, 3], mult = "1gbps") - c.wait_on_traffic(ports = [0, 3]) + c.start(ports = [port_a, port_b], mult = rate) + c.wait_on_traffic(ports = [port_a, port_b]) stats = c.get_stats() ipackets = stats['total']['ipackets'] print("Test iteration {0} - Packets Received: {1} ".format(i, ipackets)) - # (5000 + 3000) * 2 ports = 16,000 - if (ipackets != (16000)): + # two streams X 2 ports + if (ipackets != (burst_size * 2 * 2)): passed = False except STLError as e: @@ -53,12 +55,17 @@ def simple_burst (): finally: c.disconnect() - if passed: + if c.get_warnings(): + print("\n\n*** test had warnings ****\n\n") + for w in c.get_warnings(): + print(w) + + if passed and not c.get_warnings(): print("\nTest has passed :-)\n") else: print("\nTest has failed :-(\n") # run the tests -simple_burst() +simple_burst(0, 3, 256, 50000, "80%") diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py index 0f0fe83e..022077a9 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py @@ -275,18 +275,18 @@ class CTRexAsyncClient(): # stats if name == "trex-global": - self.event_handler.handle_async_stats_update(data, baseline) + self.event_handler.on_async_stats_update(data, baseline) # events elif name == "trex-event": - self.event_handler.handle_async_event(type, data) + self.event_handler.on_async_event(type, data) # barriers elif name == "trex-barrier": self.handle_async_barrier(type, data) elif name == "flow_stats": - self.event_handler.handle_async_rx_stats_event(data, baseline) + self.event_handler.on_async_rx_stats_event(data, baseline) else: pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py index bddc4ad0..77fa40bb 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py @@ -12,6 +12,7 @@ from .trex_stl_types import * from .trex_stl_async_client import CTRexAsyncClient from .utils import parsing_opts, text_tables, common +from .utils.common import list_intersect, list_difference, is_sub_list from .utils.text_opts import * from functools import wraps @@ -125,8 +126,26 @@ class DefaultLogger(LoggerApi): ############################ ############################# ############################ ############################# +# an event +class Event(object): + + def __init__ (self, origin, ev_type, msg): + self.origin = origin + self.ev_type = ev_type + self.msg = msg + + self.ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') + + def __str__ (self): + + prefix = "[{:^}][{:^}]".format(self.origin, self.ev_type) + + return "{:<10} - {:18} - {:}".format(self.ts, prefix, format_text(self.msg, 'bold')) + + # handles different async events given to the client -class AsyncEventHandler(object): +class EventsHandler(object): + def __init__ (self, client): self.client = client @@ -136,31 +155,41 @@ class AsyncEventHandler(object): # public functions - def get_events (self): - return self.events + def get_events (self, ev_type_filter = None): + if ev_type_filter: + return [ev for ev in self.events if ev.ev_type in listify(ev_type_filter)] + else: + return [ev for ev in self.events] def clear_events (self): self.events = [] + def log_warning (self, msg, show = True): + self.__add_event_log('local', 'warning', msg, show) + + + # events called internally + def on_async_dead (self): if self.client.connected: msg = 'Lost connection to server' - self.__add_event_log(msg, 'local', True) + self.__add_event_log('local', 'info', msg, True) self.client.connected = False def on_async_alive (self): pass + - def handle_async_rx_stats_event (self, data, baseline): + def on_async_rx_stats_event (self, data, baseline): self.client.flow_stats.update(data, baseline) # handles an async stats update from the subscriber - def handle_async_stats_update(self, dump_data, baseline): + def on_async_stats_update(self, dump_data, baseline): global_stats = {} port_stats = {} @@ -189,8 +218,9 @@ class AsyncEventHandler(object): self.client.ports[port_id].port_stats.update(data, baseline) + # dispatcher for server async events (port started, port stopped and etc.) - def handle_async_event (self, type, data): + def on_async_event (self, type, data): # DP stopped show_event = False @@ -234,22 +264,55 @@ class AsyncEventHandler(object): self.__async_event_port_job_done(port_id) show_event = True - # port was stolen... + # port was acquired - maybe stolen... elif (type == 5): session_id = data['session_id'] - # false alarm, its us + port_id = int(data['port_id']) + who = data['who'] + force = data['force'] + + # if we hold the port and it was not taken by this session - show it + if port_id in self.client.get_acquired_ports() and session_id != self.client.session_id: + show_event = True + + # format the thief/us... if session_id == self.client.session_id: - return + user = 'you' + elif who == self.client.username: + user = 'another session of you' + else: + user = "'{0}'".format(who) - port_id = int(data['port_id']) - who = data['who'] + if force: + ev = "Port {0} was forcely taken by {1}".format(port_id, user) + else: + ev = "Port {0} was taken by {1}".format(port_id, user) - ev = "Port {0} was forcely taken by '{1}'".format(port_id, who) + # call the handler in case its not this session + if session_id != self.client.session_id: + self.__async_event_port_acquired(port_id, who) + + + # port was released + elif (type == 6): + port_id = int(data['port_id']) + who = data['who'] + session_id = data['session_id'] + + if session_id == self.client.session_id: + user = 'you' + elif who == self.client.username: + user = 'another session of you' + else: + user = "'{0}'".format(who) + + ev = "Port {0} was released by {1}".format(port_id, user) + + # call the handler in case its not this session + if session_id != self.client.session_id: + self.__async_event_port_released(port_id) - # call the handler - self.__async_event_port_forced_acquired(port_id) - show_event = True # server stopped elif (type == 100): @@ -263,7 +326,7 @@ class AsyncEventHandler(object): return - self.__add_event_log(ev, 'server', show_event) + self.__add_event_log('server', 'info', ev, show_event) # private functions @@ -287,28 +350,23 @@ class AsyncEventHandler(object): self.client.ports[port_id].async_event_port_resumed() - def __async_event_port_forced_acquired (self, port_id): - self.client.ports[port_id].async_event_forced_acquired() + def __async_event_port_acquired (self, port_id, who): + self.client.ports[port_id].async_event_acquired(who) + def __async_event_port_released (self, port_id): + self.client.ports[port_id].async_event_released() def __async_event_server_stopped (self): self.client.connected = False # add event to log - def __add_event_log (self, msg, ev_type, show = False): - - if ev_type == "server": - prefix = "[server]" - elif ev_type == "local": - prefix = "[local]" - - ts = time.time() - st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') - self.events.append("{:<10} - {:^8} - {:}".format(st, prefix, format_text(msg, 'bold'))) + def __add_event_log (self, origin, ev_type, msg, show = False): + event = Event(origin, ev_type, msg) + self.events.append(event) if show: - self.logger.async_log(format_text("\n\n{:^8} - {:}".format(prefix, format_text(msg, 'bold')))) + self.logger.async_log("\n\n{0}".format(str(event))) @@ -452,7 +510,7 @@ class STLClient(object): self) # async event handler manager - self.event_handler = AsyncEventHandler(self) + self.event_handler = EventsHandler(self) # async subscriber level self.async_client = CTRexAsyncClient(server, @@ -472,9 +530,10 @@ class STLClient(object): self.global_stats = trex_stl_stats.CGlobalStats(self.connection_info, self.server_version, - self.ports) + self.ports, + self.event_handler) - self.flow_stats = trex_stl_stats.CRxStats() + self.flow_stats = trex_stl_stats.CRxStats(self.ports) self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats, self.ports, @@ -482,7 +541,7 @@ class STLClient(object): # API classes - self.api_vers = [ {'type': 'core', 'major': 1, 'minor':0 } + self.api_vers = [ {'type': 'core', 'major': 1, 'minor':2 } ] self.api_h = {'core': None} @@ -977,7 +1036,8 @@ class STLClient(object): """ - return not (self.get_all_ports() == self.get_acquired_ports()) + return (self.get_all_ports() == self.get_acquired_ports()) + # is the client connected ? def is_connected (self): @@ -1117,23 +1177,39 @@ class STLClient(object): if port_obj.is_acquired()] # get all active ports (TX or pause) - def get_active_ports(self): - return [port_id - for port_id, port_obj in self.ports.items() - if port_obj.is_active()] + def get_active_ports(self, owned = True): + if owned: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_active() and port_obj.is_acquired()] + else: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_active()] # get paused ports - def get_paused_ports (self): - return [port_id - for port_id, port_obj in self.ports.items() - if port_obj.is_paused()] + def get_paused_ports (self, owned = True): + if owned: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_paused() and port_obj.is_acquired()] + else: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_paused()] + # get all TX ports - def get_transmitting_ports (self): - return [port_id - for port_id, port_obj in self.ports.items() - if port_obj.is_transmitting()] + def get_transmitting_ports (self, owned = True): + if owned: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_transmitting() and port_obj.is_acquired()] + else: + return [port_id + for port_id, port_obj in self.ports.items() + if port_obj.is_transmitting()] # get stats @@ -1155,9 +1231,58 @@ class STLClient(object): return self.__get_stats(ports) - # return all async events - def get_events (self): - return self.event_handler.get_events() + + def get_events (self, ev_type_filter = None): + """ + returns all the logged events + + :parameters: + ev_type_filter - 'info', 'warning' or a list of those + default is no filter + + :return: + logged events + + :raises: + None + + """ + return self.event_handler.get_events(ev_type_filter) + + + def get_warnings (self): + """ + returns all the warnings logged events + + :parameters: + None + + :return: + warning logged events + + :raises: + None + + """ + return self.get_events(ev_type_filter = 'warning') + + + def get_info (self): + """ + returns all the info logged events + + :parameters: + None + + :return: + warning logged events + + :raises: + None + + """ + return self.get_events(ev_type_filter = 'info') + # get port(s) info as a list of dicts @__api_check(True) @@ -1473,8 +1598,8 @@ class STLClient(object): stream_id_list = [stream_id_list] # check streams - if not all([isinstance(stream_id, long) for stream_id in stream_id_list]): - raise STLArgumentError('stream_id_list', stream_id_list) + for stream_id in stream_id_list: + validate_type('stream_id', stream_id, int) # remove streams self.logger.pre_cmd("Removing {0} streams from port(s) {1}:".format(len(stream_id_list), ports)) @@ -1948,28 +2073,111 @@ class STLClient(object): return wrap + @__console + def ping_line (self, line): + '''pings the server''' + self.ping() + return True @__console def connect_line (self, line): - '''Connects to the TRex server''' - # define a parser + '''Connects to the TRex server and acquire ports''' parser = parsing_opts.gen_parser(self, "connect", self.connect_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, parsing_opts.FORCE) - opts = parser.parse_args(line.split()) - + opts = parser.parse_args(line.split(), default_ports = self.get_all_ports()) if opts is None: return - # call the API self.connect() - self.acquire(force = opts.force) + self.acquire(ports = opts.ports, force = opts.force) # true means print time return True + + @__console + def acquire_line (self, line): + '''Acquire ports\n''' + + # define a parser + parser = parsing_opts.gen_parser(self, + "acquire", + self.acquire_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, + parsing_opts.FORCE) + + opts = parser.parse_args(line.split(), default_ports = self.get_all_ports()) + if opts is None: + return + + # filter out all the already owned ports + ports = list_difference(opts.ports, self.get_acquired_ports()) + if not ports: + self.logger.log("acquire - all port(s) {0} are already acquired".format(opts.ports)) + return + + self.acquire(ports = ports, force = opts.force) + + # true means print time + return True + + + # + @__console + def release_line (self, line): + '''Release ports\n''' + + parser = parsing_opts.gen_parser(self, + "release", + self.release_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports()) + if opts is None: + return + + ports = list_intersect(opts.ports, self.get_acquired_ports()) + if not ports: + if not opts.ports: + self.logger.log("release - no acquired ports") + return + else: + self.logger.log("release - none of port(s) {0} are acquired".format(opts.ports)) + return + + + self.release(ports = ports) + + # true means print time + return True + + + @__console + def reacquire_line (self, line): + '''reacquire all the ports under your username which are not acquired by your session''' + + parser = parsing_opts.gen_parser(self, + "reacquire", + self.reacquire_line.__doc__) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + # find all the on-owned ports under your name + my_unowned_ports = list_difference([k for k, v in self.ports.items() if v.get_owner() == self.username], self.get_acquired_ports()) + if not my_unowned_ports: + self.logger.log("reacquire - no unowned ports under '{0}'".format(self.username)) + return + + self.acquire(ports = my_unowned_ports, force = True) + return True + + @__console def disconnect_line (self, line): self.disconnect() @@ -1978,12 +2186,24 @@ class STLClient(object): @__console def reset_line (self, line): - self.reset() + '''Reset ports - if no ports are provided all acquired ports will be reset''' + + parser = parsing_opts.gen_parser(self, + "reset", + self.reset_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True) + if opts is None: + return + + self.reset(ports = opts.ports) # true means print time return True + @__console def start_line (self, line): '''Start selected traffic on specified ports on TRex\n''' @@ -2000,15 +2220,11 @@ class STLClient(object): parsing_opts.MULTIPLIER_STRICT, parsing_opts.DRY_RUN) - opts = parser.parse_args(line.split()) - - + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True) if opts is None: return - - active_ports = list(set(self.get_active_ports()).intersection(opts.ports)) - + active_ports = list_intersect(self.get_active_ports(), opts.ports) if active_ports: if not opts.force: msg = "Port(s) {0} are active - please stop them or add '--force'\n".format(active_ports) @@ -2079,17 +2295,21 @@ class STLClient(object): self.stop_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_active_ports(), verify_acquired = True) if opts is None: return - # find the relevant ports - ports = list(set(self.get_active_ports()).intersection(opts.ports)) + # find the relevant ports + ports = list_intersect(opts.ports, self.get_active_ports()) if not ports: - self.logger.log(format_text("No active traffic on provided ports\n", 'bold')) + if not opts.ports: + self.logger.log('stop - no active ports') + else: + self.logger.log('stop - no active traffic on ports {0}'.format(opts.ports)) return + # call API self.stop(ports) # true means print time @@ -2107,15 +2327,18 @@ class STLClient(object): parsing_opts.TOTAL, parsing_opts.FORCE) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_active_ports(), verify_acquired = True) if opts is None: return - # find the relevant ports - ports = list(set(self.get_active_ports()).intersection(opts.ports)) + # find the relevant ports + ports = list_intersect(opts.ports, self.get_active_ports()) if not ports: - self.logger.log(format_text("No ports in valid state to update\n", 'bold')) + if not opts.ports: + self.logger.log('update - no active ports') + else: + self.logger.log('update - no active traffic on ports {0}'.format(opts.ports)) return self.update(ports, opts.mult, opts.total, opts.force) @@ -2132,15 +2355,22 @@ class STLClient(object): self.pause_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_transmitting_ports(), verify_acquired = True) if opts is None: return - # find the relevant ports - ports = list(set(self.get_transmitting_ports()).intersection(opts.ports)) + # check for already paused case + if opts.ports and is_sub_list(opts.ports, self.get_paused_ports()): + self.logger.log('pause - all of port(s) {0} are already paused'.format(opts.ports)) + return + # find the relevant ports + ports = list_intersect(opts.ports, self.get_transmitting_ports()) if not ports: - self.logger.log(format_text("No ports in valid state to pause\n", 'bold')) + if not opts.ports: + self.logger.log('pause - no transmitting ports') + else: + self.logger.log('pause - none of ports {0} are transmitting'.format(opts.ports)) return self.pause(ports) @@ -2157,18 +2387,21 @@ class STLClient(object): self.resume_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_paused_ports(), verify_acquired = True) if opts is None: return # find the relevant ports - ports = list(set(self.get_paused_ports()).intersection(opts.ports)) - + ports = list_intersect(opts.ports, self.get_paused_ports()) if not ports: - self.logger.log(format_text("No ports in valid state to resume\n", 'bold')) + if not opts.ports: + self.logger.log('resume - no paused ports') + else: + self.logger.log('resume - none of ports {0} are paused'.format(opts.ports)) return - return self.resume(ports) + + self.resume(ports) # true means print time return True @@ -2212,7 +2445,7 @@ class STLClient(object): mask = self.__get_mask_keys(**self.__filter_namespace_args(opts, trex_stl_stats.ALL_STATS_OPTS)) if not mask: # set to show all stats if no filter was given - mask = trex_stl_stats.ALL_STATS_OPTS + mask = trex_stl_stats.COMPACT stats_opts = common.list_intersect(trex_stl_stats.ALL_STATS_OPTS, mask) @@ -2320,15 +2553,20 @@ class STLClient(object): '''Sets port attributes ''' parser = parsing_opts.gen_parser(self, - "port", + "port_attr", self.set_port_attr_line.__doc__, parsing_opts.PORT_LIST_WITH_ALL, parsing_opts.PROMISCUOUS_SWITCH) - opts = parser.parse_args(line.split()) + opts = parser.parse_args(line.split(), default_ports = self.get_acquired_ports(), verify_acquired = True) if opts is None: return + # if no attributes - fall back to printing the status + if opts.prom is None: + self.show_stats_line("--ps --port {0}".format(' '.join(str(port) for port in opts.ports))) + return + self.set_port_attr(opts.ports, opts.prom) @@ -2371,3 +2609,54 @@ class STLClient(object): self.logger.log("") + + @__console + def get_events_line (self, line): + '''shows events recieved from server\n''' + + x = [parsing_opts.ArgumentPack(['-c','--clear'], + {'action' : "store_true", + 'default': False, + 'help': "clear the events log"}), + + parsing_opts.ArgumentPack(['-i','--info'], + {'action' : "store_true", + 'default': False, + 'help': "show info events"}), + + parsing_opts.ArgumentPack(['-w','--warn'], + {'action' : "store_true", + 'default': False, + 'help': "show warning events"}), + + ] + + + parser = parsing_opts.gen_parser(self, + "events", + self.get_events_line.__doc__, + *x) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + + ev_type_filter = [] + + if opts.info: + ev_type_filter.append('info') + + if opts.warn: + ev_type_filter.append('warning') + + if not ev_type_filter: + ev_type_filter = None + + events = self.get_events(ev_type_filter) + for ev in events: + self.logger.log(ev) + + if opts.clear: + self.clear_events() +
\ No newline at end of file diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py index b506137b..0afeff20 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py @@ -178,7 +178,6 @@ from .api import * from .trex_stl_types import * from .utils.common import get_number - class HLT_ERR(dict): def __init__(self, log = 'Unknown error', **kwargs): dict.__init__(self, {'status': 0}) @@ -201,7 +200,7 @@ def merge_kwargs(default_kwargs, user_kwargs): for key, value in user_kwargs.items(): if key in kwargs: kwargs[key] = value - elif key in ('save_to_yaml', 'save_to_pcap'): # internal debug arguments + elif key in ('save_to_yaml', 'save_to_pcap', 'pg_id'): # internal arguments kwargs[key] = value else: print("Warning: provided parameter '%s' is not supported" % key) @@ -305,7 +304,7 @@ class CStreamsPerPort(defaultdict): # save HLT args to modify streams later def save_stream_args(self, ports_list, stream_id, stream_hlt_args): - if stream_id is None: return # no stream_id, can't save TODO: remove this check ASAP + if stream_id is None: raise STLError('CStreamsPerPort: no stream_id in stream') if stream_hlt_args.get('load_profile'): return # can't modify profiles, don't save if not self.hlt_history: raise STLError('CStreamsPerPort: this object works only with HLT history, try init with hlt_history = True') if not is_integer(stream_id): raise STLError('CStreamsPerPort: stream_id should be number') @@ -313,10 +312,9 @@ class CStreamsPerPort(defaultdict): if not isinstance(ports_list, list): ports_list = [ports_list] for port in ports_list: - if stream_id in self[port]: - self[port][stream_id].update(stream_hlt_args) - else: - self[port][stream_id] = stream_hlt_args + if stream_id not in self[port]: + self[port][stream_id] = {} + self[port][stream_id].update(stream_hlt_args) def remove_stream(self, ports_list, stream_id): if not isinstance(ports_list, list): @@ -338,8 +336,11 @@ class CTRexHltApi(object): def __init__(self, verbose = 0): self.trex_client = None self.verbose = verbose - self._streams_history = {} # streams per stream_id per port in format of HLT arguments for modify later - + self._last_pg_id = 0 # pg_id acts as stream_handle + self._streams_history = {} # streams in format of HLT arguments for modify later + self._native_handle_by_pg_id = {} # pg_id -> native handle + port + self._pg_id_by_id = {} # stream_id -> pg_id + self._pg_id_by_name = {} # name -> pg_id ########################### # Session functions # @@ -371,13 +372,12 @@ class CTRexHltApi(object): try: port_list = self._parse_port_list(kwargs['port_list']) self.trex_client.acquire(ports = port_list, force = kwargs['break_locks']) + for port in port_list: + self._native_handle_by_pg_id[port] = {} except Exception as e: self.trex_client = None return HLT_ERR('Could not acquire ports %s: %s' % (port_list, e if isinstance(e, STLError) else traceback.format_exc())) - # since only supporting single TRex at the moment, 1:1 map - port_handle = self.trex_client.get_acquired_ports() - # arrived here, all desired ports were successfully acquired if kwargs['reset']: # remove all port traffic configuration from TRex @@ -389,7 +389,7 @@ class CTRexHltApi(object): return HLT_ERR('Error in reset traffic: %s' % e if isinstance(e, STLError) else traceback.format_exc()) self._streams_history = CStreamsPerPort(hlt_history = True) - return HLT_OK(port_handle = port_handle) + return HLT_OK(port_handle = dict([(port_id, port_id) for port_id in port_list])) def cleanup_session(self, **user_kwargs): kwargs = merge_kwargs(cleanup_session_kwargs, user_kwargs) @@ -446,23 +446,9 @@ class CTRexHltApi(object): kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs) stream_id = kwargs['stream_id'] mode = kwargs['mode'] - if type(stream_id) is list: - if len(stream_id) > 1: - streams_per_port = CStreamsPerPort() - for each_stream_id in stream_id: - user_kwargs[stream_id] = each_stream_id - res = self.traffic_config(**user_kwargs) - if type(res) is HLT_ERR: - return res - streams_per_port.add_streams_from_res(res) - if mode == 'create': - return HLT_OK(stream_id = streams_per_port) - else: - return HLT_OK() - else: - stream_id = stream_id[0] - + pg_id = None port_handle = port_list = self._parse_port_list(kwargs['port_handle']) + ALLOWED_MODES = ['create', 'modify', 'remove', 'enable', 'disable', 'reset'] if mode not in ALLOWED_MODES: return HLT_ERR('Mode must be one of the following values: %s' % ALLOWED_MODES) @@ -480,7 +466,7 @@ class CTRexHltApi(object): if mode == 'remove': if stream_id is None: return HLT_ERR('Please specify stream_id to remove.') - if type(stream_id) is str and stream_id == 'all': + if stream_id == 'all': try: self.trex_client.remove_all_streams(port_handle) for port in port_handle: @@ -504,14 +490,20 @@ class CTRexHltApi(object): # self._streams_history[stream_id].update(kwargs) # <- the modification if mode == 'modify': # we remove stream and create new one with same stream_id - stream_id = kwargs.get('stream_id') - if stream_id is None: + pg_id = kwargs.get('stream_id') + if pg_id is None: return HLT_ERR('Please specify stream_id to modify.') if len(port_handle) > 1: for port in port_handle: - user_kwargs[port_handle] = port - res = self.traffic_config(**user_kwargs) # recurse per port, each port can have different stream with such id + try: + user_kwargs['port_handle'] = port + res = self.traffic_config(**user_kwargs) + if res['status'] == 0: + return HLT_ERR('Error during modify of stream: %s' % res['log']) + except Exception as e: + return HLT_ERR('Could not remove stream(s) %s from port(s) %s: %s' % (stream_id, port_handle, e if isinstance(e, STLError) else traceback.format_exc())) + return HLT_OK() else: if type(port_handle) is list: port = port_handle[0] @@ -519,35 +511,37 @@ class CTRexHltApi(object): port = port_handle if port not in self._streams_history: return HLT_ERR('Port %s was not used/acquired' % port) - if stream_id not in self._streams_history[port]: + if pg_id not in self._streams_history[port]: return HLT_ERR('This stream_id (%s) was not used before at port %s, please create new.' % (stream_id, port)) - kwargs.update(self._streams_history[port][stream_id]) - kwargs.update(user_kwargs) + new_kwargs = {} + new_kwargs.update(self._streams_history[port][pg_id]) + new_kwargs.update(user_kwargs) + user_kwargs = new_kwargs try: - self.trex_client.remove_streams(stream_id, port_handle) + self._remove_stream(pg_id, [port]) except Exception as e: return HLT_ERR('Could not remove stream(s) %s from port(s) %s: %s' % (stream_id, port_handle, e if isinstance(e, STLError) else traceback.format_exc())) if mode == 'create' or mode == 'modify': # create a new stream with desired attributes, starting by creating packet - streams_per_port = CStreamsPerPort() if is_true(kwargs['bidirectional']): # two streams with opposite directions del user_kwargs['bidirectional'] + stream_per_port = {} save_to_yaml = user_kwargs.get('save_to_yaml') bidirect_err = 'When using bidirectional flag, ' if len(port_handle) != 1: - return HLT_ERR(bidirect_err + 'port_handle1 should be single port handle.') + return HLT_ERR(bidirect_err + 'port_handle should be single port handle.') + port_handle = port_handle[0] port_handle2 = kwargs['port_handle2'] if (type(port_handle2) is list and len(port_handle2) > 1) or port_handle2 is None: return HLT_ERR(bidirect_err + 'port_handle2 should be single port handle.') try: if save_to_yaml and type(save_to_yaml) is str: user_kwargs['save_to_yaml'] = save_to_yaml.replace('.yaml', '_bi1.yaml') - user_kwargs['port_handle'] = port_handle[0] res1 = self.traffic_config(**user_kwargs) if res1['status'] == 0: raise STLError('Could not create bidirectional stream 1: %s' % res1['log']) - streams_per_port.add_streams_from_res(res1) + stream_per_port[port_handle] = res1['stream_id'] kwargs['direction'] = 1 - kwargs['direction'] # not correct_direction(user_kwargs, kwargs) if save_to_yaml and type(save_to_yaml) is str: @@ -556,37 +550,38 @@ class CTRexHltApi(object): res2 = self.traffic_config(**user_kwargs) if res2['status'] == 0: raise STLError('Could not create bidirectional stream 2: %s' % res2['log']) - streams_per_port.add_streams_from_res(res2) + stream_per_port[port_handle2] = res2['stream_id'] except Exception as e: return HLT_ERR('Could not generate bidirectional traffic: %s' % e if isinstance(e, STLError) else traceback.format_exc()) if mode == 'create': - return HLT_OK(stream_id = streams_per_port) + return HLT_OK(stream_id = stream_per_port) else: return HLT_OK() try: + if not pg_id: + pg_id = self._get_available_pg_id() if kwargs['load_profile']: stream_obj = STLProfile.load_py(kwargs['load_profile'], direction = kwargs['direction']) else: + user_kwargs['pg_id'] = pg_id stream_obj = STLHltStream(**user_kwargs) except Exception as e: return HLT_ERR('Could not create stream: %s' % e if isinstance(e, STLError) else traceback.format_exc()) # try adding the stream per ports try: - stream_id_arr = self.trex_client.add_streams(streams = stream_obj, - ports = port_handle) - if type(stream_id_arr) is not list: - stream_id_arr = [stream_id_arr] for port in port_handle: - self._streams_history.save_stream_args(port_handle, stream_id_arr[0], user_kwargs) + stream_id_arr = self.trex_client.add_streams(streams = stream_obj, + ports = port) + self._streams_history.save_stream_args(port, pg_id, user_kwargs) + if type(stream_id_arr) is not list: + stream_id_arr = [stream_id_arr] + self._native_handle_by_pg_id[port][pg_id] = stream_id_arr except Exception as e: return HLT_ERR('Could not add stream to ports: %s' % e if isinstance(e, STLError) else traceback.format_exc()) if mode == 'create': - if len(stream_id_arr) == 1: - return HLT_OK(stream_id = dict((port, stream_id_arr[0]) for port in port_handle)) - else: - return HLT_OK(stream_id = dict((port, stream_id_arr) for port in port_handle)) + return HLT_OK(stream_id = pg_id) else: return HLT_OK() @@ -652,42 +647,64 @@ class CTRexHltApi(object): kwargs = merge_kwargs(traffic_stats_kwargs, user_kwargs) mode = kwargs['mode'] port_handle = kwargs['port_handle'] + if type(port_handle) is not list: + port_handle = [port_handle] ALLOWED_MODES = ['aggregate', 'streams', 'all'] if mode not in ALLOWED_MODES: return HLT_ERR("'mode' must be one of the following values: %s" % ALLOWED_MODES) - if mode == 'streams': - return HLT_ERR("mode 'streams' not implemented'") - if mode in ('all', 'aggregate'): - hlt_stats_dict = {} - try: - stats = self.trex_client.get_stats(port_handle) - except Exception as e: - return HLT_ERR('Could not retrieve stats: %s' % e if isinstance(e, STLError) else traceback.format_exc()) - for port_id, stat_dict in stats.items(): - if is_integer(port_id): - hlt_stats_dict[port_id] = { - 'aggregate': { - 'tx': { - 'pkt_bit_rate': stat_dict.get('tx_bps'), - 'pkt_byte_count': stat_dict.get('obytes'), - 'pkt_count': stat_dict.get('opackets'), - 'pkt_rate': stat_dict.get('tx_pps'), - 'total_pkt_bytes': stat_dict.get('obytes'), - 'total_pkt_rate': stat_dict.get('tx_pps'), - 'total_pkts': stat_dict.get('opackets'), - }, - 'rx': { - 'pkt_bit_rate': stat_dict.get('rx_bps'), - 'pkt_byte_count': stat_dict.get('ibytes'), - 'pkt_count': stat_dict.get('ipackets'), - 'pkt_rate': stat_dict.get('rx_pps'), - 'total_pkt_bytes': stat_dict.get('ibytes'), - 'total_pkt_rate': stat_dict.get('rx_pps'), - 'total_pkts': stat_dict.get('ipackets'), + hlt_stats_dict = dict([(port, {}) for port in port_handle]) + try: + stats = self.trex_client.get_stats(port_handle) + if mode in ('all', 'aggregate'): + for port_id in port_handle: + port_stats = stats[port_id] + if is_integer(port_id): + hlt_stats_dict[port_id]['aggregate'] = { + 'tx': { + 'pkt_bit_rate': port_stats.get('tx_bps', 0), + 'pkt_byte_count': port_stats.get('obytes', 0), + 'pkt_count': port_stats.get('opackets', 0), + 'pkt_rate': port_stats.get('tx_pps', 0), + 'total_pkt_bytes': port_stats.get('obytes', 0), + 'total_pkt_rate': port_stats.get('tx_pps', 0), + 'total_pkts': port_stats.get('opackets', 0), + }, + 'rx': { + 'pkt_bit_rate': port_stats.get('rx_bps', 0), + 'pkt_byte_count': port_stats.get('ibytes', 0), + 'pkt_count': port_stats.get('ipackets', 0), + 'pkt_rate': port_stats.get('rx_pps', 0), + 'total_pkt_bytes': port_stats.get('ibytes', 0), + 'total_pkt_rate': port_stats.get('rx_pps', 0), + 'total_pkts': port_stats.get('ipackets', 0), + } } - } - } - return HLT_OK(hlt_stats_dict) + if mode in ('all', 'streams'): + for pg_id, pg_stats in stats['flow_stats'].items(): + for port_id in port_handle: + if 'stream' not in hlt_stats_dict[port_id]: + hlt_stats_dict[port_id]['stream'] = {} + hlt_stats_dict[port_id]['stream'][pg_id] = { + 'tx': { + 'total_pkts': pg_stats['tx_pkts'].get(port_id, 0), + 'total_pkt_bytes': pg_stats['tx_bytes'].get(port_id, 0), + 'total_pkts_bytes': pg_stats['tx_bytes'].get(port_id, 0), + 'total_pkt_bit_rate': pg_stats['tx_bps'].get(port_id, 0), + 'total_pkt_rate': pg_stats['tx_pps'].get(port_id, 0), + 'line_rate_percentage': pg_stats['tx_line_util'].get(port_id, 0), + }, + 'rx': { + 'total_pkts': pg_stats['rx_pkts'].get(port_id, 0), + 'total_pkt_bytes': pg_stats['rx_bytes'].get(port_id, 0), + 'total_pkts_bytes': pg_stats['rx_bytes'].get(port_id, 0), + 'total_pkt_bit_rate': pg_stats['rx_bps'].get(port_id, 0), + 'total_pkt_rate': pg_stats['rx_pps'].get(port_id, 0), + 'line_rate_percentage': pg_stats['rx_line_util'].get(port_id, 0), + }, + } + except Exception as e: + return HLT_ERR('Could not retrieve stats: %s' % e if isinstance(e, STLError) else traceback.format_exc()) + return HLT_OK(hlt_stats_dict) # timeout = maximal time to wait def wait_on_traffic(self, port_handle = None, timeout = 60): @@ -700,16 +717,31 @@ class CTRexHltApi(object): # Private functions # ########################### + def _get_available_pg_id(self): + pg_id = self._last_pg_id + used_pg_ids = self.trex_client.get_stats()['flow_stats'].keys() + for i in range(65535): + pg_id += 1 + if pg_id not in used_pg_ids: + self._last_pg_id = pg_id + return pg_id + if pg_id == 65535: + pg_id = 0 + raise STLError('Could not find free pg_id in range [1, 65535].') + # remove streams from given port(s). # stream_id can be: # * int - exact stream_id value # * list - list of stream_id values or strings (see below) # * string - exact stream_id value, mix of ranges/list separated by comma: 2, 4-13 def _remove_stream(self, stream_id, port_handle): - if get_number(stream_id) is not None: # exact value of int or str - self.trex_client.remove_streams(get_number(stream_id), port_handle) # actual remove + stream_num = get_number(stream_id) + if stream_num is not None: # exact value of int or str for port in port_handle: - del self._streams_history[port][get_number(stream_id)] + native_handles = self._native_handle_by_pg_id[port][stream_num] + self.trex_client.remove_streams(native_handles, port) # actual remove + del self._native_handle_by_pg_id[port][stream_num] + del self._streams_history[port][stream_num] return if type(stream_id) is list: # list of values/strings for each_stream_id in stream_id: @@ -733,7 +765,7 @@ class CTRexHltApi(object): for each_stream_id in xrange(stream_id_min, stream_id_max + 1): self._remove_stream(each_stream_id, port_handle) # recurse return - raise STLError('_remove_stream: wrong param %s' % stream_id) + raise STLError('_remove_stream: wrong stream_id param %s' % stream_id) @staticmethod def _parse_port_list(port_list): @@ -812,32 +844,33 @@ def STLHltStream(**user_kwargs): # packet generation packet = generate_packet(**user_kwargs) + + # stream generation try: rate_types_dict = {'rate_pps': 'pps', 'rate_bps': 'bps_L2', 'rate_percent': 'percentage'} rate_stateless = {rate_types_dict[rate_key]: float(kwargs[rate_key])} transmit_mode = kwargs['transmit_mode'] pkts_per_burst = kwargs['pkts_per_burst'] if transmit_mode == 'continuous': - transmit_mode_class = STLTXCont(**rate_stateless) + transmit_mode_obj = STLTXCont(**rate_stateless) elif transmit_mode == 'single_burst': - transmit_mode_class = STLTXSingleBurst(total_pkts = pkts_per_burst, **rate_stateless) + transmit_mode_obj = STLTXSingleBurst(total_pkts = pkts_per_burst, **rate_stateless) elif transmit_mode == 'multi_burst': - transmit_mode_class = STLTXMultiBurst(total_pkts = pkts_per_burst, count = int(kwargs['burst_loop_count']), + transmit_mode_obj = STLTXMultiBurst(total_pkts = pkts_per_burst, count = int(kwargs['burst_loop_count']), ibg = kwargs['inter_burst_gap'], **rate_stateless) else: raise STLError('transmit_mode %s not supported/implemented') except Exception as e: - raise STLError('Could not create transmit_mode class %s: %s' % (transmit_mode, e if isinstance(e, STLError) else traceback.format_exc())) + raise STLError('Could not create transmit_mode object %s: %s' % (transmit_mode, e if isinstance(e, STLError) else traceback.format_exc())) - # stream generation try: + pg_id = kwargs.get('pg_id') stream = STLStream(packet = packet, random_seed = 1 if is_true(kwargs['consistent_random']) else 0, #enabled = True, #self_start = True, - mode = transmit_mode_class, - stream_id = kwargs['stream_id'], - name = kwargs['name'], + flow_stats = STLFlowStats(pg_id) if pg_id else None, + mode = transmit_mode_obj, ) except Exception as e: raise STLError('Could not create stream: %s' % e if isinstance(e, STLError) else traceback.format_exc()) @@ -848,8 +881,12 @@ def STLHltStream(**user_kwargs): stream.dump_to_yaml(debug_filename) return stream +packet_cache = LRU_cache(maxlen = 20) + def generate_packet(**user_kwargs): correct_macs(user_kwargs) + if repr(user_kwargs) in packet_cache: + return packet_cache[repr(user_kwargs)] kwargs = merge_kwargs(traffic_config_kwargs, user_kwargs) correct_sizes(kwargs) # we are producing the packet - 4 bytes fcs correct_direction(kwargs, kwargs) @@ -868,8 +905,12 @@ def generate_packet(**user_kwargs): kwargs['mac_dst'] = None kwargs['mac_src_mode'] = 'fixed' kwargs['mac_dst_mode'] = 'fixed' - - l2_layer = Ether(src = kwargs['mac_src'], dst = kwargs['mac_dst']) + ethernet_kwargs = {} + if kwargs['mac_src']: + ethernet_kwargs['src'] = kwargs['mac_src'] + if kwargs['mac_dst']: + ethernet_kwargs['dst'] = kwargs['mac_dst'] + l2_layer = Ether(**ethernet_kwargs) # Eth VM, change only 32 lsb if kwargs['mac_src_mode'] != 'fixed': @@ -1026,7 +1067,7 @@ def generate_packet(**user_kwargs): if ip_tos < 0 or ip_tos > 255: raise STLError('TOS %s is not in range 0-255' % ip_tos) l3_layer = IP(tos = ip_tos, - len = kwargs['l3_length'], + #len = kwargs['l3_length'], don't let user create corrupt packets id = kwargs['ip_id'], frag = kwargs['ip_fragment_offset'], ttl = kwargs['ip_ttl'], @@ -1475,6 +1516,7 @@ def generate_packet(**user_kwargs): debug_filename = kwargs.get('save_to_pcap') if type(debug_filename) is str: pkt.dump_pkt_to_pcap(debug_filename) + packet_cache[repr(user_kwargs)] = pkt return pkt def get_TOS(user_kwargs, kwargs): @@ -1542,7 +1584,7 @@ def correct_direction(user_kwargs, kwargs): # we produce packets without fcs, so need to reduce produced sizes def correct_sizes(kwargs): - for arg in kwargs.keys(): - if is_integer(arg): + for arg, value in kwargs.items(): + if is_integer(value): if arg.endswith(('_length', '_size', '_size_min', '_size_max', '_length_min', '_length_max')): kwargs[arg] -= 4 diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py index 89ad2663..5cf94bda 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_port.py @@ -64,6 +64,8 @@ class Port(object): self.tx_stopped_ts = None self.has_rx_streams = False + self.owner = '' + def err(self, msg): return RC_ERR("port {0} : {1}\n".format(self.port_id, msg)) @@ -74,6 +76,9 @@ class Port(object): def get_speed_bps (self): return (self.info['speed'] * 1000 * 1000 * 1000) + def get_formatted_speed (self): + return "{0} Gbps".format(self.info['speed']) + # take the port def acquire(self, force = False): params = {"port_id": self.port_id, @@ -94,9 +99,12 @@ class Port(object): "handler": self.handler} rc = self.transmit("release", params) - self.handler = None - + if rc.good(): + + self.handler = None + self.owner = '' + return self.ok() else: return self.err(rc.err()) @@ -113,6 +121,11 @@ class Port(object): def is_paused (self): return (self.state == self.STATE_PAUSE) + def get_owner (self): + if self.is_acquired(): + return self.user + else: + return self.owner def sync(self): params = {"port_id": self.port_id} @@ -137,6 +150,7 @@ class Port(object): else: raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state)) + self.owner = rc.data()['owner'] self.next_available_id = int(rc.data()['max_stream_id']) + 1 @@ -231,10 +245,11 @@ class Port(object): if single_rc.rc: stream_id = batch[i].params['stream_id'] next_id = batch[i].params['stream']['next_stream_id'] - self.streams[stream_id] = {'next_id' : next_id, - 'pkt' : streams_list[i].get_pkt(), - 'mode' : streams_list[i].get_mode(), - 'rate' : streams_list[i].get_rate()} + self.streams[stream_id] = {'next_id' : next_id, + 'pkt' : streams_list[i].get_pkt(), + 'mode' : streams_list[i].get_mode(), + 'rate' : streams_list[i].get_rate(), + 'has_flow_stats' : streams_list[i].has_flow_stats()} ret.add(RC_OK(data = stream_id)) @@ -280,12 +295,12 @@ class Port(object): for i, single_rc in enumerate(rc): if single_rc: id = batch[i].params['stream_id'] - del self.streams[stream_id] + del self.streams[id] self.state = self.STATE_STREAMS if (len(self.streams) > 0) else self.STATE_IDLE # recheck if any RX stats streams present on the port - self.has_rx_streams = any([stream.has_flow_stats() for stream in self.streams]) + self.has_rx_streams = any([stream['has_flow_stats'] for stream in self.streams.values()]) return self.ok() if rc else self.err(rc.err()) @@ -670,6 +685,10 @@ class Port(object): if not self.is_acquired(): self.state = self.STATE_TX - def async_event_forced_acquired (self): + def async_event_acquired (self, who): self.handler = None + self.owner = who + + def async_event_released (self): + self.owner = '' diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py index 1d89a599..11e80b9a 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py @@ -22,6 +22,7 @@ from .trex_stl_streams import * from .utils import parsing_opts from .trex_stl_client import STLClient from .utils import pcap +from trex_stl_lib.trex_stl_packet_builder_scapy import RawPcapReader, RawPcapWriter, hexdump from yaml import YAMLError @@ -291,13 +292,13 @@ class STLSim(object): return - print("Mering cores output to a single pcap file...\n") + if not self.silent: + print("Mering cores output to a single pcap file...\n") inputs = ["{0}-{1}".format(self.outfile, index) for index in range(0, self.dp_core_count)] pcap.merge_cap_files(inputs, self.outfile, delete_src = True) - def is_valid_file(filename): if not os.path.isfile(filename): raise argparse.ArgumentTypeError("The file '%s' does not exist" % filename) @@ -421,6 +422,11 @@ def setParserOptions(): action = "store_true", default = False) + group.add_argument("--test_multi_core", + help = "runs the profile with c=1-8", + action = "store_true", + default = False) + return parser @@ -435,6 +441,110 @@ def validate_args (parser, options): parser.error("limit cannot be lower than number of DP cores") +# a more flexible check +def compare_caps (cap1, cap2, max_diff_sec = (5 * 1e-6)): + pkts1 = list(RawPcapReader(cap1)) + pkts2 = list(RawPcapReader(cap2)) + + if len(pkts1) != len(pkts2): + print('{0} contains {1} packets vs. {2} contains {3} packets'.format(cap1, len(pkts1), cap2, len(pkts2))) + return False + + # to be less strict we define equality if all packets from cap1 exists and in cap2 + # and vice versa + # 'exists' means the same packet with abs(TS1-TS2) < 5nsec + # its O(n^2) but who cares, right ? + for i, pkt1 in enumerate(pkts1): + ts1 = float(pkt1[1][0]) + (float(pkt1[1][1]) / 1e6) + found = None + for j, pkt2 in enumerate(pkts2): + ts2 = float(pkt2[1][0]) + (float(pkt2[1][1]) / 1e6) + + if abs(ts1-ts2) > max_diff_sec: + break + + if pkt1[0] == pkt2[0]: + found = j + break + + + if found is None: + print(format_text("cannot find packet #{0} from {1} in {2}\n".format(i, cap1, cap2), 'bold')) + return False + else: + del pkts2[found] + + return True + + + + +# a more strict comparsion 1 <--> 1 +def compare_caps_strict (cap1, cap2, max_diff_sec = (5 * 1e-6)): + pkts1 = list(RawPcapReader(cap1)) + pkts2 = list(RawPcapReader(cap2)) + + if len(pkts1) != len(pkts2): + print('{0} contains {1} packets vs. {1} contains {2} packets'.format(cap1, len(pkts1), cap2, len(pkts2))) + return False + + # a strict check + for pkt1, pkt2, i in zip(pkts1, pkts2, range(1, len(pkts1))): + ts1 = float(pkt1[1][0]) + (float(pkt1[1][1]) / 1e6) + ts2 = float(pkt2[1][0]) + (float(pkt2[1][1]) / 1e6) + + if abs(ts1-ts2) > 0.000005: # 5 nsec + print(format_text("TS error: cap files '{0}', '{1}' differ in cap #{2} - '{3}' vs. '{4}'\n".format(cap1, cap2, i, ts1, ts2), 'bold')) + return False + + if pkt1[0] != pkt2[0]: + print(format_text("RAW error: cap files '{0}', '{1}' differ in cap #{2}\n".format(cap1, cap2, i), 'bold')) + print(hexdump(pkt1[0])) + print("") + print(hexdump(pkt2[0])) + print("") + return False + + return True + +# +def test_multi_core (r, options): + + for core_count in [1, 2, 4, 6, 8]: + r.run(input_list = options.input_file, + outfile = '{0}.cap'.format(core_count), + dp_core_count = core_count, + is_debug = (not options.release), + pkt_limit = options.limit, + mult = options.mult, + duration = options.duration, + mode = 'none', + silent = True, + tunables = options.tunables) + + print("") + + print(format_text("comparing 2 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '2.cap') + if rc: + print("[Passed]\n") + + print(format_text("comparing 4 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '4.cap') + if rc: + print("[Passed]\n") + + print(format_text("comparing 6 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '6.cap') + if rc: + print("[Passed]\n") + + print(format_text("comparing 8 cores to 1 core:\n", 'underline')) + rc = compare_caps('1.cap', '8.cap') + if rc: + print("[Passed]\n") + + def main (args = None): parser = setParserOptions() options = parser.parse_args(args = args) @@ -455,23 +565,28 @@ def main (args = None): mode = 'native' elif options.pkt: mode = 'pkt' + elif options.test_multi_core: + mode = 'test_multi_core' else: mode = 'none' try: r = STLSim(bp_sim_path = options.bp_sim_path, port_id = options.port_id) - r.run(input_list = options.input_file, - outfile = options.output_file, - dp_core_count = options.dp_core_count, - dp_core_index = options.dp_core_index, - is_debug = (not options.release), - pkt_limit = options.limit, - mult = options.mult, - duration = options.duration, - mode = mode, - silent = options.silent, - tunables = options.tunables) + if mode == 'test_multi_core': + test_multi_core(r, options) + else: + r.run(input_list = options.input_file, + outfile = options.output_file, + dp_core_count = options.dp_core_count, + dp_core_index = options.dp_core_index, + is_debug = (not options.release), + pkt_limit = options.limit, + mult = options.mult, + duration = options.duration, + mode = mode, + silent = options.silent, + tunables = options.tunables) except KeyboardInterrupt as e: print("\n\n*** Caught Ctrl + C... Exiting...\n\n") diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py index a4bb64db..42bef360 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py @@ -2,6 +2,7 @@ from .utils import text_tables from .utils.text_opts import format_text, format_threshold, format_num +from .trex_stl_types import StatNotAvailable from collections import namedtuple, OrderedDict, deque import sys @@ -16,11 +17,13 @@ import pprint GLOBAL_STATS = 'g' PORT_STATS = 'p' +PORT_GRAPH = 'pg' PORT_STATUS = 'ps' STREAMS_STATS = 's' -ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS] +ALL_STATS_OPTS = [GLOBAL_STATS, PORT_STATS, PORT_STATUS, STREAMS_STATS, PORT_GRAPH] COMPACT = [GLOBAL_STATS, PORT_STATS] +GRAPH_PORT_COMPACT = [GLOBAL_STATS, PORT_GRAPH] SS_COMPAT = [GLOBAL_STATS, STREAMS_STATS] ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) @@ -88,6 +91,33 @@ def calculate_diff_raw (samples): return total +# a simple object to keep a watch over a field +class WatchedField(object): + + def __init__ (self, name, suffix, high_th, low_th, events_handler): + self.name = name + self.suffix = suffix + self.high_th = high_th + self.low_th = low_th + self.events_handler = events_handler + + self.hot = False + self.current = None + + def update (self, value): + if value is None: + return + + if value > self.high_th and not self.hot: + self.events_handler.log_warning("{0} is high: {1}{2}".format(self.name, value, self.suffix)) + self.hot = True + + if value < self.low_th and self.hot: + self.hot = False + + self.current = value + + class CTRexInfoGenerator(object): """ @@ -107,11 +137,15 @@ class CTRexInfoGenerator(object): elif statistic_type == PORT_STATS: return self._generate_port_stats(port_id_list) + elif statistic_type == PORT_GRAPH: + return self._generate_port_graph(port_id_list) + elif statistic_type == PORT_STATUS: return self._generate_port_status(port_id_list) elif statistic_type == STREAMS_STATS: return self._generate_streams_stats() + else: # ignore by returning empty object return {} @@ -163,70 +197,53 @@ class CTRexInfoGenerator(object): return {"streams_statistics": ExportableStats(sstats_data, stats_table)} - - - per_stream_stats = OrderedDict([("owner", []), - ("state", []), - ("--", []), - ("Tx bps L2", []), - ("Tx bps L1", []), - ("Tx pps", []), - ("Line Util.", []), - - ("---", []), - ("Rx bps", []), - ("Rx pps", []), - - ("----", []), - ("opackets", []), - ("ipackets", []), - ("obytes", []), - ("ibytes", []), - ("tx-bytes", []), - ("rx-bytes", []), - ("tx-pkts", []), - ("rx-pkts", []), - - ("-----", []), - ("oerrors", []), - ("ierrors", []), - - ] - ) - - total_stats = CPortStats(None) - - for port_obj in relevant_ports: - # fetch port data - port_stats = port_obj.generate_port_stats() - - total_stats += port_obj.port_stats - - # populate to data structures - return_stats_data[port_obj.port_id] = port_stats - self.__update_per_field_dict(port_stats, per_field_stats) - - total_cols = len(relevant_ports) - header = ["port"] + [port.port_id for port in relevant_ports] + @staticmethod + def _get_rational_block_char(value, range_start, interval): + # in Konsole, utf-8 is sometimes printed with artifacts, return ascii for now + #return 'X' if value >= range_start + float(interval) / 2 else ' ' + + if sys.__stdout__.encoding != 'UTF-8': + return 'X' if value >= range_start + float(interval) / 2 else ' ' + + value -= range_start + ratio = float(value) / interval + if ratio <= 0.0625: + return u' ' # empty block + if ratio <= 0.1875: + return u'\u2581' # 1/8 + if ratio <= 0.3125: + return u'\u2582' # 2/4 + if ratio <= 0.4375: + return u'\u2583' # 3/8 + if ratio <= 0.5625: + return u'\u2584' # 4/8 + if ratio <= 0.6875: + return u'\u2585' # 5/8 + if ratio <= 0.8125: + return u'\u2586' # 6/8 + if ratio <= 0.9375: + return u'\u2587' # 7/8 + return u'\u2588' # full block + + def _generate_port_graph(self, port_id_list): + relevant_port = self.__get_relevant_ports(port_id_list)[0] + hist_len = len(relevant_port.port_stats.history) + hist_maxlen = relevant_port.port_stats.history.maxlen + util_tx_hist = [0] * (hist_maxlen - hist_len) + [round(relevant_port.port_stats.history[i]['tx_percentage']) for i in range(hist_len)] + util_rx_hist = [0] * (hist_maxlen - hist_len) + [round(relevant_port.port_stats.history[i]['rx_percentage']) for i in range(hist_len)] - if (total_cols > 1): - self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats) - header += ['total'] - total_cols += 1 stats_table = text_tables.TRexTextTable() - stats_table.set_cols_align(["l"] + ["r"] * total_cols) - stats_table.set_cols_width([10] + [17] * total_cols) - stats_table.set_cols_dtype(['t'] + ['t'] * total_cols) - - stats_table.add_rows([[k] + v - for k, v in per_field_stats.items()], - header=False) + stats_table.header([' Util(%)', 'TX', 'RX']) + stats_table.set_cols_align(['c', 'c', 'c']) + stats_table.set_cols_width([8, hist_maxlen, hist_maxlen]) + stats_table.set_cols_dtype(['t', 't', 't']) - stats_table.header(header) - - return {"streams_statistics": ExportableStats(return_stats_data, stats_table)} + for y in range(95, -1, -5): + stats_table.add_row([y, ''.join([self._get_rational_block_char(util_tx, y, 5) for util_tx in util_tx_hist]), + ''.join([self._get_rational_block_char(util_rx, y, 5) for util_rx in util_rx_hist])]) + return {"port_graph": ExportableStats({}, stats_table)} def _generate_port_stats(self, port_id_list): relevant_ports = self.__get_relevant_ports(port_id_list) @@ -234,6 +251,7 @@ class CTRexInfoGenerator(object): return_stats_data = {} per_field_stats = OrderedDict([("owner", []), ("state", []), + ("speed", []), ("--", []), ("Tx bps L2", []), ("Tx bps L1", []), @@ -374,7 +392,8 @@ class CTRexInfoGenerator(object): # display only the first FOUR options, by design if len(ports) > 4: - self.logger.log(format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta')) + #self.logger is not defined + #self.logger.log(format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta')) ports = ports[:4] return ports @@ -400,7 +419,7 @@ class CTRexStats(object): self.reference_stats = {} self.latest_stats = {} self.last_update_ts = time.time() - self.history = deque(maxlen = 10) + self.history = deque(maxlen = 47) self.lock = threading.Lock() self.has_baseline = False @@ -444,6 +463,7 @@ class CTRexStats(object): def clear_stats(self): self.reference_stats = copy.deepcopy(self.latest_stats) + self.history.clear() def invalidate (self): @@ -469,19 +489,18 @@ class CTRexStats(object): def get(self, field, format=False, suffix=""): value = self._get(self.latest_stats, field) if value == None: - return "N/A" + return 'N/A' return value if not format else format_num(value, suffix) def get_rel(self, field, format=False, suffix=""): - ref_value = self._get(self.reference_stats, field) latest_value = self._get(self.latest_stats, field) # latest value is an aggregation - must contain the value if latest_value == None: - return "N/A" + return 'N/A' if ref_value == None: ref_value = 0 @@ -493,7 +512,7 @@ class CTRexStats(object): # get trend for a field def get_trend (self, field, use_raw = False, percision = 10.0): - if not field in self.latest_stats: + if field not in self.latest_stats: return 0 # not enough history - no trend @@ -506,7 +525,7 @@ class CTRexStats(object): # must lock, deque is not thread-safe for iteration with self.lock: - field_samples = [sample[field] for sample in self.history] + field_samples = [sample[field] for sample in list(self.history)[-5:]] if use_raw: return calculate_diff_raw(field_samples) @@ -518,7 +537,12 @@ class CTRexStats(object): v = self.get_trend(field, use_raw) value = abs(v) - arrow = u'\u25b2' if v > 0 else u'\u25bc' + + # use arrows if utf-8 is supported + if sys.__stdout__.encoding == 'UTF-8': + arrow = u'\u25b2' if v > 0 else u'\u25bc' + else: + arrow = '' if sys.version_info < (3,0): arrow = arrow.encode('utf-8') @@ -553,17 +577,24 @@ class CTRexStats(object): class CGlobalStats(CTRexStats): - def __init__(self, connection_info, server_version, ports_dict_ref): + def __init__(self, connection_info, server_version, ports_dict_ref, events_handler): super(CGlobalStats, self).__init__() + self.connection_info = connection_info - self.server_version = server_version - self._ports_dict = ports_dict_ref + self.server_version = server_version + self._ports_dict = ports_dict_ref + self.events_handler = events_handler + + self.watched_cpu_util = WatchedField('CPU util.', '%', 85, 60, events_handler) + self.watched_rx_cpu_util = WatchedField('RX core util.', '%', 85, 60, events_handler) def get_stats (self): stats = {} # absolute - stats['cpu_util'] = self.get("m_cpu_util") + stats['cpu_util'] = self.get("m_cpu_util") + stats['rx_cpu_util'] = self.get("m_rx_cpu_util") + stats['tx_bps'] = self.get("m_tx_bps") stats['tx_pps'] = self.get("m_tx_pps") @@ -589,6 +620,9 @@ class CGlobalStats(CTRexStats): # simple... self.latest_stats = snapshot + self.watched_cpu_util.update(snapshot.get('m_cpu_util')) + self.watched_rx_cpu_util.update(snapshot.get('m_rx_cpu_util')) + return True @@ -601,6 +635,9 @@ class CGlobalStats(CTRexStats): ("cpu_util", "{0}% {1}".format( format_threshold(self.get("m_cpu_util"), [85, 100], [0, 85]), self.get_trend_gui("m_cpu_util", use_raw = True))), + ("rx_cpu_util", "{0}% {1}".format( format_threshold(self.get("m_rx_cpu_util"), [85, 100], [0, 85]), + self.get_trend_gui("m_rx_cpu_util", use_raw = True))), + (" ", ""), ("total_tx_L2", "{0} {1}".format( self.get("m_tx_bps", format=True, suffix="b/sec"), @@ -694,11 +731,27 @@ class CPortStats(CTRexStats): # L1 bps bps = snapshot.get("m_total_tx_bps") pps = snapshot.get("m_total_tx_pps") + rx_bps = snapshot.get("m_total_rx_bps") + rx_pps = snapshot.get("m_total_rx_pps") + ts_diff = 0.5 # TODO: change this to real ts diff from server bps_L1 = calc_bps_L1(bps, pps) + bps_rx_L1 = calc_bps_L1(rx_bps, rx_pps) snapshot['m_total_tx_bps_L1'] = bps_L1 snapshot['m_percentage'] = (bps_L1 / self._port_obj.get_speed_bps()) * 100 + # TX line util not smoothed + diff_tx_pkts = snapshot.get('opackets', 0) - self.latest_stats.get('opackets', 0) + diff_tx_bytes = snapshot.get('obytes', 0) - self.latest_stats.get('obytes', 0) + tx_bps_L1 = calc_bps_L1(8.0 * diff_tx_bytes / ts_diff, float(diff_tx_pkts) / ts_diff) + snapshot['tx_percentage'] = 100.0 * tx_bps_L1 / self._port_obj.get_speed_bps() + + # RX line util not smoothed + diff_rx_pkts = snapshot.get('ipackets', 0) - self.latest_stats.get('ipackets', 0) + diff_rx_bytes = snapshot.get('ibytes', 0) - self.latest_stats.get('ibytes', 0) + rx_bps_L1 = calc_bps_L1(8.0 * diff_rx_bytes / ts_diff, float(diff_rx_pkts) / ts_diff) + snapshot['rx_percentage'] = 100.0 * rx_bps_L1 / self._port_obj.get_speed_bps() + # simple... self.latest_stats = snapshot @@ -715,9 +768,17 @@ class CPortStats(CTRexStats): else: state = format_text(state, 'bold') + # mark owned ports by color + if self._port_obj: + owner = self._port_obj.get_owner() + if self._port_obj.is_acquired(): + owner = format_text(owner, 'green') + else: + owner = '' - return {"owner": self._port_obj.user if self._port_obj else "", + return {"owner": owner, "state": "{0}".format(state), + "speed": self._port_obj.get_formatted_speed() if self._port_obj else '', "--": " ", "---": " ", @@ -769,9 +830,15 @@ class CPortStats(CTRexStats): # RX stats objects - COMPLEX :-( class CRxStats(CTRexStats): - def __init__(self): + def __init__(self, ports): super(CRxStats, self).__init__() + self.ports = ports + self.ports_speed = {} + def get_ports_speed(self): + for port in self.ports: + self.ports_speed[str(port)] = self.ports[port].get_speed_bps() + self.ports_speed['total'] = sum(self.ports_speed.values()) # calculates a diff between previous snapshot # and current one @@ -797,7 +864,7 @@ class CRxStats(CTRexStats): for field in ['tx_pkts', 'tx_bytes', 'rx_pkts', 'rx_bytes']: # is in the first time ? (nothing in prev) - if not field in output: + if field not in output: output[field] = {} # does the current snapshot has this field ? @@ -832,7 +899,7 @@ class CRxStats(CTRexStats): output['ts'] = current['ts'] # we care only about the current active keys - pg_ids = filter(is_intable, current.keys()) + pg_ids = list(filter(is_intable, current.keys())) for pg_id in pg_ids: @@ -858,7 +925,7 @@ class CRxStats(CTRexStats): # cleanp old reference values - they are dead - ref_pg_ids = filter(is_intable, self.reference_stats.keys()) + ref_pg_ids = list(filter(is_intable, self.reference_stats.keys())) deleted_pg_ids = set(ref_pg_ids).difference(pg_ids) for d_pg_id in deleted_pg_ids: @@ -869,46 +936,66 @@ class CRxStats(CTRexStats): def calculate_bw_for_pg (self, pg_current, pg_prev = None, diff_sec = 0.0): - - # if no previous values - its None + # no previous values if (pg_prev == None) or not (diff_sec > 0): - pg_current['tx_pps'] = None - pg_current['tx_bps'] = None - pg_current['tx_bps_L1'] = None - pg_current['rx_pps'] = None - pg_current['rx_bps'] = None + pg_current['tx_pps'] = {} + pg_current['tx_bps'] = {} + pg_current['tx_bps_L1'] = {} + pg_current['tx_line_util'] = {} + pg_current['rx_pps'] = {} + pg_current['rx_bps'] = {} + pg_current['rx_bps_L1'] = {} + pg_current['rx_line_util'] = {} + + pg_current['tx_pps_lpf'] = {} + pg_current['tx_bps_lpf'] = {} + pg_current['tx_bps_L1_lpf'] = {} + pg_current['rx_pps_lpf'] = {} + pg_current['rx_bps_lpf'] = {} + pg_current['rx_bps_L1_lpf'] = {} return - - # read the current values - now_tx_pkts = pg_current['tx_pkts']['total'] - now_tx_bytes = pg_current['tx_bytes']['total'] - now_rx_pkts = pg_current['rx_pkts']['total'] - now_rx_bytes = pg_current['rx_bytes']['total'] - - # prev values - prev_tx_pkts = pg_prev['tx_pkts']['total'] - prev_tx_bytes = pg_prev['tx_bytes']['total'] - prev_rx_pkts = pg_prev['rx_pkts']['total'] - prev_rx_bytes = pg_prev['rx_bytes']['total'] - - # prev B/W - prev_tx_pps = pg_prev['tx_pps'] - prev_tx_bps = pg_prev['tx_bps'] - prev_rx_pps = pg_prev['rx_pps'] - prev_rx_bps = pg_prev['rx_bps'] - - - #assert(now_tx_pkts >= prev_tx_pkts) - pg_current['tx_pps'] = self.calc_pps(prev_tx_pps, now_tx_pkts, prev_tx_pkts, diff_sec) - pg_current['tx_bps'] = self.calc_bps(prev_tx_bps, now_tx_bytes, prev_tx_bytes, diff_sec) - pg_current['rx_pps'] = self.calc_pps(prev_rx_pps, now_rx_pkts, prev_rx_pkts, diff_sec) - pg_current['rx_bps'] = self.calc_bps(prev_rx_bps, now_rx_bytes, prev_rx_bytes, diff_sec) - - if pg_current['tx_bps'] != None and pg_current['tx_pps'] != None: - pg_current['tx_bps_L1'] = calc_bps_L1(pg_current['tx_bps'], pg_current['tx_pps']) - else: - pg_current['tx_bps_L1'] = None + # TX + self.get_ports_speed() + for port in pg_current['tx_pkts'].keys(): + prev_tx_pps = pg_prev['tx_pps'].get(port) + now_tx_pkts = pg_current['tx_pkts'].get(port) + prev_tx_pkts = pg_prev['tx_pkts'].get(port) + pg_current['tx_pps'][port], pg_current['tx_pps_lpf'][port] = self.calc_pps(prev_tx_pps, now_tx_pkts, prev_tx_pkts, diff_sec) + + prev_tx_bps = pg_prev['tx_bps'].get(port) + now_tx_bytes = pg_current['tx_bytes'].get(port) + prev_tx_bytes = pg_prev['tx_bytes'].get(port) + pg_current['tx_bps'][port], pg_current['tx_bps_lpf'][port] = self.calc_bps(prev_tx_bps, now_tx_bytes, prev_tx_bytes, diff_sec) + + if pg_current['tx_bps'].get(port) != None and pg_current['tx_pps'].get(port) != None: + pg_current['tx_bps_L1'][port] = calc_bps_L1(pg_current['tx_bps'][port], pg_current['tx_pps'][port]) + pg_current['tx_bps_L1_lpf'][port] = calc_bps_L1(pg_current['tx_bps_lpf'][port], pg_current['tx_pps_lpf'][port]) + pg_current['tx_line_util'][port] = 100.0 * pg_current['tx_bps_L1'][port] / self.ports_speed[port] + else: + pg_current['tx_bps_L1'][port] = None + pg_current['tx_bps_L1_lpf'][port] = None + pg_current['tx_line_util'][port] = None + + # RX + for port in pg_current['rx_pkts'].keys(): + prev_rx_pps = pg_prev['rx_pps'].get(port) + now_rx_pkts = pg_current['rx_pkts'].get(port) + prev_rx_pkts = pg_prev['rx_pkts'].get(port) + pg_current['rx_pps'][port], pg_current['rx_pps_lpf'][port] = self.calc_pps(prev_rx_pps, now_rx_pkts, prev_rx_pkts, diff_sec) + + prev_rx_bps = pg_prev['rx_bps'].get(port) + now_rx_bytes = pg_current['rx_bytes'].get(port) + prev_rx_bytes = pg_prev['rx_bytes'].get(port) + pg_current['rx_bps'][port], pg_current['rx_bps_lpf'][port] = self.calc_bps(prev_rx_bps, now_rx_bytes, prev_rx_bytes, diff_sec) + if pg_current['rx_bps'].get(port) != None and pg_current['rx_pps'].get(port) != None: + pg_current['rx_bps_L1'][port] = calc_bps_L1(pg_current['rx_bps'][port], pg_current['rx_pps'][port]) + pg_current['rx_bps_L1_lpf'][port] = calc_bps_L1(pg_current['rx_bps_lpf'][port], pg_current['rx_pps_lpf'][port]) + pg_current['rx_line_util'][port] = 100.0 * pg_current['rx_bps_L1'][port] / self.ports_speed[port] + else: + pg_current['rx_bps_L1'][port] = None + pg_current['rx_bps_L1_lpf'][port] = None + pg_current['rx_line_util'][port] = None def calc_pps (self, prev_bw, now, prev, diff_sec): @@ -918,11 +1005,11 @@ class CRxStats(CTRexStats): def calc_bps (self, prev_bw, now, prev, diff_sec): return self.calc_bw(prev_bw, now, prev, diff_sec, True) - + # returns tuple - first value is real, second is low pass filtered def calc_bw (self, prev_bw, now, prev, diff_sec, is_bps): # B/W is not valid when the values are None if (now is None) or (prev is None): - return None + return (None, None) # calculate the B/W for current snapshot current_bw = (now - prev) / diff_sec @@ -933,7 +1020,7 @@ class CRxStats(CTRexStats): if prev_bw is None: prev_bw = 0 - return ( (0.5 * prev_bw) + (0.5 * current_bw) ) + return (current_bw, 0.5 * prev_bw + 0.5 * current_bw) @@ -960,22 +1047,29 @@ class CRxStats(CTRexStats): # skip non ints if not is_intable(pg_id): continue - + # bare counters stats[int(pg_id)] = {} - for field in ['tx_pkts', 'tx_bytes', 'rx_pkts']: - stats[int(pg_id)][field] = {'total': self.get_rel([pg_id, field, 'total'])} - - for port, pv in value[field].items(): - try: - int(port) - except ValueError: - continue - stats[int(pg_id)][field][int(port)] = self.get_rel([pg_id, field, port]) + for field in ['tx_pkts', 'tx_bytes', 'rx_pkts', 'rx_bytes']: + val = self.get_rel([pg_id, field, 'total']) + stats[int(pg_id)][field] = {'total': val if val != 'N/A' else StatNotAvailable(field)} + for port in value[field].keys(): + if is_intable(port): + val = self.get_rel([pg_id, field, port]) + stats[int(pg_id)][field][int(port)] = val if val != 'N/A' else StatNotAvailable(field) + + # BW values + for field in ['tx_pps', 'tx_bps', 'tx_bps_L1', 'rx_pps', 'rx_bps', 'rx_bps_L1', 'tx_line_util', 'rx_line_util']: + val = self.get([pg_id, field, 'total']) + stats[int(pg_id)][field] = {'total': val if val != 'N/A' else StatNotAvailable(field)} + for port in value[field].keys(): + if is_intable(port): + val = self.get([pg_id, field, port]) + stats[int(pg_id)][field][int(port)] = val if val != 'N/A' else StatNotAvailable(field) return stats - + # for Console def generate_stats (self): # for TUI - maximum 4 @@ -1005,13 +1099,13 @@ class CRxStats(CTRexStats): # maximum 4 for pg_id in pg_ids: - formatted_stats['Tx pps'].append(self.get([pg_id, 'tx_pps'], format = True, suffix = "pps")) - formatted_stats['Tx bps L2'].append(self.get([pg_id, 'tx_bps'], format = True, suffix = "bps")) + formatted_stats['Tx pps'].append(self.get([pg_id, 'tx_pps_lpf', 'total'], format = True, suffix = "pps")) + formatted_stats['Tx bps L2'].append(self.get([pg_id, 'tx_bps_lpf', 'total'], format = True, suffix = "bps")) - formatted_stats['Tx bps L1'].append(self.get([pg_id, 'tx_bps_L1'], format = True, suffix = "bps")) + formatted_stats['Tx bps L1'].append(self.get([pg_id, 'tx_bps_L1_lpf', 'total'], format = True, suffix = "bps")) - formatted_stats['Rx pps'].append(self.get([pg_id, 'rx_pps'], format = True, suffix = "pps")) - formatted_stats['Rx bps'].append(self.get([pg_id, 'rx_bps'], format = True, suffix = "bps")) + formatted_stats['Rx pps'].append(self.get([pg_id, 'rx_pps_lpf', 'total'], format = True, suffix = "pps")) + formatted_stats['Rx bps'].append(self.get([pg_id, 'rx_bps_lpf', 'total'], format = True, suffix = "bps")) formatted_stats['opackets'].append(self.get_rel([pg_id, 'tx_pkts', 'total'])) formatted_stats['ipackets'].append(self.get_rel([pg_id, 'rx_pkts', 'total'])) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py index 3ce876ad..165942d8 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py @@ -361,6 +361,8 @@ class STLStream(object): int_mac_dst_override_mode = int(mac_dst_override_mode); + self.is_default_mac = not (int_mac_src_override_by_pkt or int_mac_dst_override_mode) + self.fields['flags'] = (int_mac_src_override_by_pkt&1) + ((int_mac_dst_override_mode&3)<<1) self.fields['action_count'] = action_count @@ -421,6 +423,10 @@ class STLStream(object): return self.id + def has_custom_mac_addr (self): + """ Return True if src or dst MAC were set as custom """ + return not self.is_default_mac + def get_name (self): """ Get the stream name """ return self.name @@ -835,6 +841,9 @@ class STLProfile(object): def is_pauseable (self): return all([x.get_mode() == "Continuous" for x in self.get_streams()]) + def has_custom_mac_addr (self): + return any([x.has_custom_mac_addr() for x in self.get_streams()]) + def has_flow_stats (self): return any([x.has_flow_stats() for x in self.get_streams()]) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py index cd15b831..d84af22f 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_types.py @@ -1,5 +1,5 @@ -from collections import namedtuple +from collections import namedtuple, OrderedDict from .utils.text_opts import * from .trex_stl_exceptions import * import types @@ -139,4 +139,30 @@ def validate_type(arg_name, arg, valid_types): def verify_exclusive_arg (args_list): if not (len(list(filter(lambda x: x is not None, args_list))) == 1): raise STLError('exactly one parameter from {0} should be provided'.format(args_list)) - + +def listify (x): + if isinstance(x, list): + return x + else: + return [x] + +# shows as 'N/A', but does not let any compares for user to not mistake in automation +class StatNotAvailable(object): + def __init__(self, stat_name): + self.stat_name = stat_name + + def __repr__(self, *args, **kwargs): + return 'N/A' + + def __cmp__(self, *args, **kwargs): + raise Exception("Stat '%s' not available at this setup" % self.stat_name) + +class LRU_cache(OrderedDict): + def __init__(self, maxlen = 20, *args, **kwargs): + OrderedDict.__init__(self, *args, **kwargs) + self.maxlen = maxlen + + def __setitem__(self, *args, **kwargs): + OrderedDict.__setitem__(self, *args, **kwargs) + if len(self) > self.maxlen: + self.popitem(last = False) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py index ae74e932..b4903e81 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/common.py @@ -49,12 +49,18 @@ def random_id_gen(length=8): # try to get number from input, return None in case of fail def get_number(input): try: - if type(input) in (int, long): - return input - return int(input) + return long(input) except: - return None + try: + return int(input) + except: + return None def list_intersect(l1, l2): return list(filter(lambda x: x in l2, l1)) +def list_difference (l1, l2): + return list(filter(lambda x: x not in l2, l1)) + +def is_sub_list (l1, l2): + return set(l1) <= set(l2) diff --git a/scripts/automation/trex_control_plane/common/filters.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/filters.py index bf04a775..714f7807 100644 --- a/scripts/automation/trex_control_plane/common/filters.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/filters.py @@ -28,6 +28,13 @@ class ToggleFilter(object): self._filter_method = filter self.__set_initial_state(show_by_default) + def reset (self): + """ + Toggles off all the items + """ + self._toggle_db = set() + + def toggle_item(self, item_key): """ Toggle a single item in/out. diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py index c4f2b358..ad46625b 100755 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py @@ -1,5 +1,6 @@ import argparse from collections import namedtuple +from .common import list_intersect, list_difference import sys import re import os @@ -262,7 +263,7 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'], 'action': "store_false"}), - PORT_LIST: ArgumentPack(['--port'], + PORT_LIST: ArgumentPack(['--port', '-p'], {"nargs": '+', 'dest':'ports', 'metavar': 'PORTS', @@ -374,22 +375,49 @@ class CCmdArgParser(argparse.ArgumentParser): def __init__(self, stateless_client, *args, **kwargs): super(CCmdArgParser, self).__init__(*args, **kwargs) self.stateless_client = stateless_client + self.cmd_name = kwargs.get('prog') - def parse_args(self, args=None, namespace=None): + + def has_ports_cfg (self, opts): + return hasattr(opts, "all_ports") or hasattr(opts, "ports") + + def parse_args(self, args=None, namespace=None, default_ports=None, verify_acquired=False): try: opts = super(CCmdArgParser, self).parse_args(args, namespace) if opts is None: return None + if not self.has_ports_cfg(opts): + return opts + # if all ports are marked or if (getattr(opts, "all_ports", None) == True) or (getattr(opts, "ports", None) == []): - opts.ports = self.stateless_client.get_all_ports() + if default_ports is None: + opts.ports = self.stateless_client.get_acquired_ports() + else: + opts.ports = default_ports # so maybe we have ports configured - elif getattr(opts, "ports", None): - for port in opts.ports: - if not self.stateless_client._validate_port_list(port): - self.error("port id '{0}' is not a valid port id\n".format(port)) + invalid_ports = list_difference(opts.ports, self.stateless_client.get_all_ports()) + if invalid_ports: + self.stateless_client.logger.log("{0}: port(s) {1} are not valid port IDs".format(self.cmd_name, invalid_ports)) + return None + + # verify acquired ports + if verify_acquired: + acquired_ports = self.stateless_client.get_acquired_ports() + + diff = list_difference(opts.ports, acquired_ports) + if diff: + self.stateless_client.logger.log("{0} - port(s) {1} are not acquired".format(self.cmd_name, diff)) + return None + + # no acquire ports at all + if not acquired_ports: + self.stateless_client.logger.log("{0} - no acquired ports".format(self.cmd_name)) + return None + + return opts diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py index bc2d44f4..5c0dfb14 100644 --- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/text_opts.py @@ -124,16 +124,9 @@ def underline(text): def text_attribute(text, attribute): - if isinstance(text, str): - return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], - txt=text, - stop=TEXT_CODES[attribute]['end']) - elif isinstance(text, unicode): - return u"{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], - txt=text, - stop=TEXT_CODES[attribute]['end']) - else: - raise Exception("not a string") + return "{start}{txt}{stop}".format(start=TEXT_CODES[attribute]['start'], + txt=text, + stop=TEXT_CODES[attribute]['end']) FUNC_DICT = {'blue': blue, |