From 86138e3eb1fbaad89eb1f91937ae6bd6ba55da37 Mon Sep 17 00:00:00 2001 From: pmikus Date: Fri, 9 Dec 2022 08:50:06 +0000 Subject: feat(model): Hoststack type Signed-off-by: pmikus Change-Id: Ia7eefc28645c78ad346d294099ef6258faa9814f --- resources/libraries/python/Constants.py | 4 +- resources/libraries/python/HoststackUtil.py | 23 ++-- resources/libraries/python/model/ExportResult.py | 158 +++++++++++++++-------- resources/tools/ab/ABFork.py | 4 +- resources/tools/ab/ABTools.py | 87 ++++--------- 5 files changed, 151 insertions(+), 125 deletions(-) (limited to 'resources') diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index 0ae92f68ac..2bac8dc1cb 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Cisco and/or its affiliates. +# Copyright (c) 2023 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -120,7 +120,7 @@ class Constants: """Constants used in CSIT.""" # Version for CSIT data model. See docs/model/. - MODEL_VERSION = u"1.2.0" + MODEL_VERSION = u"1.3.0" # Global off-switch in case JSON export is large or slow. EXPORT_JSON = get_optimistic_bool_from_env(u"EXPORT_JSON") diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py index 7e6ba56913..35acdd70ee 100644 --- a/resources/libraries/python/HoststackUtil.py +++ b/resources/libraries/python/HoststackUtil.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2023 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,9 +17,12 @@ from time import sleep from robot.api import logger from resources.libraries.python.Constants import Constants -from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error -from resources.libraries.python.PapiExecutor import PapiSocketExecutor from resources.libraries.python.DUTSetup import DUTSetup +from resources.libraries.python.model.ExportResult import ( + export_hoststack_results +) +from resources.libraries.python.PapiExecutor import PapiSocketExecutor +from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error class HoststackUtil(): """Utilities for Host Stack tests.""" @@ -84,7 +87,6 @@ class HoststackUtil(): ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \ in iperf3_attributes else u"" iperf3_cmd[u"name"] = u"iperf3" - # TODO: Use OptionString library. iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \ f"--interval 0{json_results} " \ f"--version{iperf3_attributes[u'ip_version']}" @@ -289,7 +291,6 @@ class HoststackUtil(): cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'" exec_cmd(node, cmd, sudo=True) # Wait a bit for stdout/stderr to be flushed to log files - # TODO: see if sub-second sleep works e.g. sleep(0.1) sleep(1) @staticmethod @@ -346,7 +347,6 @@ class HoststackUtil(): f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \ f"pkts/drop\n" - # TODO: Incorporate show error stats into results analysis test_results += \ f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \ f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n" @@ -364,11 +364,13 @@ class HoststackUtil(): if u"JSON stats" in program_stdout and \ u'"has_failed": "0"' in program_stdout: json_start = program_stdout.find(u"{") - #TODO: Fix parsing once vpp_echo produces valid - # JSON output. Truncate for now. json_end = program_stdout.find(u',\n "closing"') json_results = f"{program_stdout[json_start:json_end]}\n}}" program_json = json.loads(json_results) + export_hoststack_results( + bandwidth=program_json["rx_bits_per_second"], + duration=program_json["time"] + ) else: test_results += u"Invalid test data output!\n" + program_stdout return (True, test_results) @@ -376,6 +378,11 @@ class HoststackUtil(): test_results += program_stdout iperf3_json = json.loads(program_stdout) program_json = iperf3_json[u"intervals"][0][u"sum"] + export_hoststack_results( + bandwidth=program_json["bits_per_second"], + duration=program_json["seconds"], + retransmits=program_json["retransmits"] + ) else: test_results += u"Unknown HostStack Test Program!\n" + \ program_stdout diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py index b5de27ebe5..dd0968419a 100644 --- a/resources/libraries/python/model/ExportResult.py +++ b/resources/libraries/python/model/ExportResult.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Cisco and/or its affiliates. +# Copyright (c) 2023 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -18,7 +18,7 @@ from robot.libraries.BuiltIn import BuiltIn from resources.libraries.python.model.util import descend, get_export_data -def export_dut_type_and_version(dut_type=u"unknown", dut_version=u"unknown"): +def export_dut_type_and_version(dut_type="unknown", dut_version="unknown"): """Export the arguments as dut type and version. Robot tends to convert "none" into None, hence the unusual default values. @@ -32,32 +32,32 @@ def export_dut_type_and_version(dut_type=u"unknown", dut_version=u"unknown"): :type dut_version: Optiona[str] :raises RuntimeError: If value is neither in argument not robot variable. """ - if dut_type == u"unknown": - dut_type = BuiltIn().get_variable_value(u"\\${DUT_TYPE}", u"unknown") - if dut_type == u"unknown": - raise RuntimeError(u"Dut type not provided.") + if dut_type == "unknown": + dut_type = BuiltIn().get_variable_value("\\${DUT_TYPE}", "unknown") + if dut_type == "unknown": + raise RuntimeError("Dut type not provided.") else: # We want to set a variable in higher level suite setup # to be available to test setup several levels lower. BuiltIn().set_suite_variable( - u"\\${DUT_TYPE}", dut_type, u"children=True" + "\\${DUT_TYPE}", dut_type, "children=True" ) - if dut_version == u"unknown": + if dut_version == "unknown": dut_version = BuiltIn().get_variable_value( - u"\\${DUT_VERSION}", u"unknown" + "\\${DUT_VERSION}", "unknown" ) - if dut_type == u"unknown": - raise RuntimeError(u"Dut version not provided.") + if dut_type == "unknown": + raise RuntimeError("Dut version not provided.") else: BuiltIn().set_suite_variable( - u"\\${DUT_VERSION}", dut_version, u"children=True" + "\\${DUT_VERSION}", dut_version, "children=True" ) data = get_export_data() - data[u"dut_type"] = dut_type.lower() - data[u"dut_version"] = dut_version + data["dut_type"] = dut_type.lower() + data["dut_version"] = dut_version -def export_tg_type_and_version(tg_type=u"unknown", tg_version=u"unknown"): +def export_tg_type_and_version(tg_type="unknown", tg_version="unknown"): """Export the arguments as tg type and version. Robot tends to convert "none" into None, hence the unusual default values. @@ -71,29 +71,29 @@ def export_tg_type_and_version(tg_type=u"unknown", tg_version=u"unknown"): :type tg_version: Optiona[str] :raises RuntimeError: If value is neither in argument not robot variable. """ - if tg_type == u"unknown": - tg_type = BuiltIn().get_variable_value(u"\\${TG_TYPE}", u"unknown") - if tg_type == u"unknown": - raise RuntimeError(u"TG type not provided.") + if tg_type == "unknown": + tg_type = BuiltIn().get_variable_value("\\${TG_TYPE}", "unknown") + if tg_type == "unknown": + raise RuntimeError("TG type not provided!") else: # We want to set a variable in higher level suite setup # to be available to test setup several levels lower. BuiltIn().set_suite_variable( - u"\\${TG_TYPE}", tg_type, u"children=True" + "\\${TG_TYPE}", tg_type, "children=True" ) - if tg_version == u"unknown": + if tg_version == "unknown": tg_version = BuiltIn().get_variable_value( - u"\\${TG_VERSION}", u"unknown" + "\\${TG_VERSION}", "unknown" ) - if tg_type == u"unknown": - raise RuntimeError(u"TG version not provided.") + if tg_type == "unknown": + raise RuntimeError("TG version not provided!") else: BuiltIn().set_suite_variable( - u"\\${TG_VERSION}", tg_version, u"children=True" + "\\${TG_VERSION}", tg_version, "children=True" ) data = get_export_data() - data[u"tg_type"] = tg_type.lower() - data[u"tg_version"] = tg_version + data["tg_type"] = tg_type.lower() + data["tg_version"] = tg_version def append_mrr_value(mrr_value, unit): @@ -109,10 +109,10 @@ def append_mrr_value(mrr_value, unit): if not unit: return data = get_export_data() - data[u"result"][u"type"] = u"mrr" - rate_node = descend(descend(data[u"result"], u"receive_rate"), "rate") - rate_node[u"unit"] = str(unit) - values_list = descend(rate_node, u"values", list) + data["result"]["type"] = "mrr" + rate_node = descend(descend(data["result"], "receive_rate"), "rate") + rate_node["unit"] = str(unit) + values_list = descend(rate_node, "values", list) values_list.append(float(mrr_value)) @@ -139,17 +139,17 @@ def export_search_bound(text, value, unit, bandwidth=None): """ value = float(value) text = str(text).lower() - result_type = u"soak" if u"plrsearch" in text else u"ndrpdr" - upper_or_lower = u"upper" if u"upper" in text else u"lower" - ndr_or_pdr = u"ndr" if u"ndr" in text else u"pdr" + result_type = "soak" if "plrsearch" in text else "ndrpdr" + upper_or_lower = "upper" if "upper" in text else "lower" + ndr_or_pdr = "ndr" if "ndr" in text else "pdr" - result_node = get_export_data()[u"result"] - result_node[u"type"] = result_type + result_node = get_export_data()["result"] + result_node["type"] = result_type rate_item = dict(rate=dict(value=value, unit=unit)) if bandwidth: - rate_item[u"bandwidth"] = dict(value=float(bandwidth), unit=u"bps") - if result_type == u"soak": - descend(result_node, u"critical_rate")[upper_or_lower] = rate_item + rate_item["bandwidth"] = dict(value=float(bandwidth), unit="bps") + if result_type == "soak": + descend(result_node, "critical_rate")[upper_or_lower] = rate_item return descend(result_node, ndr_or_pdr)[upper_or_lower] = rate_item @@ -169,14 +169,14 @@ def _add_latency(result_node, percent, whichward, latency_string): :type whichward: str :latency_string: str """ - l_min, l_avg, l_max, l_hdrh = latency_string.split(u"/", 3) + l_min, l_avg, l_max, l_hdrh = latency_string.split("/", 3) whichward_node = descend(result_node, f"latency_{whichward}") percent_node = descend(whichward_node, f"pdr_{percent}") - percent_node[u"min"] = int(l_min) - percent_node[u"avg"] = int(l_avg) - percent_node[u"max"] = int(l_max) - percent_node[u"hdrh"] = l_hdrh - percent_node[u"unit"] = u"us" + percent_node["min"] = int(l_min) + percent_node["avg"] = int(l_avg) + percent_node["max"] = int(l_max) + percent_node["hdrh"] = l_hdrh + percent_node["unit"] = "us" def export_ndrpdr_latency(text, latency): @@ -195,23 +195,25 @@ def export_ndrpdr_latency(text, latency): :type text: str :type latency: 1-tuple or 2-tuple of str """ - result_node = get_export_data()[u"result"] + result_node = get_export_data()["result"] percent = 0 - if u"90" in text: + if "90" in text: percent = 90 - elif u"50" in text: + elif "50" in text: percent = 50 - elif u"10" in text: + elif "10" in text: percent = 10 - _add_latency(result_node, percent, u"forward", latency[0]) + _add_latency(result_node, percent, "forward", latency[0]) # Else TRex does not support latency measurement for this traffic profile. if len(latency) < 2: return - _add_latency(result_node, percent, u"reverse", latency[1]) + _add_latency(result_node, percent, "reverse", latency[1]) def export_reconf_result(packet_rate, packet_loss, bandwidth): - """Export the results from a reconf test. + """Export the RECONF type results. + + Result type is set to reconf. :param packet_rate: Aggregate offered load in packets per second. :param packet_loss: How many of the packets were dropped or unsent. @@ -246,6 +248,56 @@ def export_reconf_result(packet_rate, packet_loss, bandwidth): ) +def export_hoststack_results( + bandwidth, rate=None, rate_unit=None, latency=None, + failed_requests=None, completed_requests=None, retransmits=None, + duration=None +): + """Export the HOSTSTACK type results. + + Result type is set to hoststack. + + :param bandwidth: Measured transfer rate using bps as a unit. + :param rate: Resulting rate measured by the test. [Optional] + :param rate_unit: CPS or RPS. [Optional] + :param latency: Measure latency. [Optional] + :param failed_requests: Number of failed requests. [Optional] + :param completed_requests: Number of completed requests. [Optional] + :param retransmits: Retransmitted TCP packets. [Optional] + :param duration: Measurment duration. [Optional] + :type bandwidth: float + :type rate: float + :type rate_unit: str + :type latency: float + :type failed_requests: int + :type completed_requests: int + :type retransmits: int + :type duration: float + """ + result_node = get_export_data()["result"] + result_node["type"] = "hoststack" + + result_node["bandwidth"] = dict(unit="bps", value=bandwidth) + if rate is not None: + result_node["rate"] = \ + dict(unit=rate_unit, value=rate) + if latency is not None: + result_node["latency"] = \ + dict(unit="ms", value=latency) + if failed_requests is not None: + result_node["failed_requests"] = \ + dict(unit="requests", value=failed_requests) + if completed_requests is not None: + result_node["completed_requests"] = \ + dict(unit="requests", value=completed_requests) + if retransmits is not None: + result_node["retransmits"] = \ + dict(unit="packets", value=retransmits) + if duration is not None: + result_node["duration"] = \ + dict(unit="s", value=duration) + + def append_telemetry(telemetry_item): """Append telemetry entry to proper place so it is dumped into json. @@ -253,4 +305,4 @@ def append_telemetry(telemetry_item): :type telemetry_item: str """ data = get_export_data() - data[u"telemetry"].append(telemetry_item) + data["telemetry"].append(telemetry_item) diff --git a/resources/tools/ab/ABFork.py b/resources/tools/ab/ABFork.py index 8436ed38be..55288a9c92 100755 --- a/resources/tools/ab/ABFork.py +++ b/resources/tools/ab/ABFork.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2021 Intel and/or its affiliates. +# Copyright (c) 2023 Intel and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -136,7 +136,7 @@ def main(): # Output results. print(f"Transfer Rate: {round(info_list[6], 2)} [Kbytes/sec]") print(f"Latency: {round(info_list[4] / 8, 2)} ms") - print(f"Connection {mode} rate:{round(info_list[3], 2)} per sec") + print(f"Connection {mode} rate: {round(info_list[3], 2)} per sec") print(f"Total data transferred: {round(info_list[2])} bytes") print(f"Completed requests: {round(info_list[0])} ") print(f"Failed requests: {round(info_list[1])} ") diff --git a/resources/tools/ab/ABTools.py b/resources/tools/ab/ABTools.py index 54aff19e92..f3e97a720a 100644 --- a/resources/tools/ab/ABTools.py +++ b/resources/tools/ab/ABTools.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Intel and/or its affiliates. +# Copyright (c) 2023 Intel and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -13,11 +13,13 @@ """ab implementation into CSIT framework.""" -from robot.api import logger -from resources.libraries.python.topology import NodeType from resources.libraries.python.Constants import Constants -from resources.libraries.python.ssh import exec_cmd_no_error +from resources.libraries.python.model.ExportResult import ( + export_hoststack_results +) from resources.libraries.python.OptionString import OptionString +from resources.libraries.python.ssh import exec_cmd_no_error +from resources.libraries.python.topology import NodeType class ABTools: @@ -153,66 +155,31 @@ class ABTools: port=port, mode=rps_cps, ) - stdout, _ = exec_cmd_no_error(tg_node, cmd, timeout=180, sudo=True, - message=u"ab runtime error!") - log_msg = ABTools._parse_ab_output(stdout, rps_cps, tls_tcp) - - logger.info(log_msg) - - return log_msg - - @staticmethod - def _parse_ab_output(msg, rps_cps, tls_tcp): - """Parse the ab stdout with the results. - - :param msg: Ab Stdout. - :param rps_cps: RPS or CPS. - :param tls_tcp: TLS or TCP. - :type msg: str - :type rps_cps: str - :type tls_tcp: str - :return: Message with measured data. - :rtype: str - """ - - msg_lst = msg.splitlines(keepends=False) - - total_cps = u"" - latency = u"" - processing = u"" - complete_req = u"" - failed_req = u"" - total_bytes = u"" - rate = u"" - - if tls_tcp == u"tls": - log_msg = u"\nMeasured HTTPS values:\n" - else: - log_msg = u"\nMeasured HTTP values:\n" + stdout, _ = exec_cmd_no_error( + tg_node, cmd, timeout=180, sudo=True, message=u"ab runtime error!" + ) - for line in msg_lst: + rate_unit = rps_cps + rate = None + bandwidth = None + latency = None + completed_requests = None + failed_requests = None + for line in stdout.splitlines(): if f"Connection {rps_cps} rate:" in line: - # rps (cps) - total_cps = line + u"\n" + rate = float(line.split(u" ")[3]) elif u"Transfer Rate:" in line: - # Rate - rate = line + u"\n" + bandwidth = float(line.split(u" ")[2]) * 8000 elif u"Latency:" in line: - # Latency - latency = line + u"\n" - elif u"Total data transferred" in line: - total_bytes = line + u"\n" - elif u"Completed requests" in line: - complete_req = line + u"\n" + latency = float(line.split(u" ")[1]) + elif u"Completed requests:" in line: + completed_requests = int(line.split(u" ")[2]) elif u"Failed requests" in line: - failed_req = line + u"\n" + failed_requests = int(line.split(u" ")[2]) - log_msg += rate - log_msg += latency - log_msg += processing - log_msg += complete_req - log_msg += failed_req - log_msg += total_bytes - log_msg += total_cps + export_hoststack_results( + rate, rate_unit, bandwidth, latency, failed_requests, + completed_requests + ) - return log_msg + return stdout -- cgit 1.2.3-korg