diff options
| -rw-r--r-- | resources/libraries/python/model/ExportJson.py | 216 | ||||
| -rw-r--r-- | resources/libraries/python/model/ExportResult.py | 24 | ||||
| -rw-r--r-- | resources/libraries/robot/performance/performance_utils.robot | 4 |
3 files changed, 127 insertions, 117 deletions
diff --git a/resources/libraries/python/model/ExportJson.py b/resources/libraries/python/model/ExportJson.py index 73e822491c..843949eb9f 100644 --- a/resources/libraries/python/model/ExportJson.py +++ b/resources/libraries/python/model/ExportJson.py @@ -43,14 +43,14 @@ from resources.libraries.python.model.validate import ( class ExportJson(): """Class handling the json data setting and export.""" - ROBOT_LIBRARY_SCOPE = u"GLOBAL" + ROBOT_LIBRARY_SCOPE = "GLOBAL" def __init__(self): """Declare required fields, cache output dir. Also memorize schema validator instances. """ - self.output_dir = BuiltIn().get_variable_value(u"\\${OUTPUT_DIR}", ".") + self.output_dir = BuiltIn().get_variable_value("\\${OUTPUT_DIR}", ".") self.file_path = None self.data = None self.validators = get_validators() @@ -62,25 +62,25 @@ class ExportJson(): :rtype: str :raises RuntimeError: If the test tags does not contain expected values. """ - tags = self.data[u"tags"] + tags = self.data["tags"] # First 5 options are specific for VPP tests. - if u"DEVICETEST" in tags: - test_type = u"device" - elif u"LDP_NGINX" in tags: - test_type = u"hoststack" - elif u"HOSTSTACK" in tags: - test_type = u"hoststack" - elif u"GSO_TRUE" in tags or u"GSO_FALSE" in tags: - test_type = u"mrr" - elif u"RECONF" in tags: - test_type = u"reconf" + if "DEVICETEST" in tags: + test_type = "device" + elif "LDP_NGINX" in tags: + test_type = "hoststack" + elif "HOSTSTACK" in tags: + test_type = "hoststack" + elif "GSO_TRUE" in tags or "GSO_FALSE" in tags: + test_type = "mrr" + elif "RECONF" in tags: + test_type = "reconf" # The remaining 3 options could also apply to DPDK and TRex tests. - elif u"SOAK" in tags: - test_type = u"soak" - elif u"NDRPDR" in tags: - test_type = u"ndrpdr" - elif u"MRR" in tags: - test_type = u"mrr" + elif "SOAK" in tags: + test_type = "soak" + elif "NDRPDR" in tags: + test_type = "ndrpdr" + elif "MRR" in tags: + test_type = "mrr" else: raise RuntimeError(f"Unable to infer test type from tags: {tags}") return test_type @@ -108,12 +108,12 @@ class ExportJson(): new_file_path = write_output(self.file_path, self.data) # Data is going to be cleared (as a sign that export succeeded), # so this is the last chance to detect if it was for a test case. - is_testcase = u"result" in self.data + is_testcase = "result" in self.data self.data = None # Validation for output goes here when ready. self.file_path = None if is_testcase: - validate(new_file_path, self.validators[u"tc_info"]) + validate(new_file_path, self.validators["tc_info"]) def warn_on_bad_export(self): """If bad state is detected, log a warning and clean up state.""" @@ -133,25 +133,25 @@ class ExportJson(): """ self.warn_on_bad_export() start_time = datetime.datetime.utcnow().strftime( - u"%Y-%m-%dT%H:%M:%S.%fZ" + "%Y-%m-%dT%H:%M:%S.%fZ" ) - suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") - suite_id = suite_name.lower().replace(u" ", u"_") - suite_path_part = os.path.join(*suite_id.split(u".")) + suite_name = BuiltIn().get_variable_value("\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(" ", "_") + suite_path_part = os.path.join(*suite_id.split(".")) output_dir = self.output_dir self.file_path = os.path.join( - output_dir, suite_path_part, u"setup.info.json" + output_dir, suite_path_part, "setup.info.json" ) self.data = dict() - self.data[u"version"] = Constants.MODEL_VERSION - self.data[u"start_time"] = start_time - self.data[u"suite_name"] = suite_name - self.data[u"suite_documentation"] = BuiltIn().get_variable_value( - u"\\${SUITE_DOCUMENTATION}" + self.data["version"] = Constants.MODEL_VERSION + self.data["start_time"] = start_time + self.data["suite_name"] = suite_name + self.data["suite_documentation"] = BuiltIn().get_variable_value( + "\\${SUITE_DOCUMENTATION}" ) # "end_time" and "duration" are added on flush. - self.data[u"hosts"] = set() - self.data[u"telemetry"] = list() + self.data["hosts"] = set() + self.data["telemetry"] = list() def start_test_export(self): """Set new file path, initialize data to minimal tree for the test case. @@ -167,30 +167,30 @@ class ExportJson(): """ self.warn_on_bad_export() start_time = datetime.datetime.utcnow().strftime( - u"%Y-%m-%dT%H:%M:%S.%fZ" + "%Y-%m-%dT%H:%M:%S.%fZ" ) - suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") - suite_id = suite_name.lower().replace(u" ", u"_") - suite_path_part = os.path.join(*suite_id.split(u".")) - test_name = BuiltIn().get_variable_value(u"\\${TEST_NAME}") + suite_name = BuiltIn().get_variable_value("\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(" ", "_") + suite_path_part = os.path.join(*suite_id.split(".")) + test_name = BuiltIn().get_variable_value("\\${TEST_NAME}") self.file_path = os.path.join( self.output_dir, suite_path_part, - test_name.lower().replace(u" ", u"_") + u".info.json" + test_name.lower().replace(" ", "_") + ".info.json" ) self.data = dict() - self.data[u"version"] = Constants.MODEL_VERSION - self.data[u"start_time"] = start_time - self.data[u"suite_name"] = suite_name - self.data[u"test_name"] = test_name - test_doc = BuiltIn().get_variable_value(u"\\${TEST_DOCUMENTATION}", u"") - self.data[u"test_documentation"] = test_doc + self.data["version"] = Constants.MODEL_VERSION + self.data["start_time"] = start_time + self.data["suite_name"] = suite_name + self.data["test_name"] = test_name + test_doc = BuiltIn().get_variable_value("\\${TEST_DOCUMENTATION}", "") + self.data["test_documentation"] = test_doc # "test_type" is added on flush. # "tags" is detected and added on flush. # "end_time" and "duration" is added on flush. # Robot status and message are added on flush. - self.data[u"result"] = dict(type=u"unknown") - self.data[u"hosts"] = BuiltIn().get_variable_value(u"\\${hosts}") - self.data[u"telemetry"] = list() + self.data["result"] = dict(type="unknown") + self.data["hosts"] = BuiltIn().get_variable_value("\\${hosts}") + self.data["telemetry"] = list() export_dut_type_and_version() export_tg_type_and_version() @@ -205,21 +205,21 @@ class ExportJson(): """ self.warn_on_bad_export() start_time = datetime.datetime.utcnow().strftime( - u"%Y-%m-%dT%H:%M:%S.%fZ" + "%Y-%m-%dT%H:%M:%S.%fZ" ) - suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") - suite_id = suite_name.lower().replace(u" ", u"_") - suite_path_part = os.path.join(*suite_id.split(u".")) + suite_name = BuiltIn().get_variable_value("\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(" ", "_") + suite_path_part = os.path.join(*suite_id.split(".")) self.file_path = os.path.join( - self.output_dir, suite_path_part, u"teardown.info.json" + self.output_dir, suite_path_part, "teardown.info.json" ) self.data = dict() - self.data[u"version"] = Constants.MODEL_VERSION - self.data[u"start_time"] = start_time - self.data[u"suite_name"] = suite_name + self.data["version"] = Constants.MODEL_VERSION + self.data["start_time"] = start_time + self.data["suite_name"] = suite_name # "end_time" and "duration" is added on flush. - self.data[u"hosts"] = BuiltIn().get_variable_value(u"\\${hosts}") - self.data[u"telemetry"] = list() + self.data["hosts"] = BuiltIn().get_variable_value("\\${hosts}") + self.data["telemetry"] = list() def finalize_suite_setup_export(self): """Add the missing fields to data. Do not write yet. @@ -227,9 +227,9 @@ class ExportJson(): Should be run at the end of suite setup. The write is done at next start (or at the end of global teardown). """ - end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - self.data[u"hosts"] = BuiltIn().get_variable_value(u"\\${hosts}") - self.data[u"end_time"] = end_time + end_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.data["hosts"] = BuiltIn().get_variable_value("\\${hosts}") + self.data["end_time"] = end_time self.export_pending_data() def finalize_test_export(self): @@ -240,15 +240,15 @@ class ExportJson(): The write is done at next start (or at the end of global teardown). """ - end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - message = BuiltIn().get_variable_value(u"\\${TEST_MESSAGE}") - test_tags = BuiltIn().get_variable_value(u"\\${TEST_TAGS}") - self.data[u"end_time"] = end_time - start_float = parse(self.data[u"start_time"]).timestamp() - end_float = parse(self.data[u"end_time"]).timestamp() - self.data[u"duration"] = end_float - start_float - self.data[u"tags"] = list(test_tags) - self.data[u"message"] = message + end_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + message = BuiltIn().get_variable_value("\\${TEST_MESSAGE}") + test_tags = BuiltIn().get_variable_value("\\${TEST_TAGS}") + self.data["end_time"] = end_time + start_float = parse(self.data["start_time"]).timestamp() + end_float = parse(self.data["end_time"]).timestamp() + self.data["duration"] = end_float - start_float + self.data["tags"] = list(test_tags) + self.data["message"] = message self.process_passed() self.process_test_name() self.process_results() @@ -261,8 +261,8 @@ class ExportJson(): (but before the explicit write in the global suite teardown). The write is done at next start (or explicitly for global teardown). """ - end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - self.data[u"end_time"] = end_time + end_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.data["end_time"] = end_time self.export_pending_data() def process_test_name(self): @@ -292,28 +292,28 @@ class ExportJson(): :raises RuntimeError: If the data does not contain expected values. """ - suite_part = self.data.pop(u"suite_name").lower().replace(u" ", u"_") - if u"test_name" not in self.data: + suite_part = self.data.pop("suite_name").lower().replace(" ", "_") + if "test_name" not in self.data: # There will be no test_id, provide suite_id instead. - self.data[u"suite_id"] = suite_part + self.data["suite_id"] = suite_part return - test_part = self.data.pop(u"test_name").lower().replace(u" ", u"_") - self.data[u"test_id"] = f"{suite_part}.{test_part}" - tags = self.data[u"tags"] + test_part = self.data.pop("test_name").lower().replace(" ", "_") + self.data["test_id"] = f"{suite_part}.{test_part}" + tags = self.data["tags"] # Test name does not contain thread count. - subparts = test_part.split(u"c-", 1) - if len(subparts) < 2 or subparts[0][-2:-1] != u"-": + subparts = test_part.split("c-", 1) + if len(subparts) < 2 or subparts[0][-2:-1] != "-": # Physical core count not detected, assume it is a TRex test. - if u"--" not in test_part: + if "--" not in test_part: raise RuntimeError(f"Cores not found for {subparts}") - short_name = test_part.split(u"--", 1)[1] + short_name = test_part.split("--", 1)[1] else: short_name = subparts[1] # Add threads to test_part. - core_part = subparts[0][-1] + u"c" + core_part = subparts[0][-1] + "c" for tag in tags: tag = tag.lower() - if len(tag) == 4 and core_part == tag[2:] and tag[1] == u"t": + if len(tag) == 4 and core_part == tag[2:] and tag[1] == "t": test_part = test_part.replace(f"-{core_part}-", f"-{tag}-") break else: @@ -321,24 +321,24 @@ class ExportJson(): f"Threads not found for {test_part} tags {tags}" ) # For long name we need NIC model, which is only in suite name. - last_suite_part = suite_part.split(u".")[-1] + last_suite_part = suite_part.split(".")[-1] # Short name happens to be the suffix we want to ignore. prefix_part = last_suite_part.split(short_name)[0] # Also remove the trailing dash. prefix_part = prefix_part[:-1] # Throw away possible link prefix such as "1n1l-". - nic_code = prefix_part.split(u"-", 1)[-1] + nic_code = prefix_part.split("-", 1)[-1] nic_short = Constants.NIC_CODE_TO_SHORT_NAME[nic_code] long_name = f"{nic_short}-{test_part}" # Set test type. test_type = self._detect_test_type() - self.data[u"test_type"] = test_type + self.data["test_type"] = test_type # Remove trailing test type from names (if present). short_name = short_name.split(f"-{test_type}")[0] long_name = long_name.split(f"-{test_type}")[0] # Store names. - self.data[u"test_name_short"] = short_name - self.data[u"test_name_long"] = long_name + self.data["test_name_short"] = short_name + self.data["test_name_long"] = long_name def process_passed(self): """Process the test status information as boolean. @@ -346,12 +346,12 @@ class ExportJson(): Boolean is used to make post processing more efficient. In case the test status is PASS, we will truncate the test message. """ - status = BuiltIn().get_variable_value(u"\\${TEST_STATUS}") + status = BuiltIn().get_variable_value("\\${TEST_STATUS}") if status is not None: - self.data[u"passed"] = (status == u"PASS") - if self.data[u"passed"]: + self.data["passed"] = (status == "PASS") + if self.data["passed"]: # Also truncate success test messages. - self.data[u"message"] = u"" + self.data["message"] = "" def process_results(self): """Process measured results. @@ -364,32 +364,34 @@ class ExportJson(): telemetry_compress = compress(telemetry_encode, level=9) telemetry_base64 = b2a_base64(telemetry_compress, newline=False) self.data["telemetry"] = [telemetry_base64.decode()] - if u"result" not in self.data: + if "result" not in self.data: return - result_node = self.data[u"result"] - result_type = result_node[u"type"] - if result_type == u"unknown": + result_node = self.data["result"] + result_type = result_node["type"] + if result_type == "unknown": # Device or something else not supported. return - # Compute avg and stdev for mrr. - if result_type == u"mrr": - rate_node = result_node[u"receive_rate"][u"rate"] - stats = AvgStdevStats.for_runs(rate_node[u"values"]) - rate_node[u"avg"] = stats.avg - rate_node[u"stdev"] = stats.stdev + # Compute avg and stdev for mrr (rate and bandwidth). + if result_type == "mrr": + for node_name in ("rate", "bandwidth"): + node = result_node["receive_rate"].get(node_name, None) + if node is not None: + stats = AvgStdevStats.for_runs(node["values"]) + node["avg"] = stats.avg + node["stdev"] = stats.stdev return # Multiple processing steps for ndrpdr. - if result_type != u"ndrpdr": + if result_type != "ndrpdr": return # Filter out invalid latencies. - for which_key in (u"latency_forward", u"latency_reverse"): + for which_key in ("latency_forward", "latency_reverse"): if which_key not in result_node: # Probably just an unidir test. continue - for load in (u"pdr_0", u"pdr_10", u"pdr_50", u"pdr_90"): - if result_node[which_key][load][u"max"] <= 0: + for load in ("pdr_0", "pdr_10", "pdr_50", "pdr_90"): + if result_node[which_key][load]["max"] <= 0: # One invalid number is enough to remove all loads. break else: diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py index dd0968419a..f155848913 100644 --- a/resources/libraries/python/model/ExportResult.py +++ b/resources/libraries/python/model/ExportResult.py @@ -96,24 +96,32 @@ def export_tg_type_and_version(tg_type="unknown", tg_version="unknown"): data["tg_version"] = tg_version -def append_mrr_value(mrr_value, unit): +def append_mrr_value(mrr_value, mrr_unit, bandwidth_value=None, + bandwidth_unit="bps"): """Store mrr value to proper place so it is dumped into json. The value is appended only when unit is not empty. :param mrr_value: Forwarding rate from MRR trial. - :param unit: Unit of measurement for the rate. + :param mrr_unit: Unit of measurement for the rate. + :param bandwidth_value: The same value recomputed into L1 bits per second. :type mrr_value: float - :type unit: str + :type mrr_unit: str + :type bandwidth_value: Optional[float] + :type bandwidth_unit: Optional[str] """ - if not unit: + if not mrr_unit: return data = get_export_data() data["result"]["type"] = "mrr" - rate_node = descend(descend(data["result"], "receive_rate"), "rate") - rate_node["unit"] = str(unit) - values_list = descend(rate_node, "values", list) - values_list.append(float(mrr_value)) + + for node_val, node_unit, node_name in ((mrr_value, mrr_unit, "rate"), + (bandwidth_value, bandwidth_unit, "bandwidth")): + if node_val is not None: + node = descend(descend(data["result"], "receive_rate"), node_name) + node["unit"] = str(node_unit) + values_list = descend(node, "values", list) + values_list.append(float(node_val)) def export_search_bound(text, value, unit, bandwidth=None): diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot index 34b9ab6bef..78a1fcc0f9 100644 --- a/resources/libraries/robot/performance/performance_utils.robot +++ b/resources/libraries/robot/performance/performance_utils.robot @@ -451,8 +451,8 @@ | | | # Out of several quantities for aborted traffic (duration stretching), | | | # the approximated receive rate is the best estimate we have. | | | ${value} = | Set Variable | ${result.approximated_receive_rate} -| | | # TODO: Add correct bandwidth computation. -| | | Append Mrr Value | ${value} | ${export_mrr_unit} +| | | ${bandwidth} | ${pps} = | Compute Bandwidth | ${value} / ${ppta} +| | | Append Mrr Value | ${value} | ${export_mrr_unit} | ${bandwidth * 1e9} | | | Append To List | ${results} | ${value} | | END | | FOR | ${action} | IN | @{stat_post_trial} |