diff options
Diffstat (limited to 'resources/libraries/python/model')
| -rw-r--r-- | resources/libraries/python/model/ExportJson.py | 395 | ||||
| -rw-r--r-- | resources/libraries/python/model/ExportResult.py | 316 | ||||
| -rw-r--r-- | resources/libraries/python/model/MemDump.py | 194 | ||||
| -rw-r--r-- | resources/libraries/python/model/__init__.py | 16 | ||||
| -rw-r--r-- | resources/libraries/python/model/parse.py | 112 | ||||
| -rw-r--r-- | resources/libraries/python/model/util.py | 69 | ||||
| -rw-r--r-- | resources/libraries/python/model/validate.py | 62 |
7 files changed, 1164 insertions, 0 deletions
diff --git a/resources/libraries/python/model/ExportJson.py b/resources/libraries/python/model/ExportJson.py new file mode 100644 index 0000000000..3f923d6d0e --- /dev/null +++ b/resources/libraries/python/model/ExportJson.py @@ -0,0 +1,395 @@ +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module tracking json in-memory data and saving it to files. + +Each test case, suite setup (hierarchical) and teardown has its own file pair. + +Validation is performed for output files with available JSON schema. +Validation is performed in data deserialized from disk, +as serialization might have introduced subtle errors. +""" + +import datetime +import os.path + +from binascii import b2a_base64 +from dateutil.parser import parse +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn +from zlib import compress + +from resources.libraries.python.Constants import Constants +from resources.libraries.python.jumpavg import AvgStdevStats +from resources.libraries.python.model.ExportResult import ( + export_dut_type_and_version, export_tg_type_and_version +) +from resources.libraries.python.model.MemDump import write_output +from resources.libraries.python.model.validate import ( + get_validators, validate +) + + +class ExportJson(): + """Class handling the json data setting and export.""" + + ROBOT_LIBRARY_SCOPE = "GLOBAL" + + def __init__(self): + """Declare required fields, cache output dir. + + Also memorize schema validator instances. + """ + self.output_dir = BuiltIn().get_variable_value("\\${OUTPUT_DIR}", ".") + self.file_path = None + self.data = None + self.validators = get_validators() + + def _detect_test_type(self): + """Return test_type, as inferred from robot test tags. + + :returns: The inferred test type value. + :rtype: str + :raises RuntimeError: If the test tags does not contain expected values. + """ + tags = self.data["tags"] + # First 5 options are specific for VPP tests. + if "DEVICETEST" in tags: + test_type = "device" + elif "LDP_NGINX" in tags: + test_type = "hoststack" + elif "HOSTSTACK" in tags: + test_type = "hoststack" + elif "GSO_TRUE" in tags or "GSO_FALSE" in tags: + test_type = "mrr" + elif "RECONF" in tags: + test_type = "reconf" + # The remaining 3 options could also apply to DPDK and TRex tests. + elif "SOAK" in tags: + test_type = "soak" + elif "NDRPDR" in tags: + test_type = "ndrpdr" + elif "MRR" in tags: + test_type = "mrr" + else: + raise RuntimeError(f"Unable to infer test type from tags: {tags}") + return test_type + + def export_pending_data(self): + """Write the accumulated data to disk. + + Create missing directories. + Reset both file path and data to avoid writing multiple times. + + Functions which finalize content for given file are calling this, + so make sure each test and non-empty suite setup or teardown + is calling this as their last keyword. + + If no file path is set, do not write anything, + as that is the failsafe behavior when caller from unexpected place. + Aso do not write anything when EXPORT_JSON constant is false. + + Regardless of whether data was written, it is cleared. + """ + if not Constants.EXPORT_JSON or not self.file_path: + self.data = None + self.file_path = None + return + new_file_path = write_output(self.file_path, self.data) + # Data is going to be cleared (as a sign that export succeeded), + # so this is the last chance to detect if it was for a test case. + is_testcase = "result" in self.data + self.data = None + # Validation for output goes here when ready. + self.file_path = None + if is_testcase: + validate(new_file_path, self.validators["tc_info"]) + + def warn_on_bad_export(self): + """If bad state is detected, log a warning and clean up state.""" + if self.file_path is not None or self.data is not None: + logger.warn(f"Previous export not clean, path {self.file_path}") + self.data = None + self.file_path = None + + def start_suite_setup_export(self): + """Set new file path, initialize data for the suite setup. + + This has to be called explicitly at start of suite setup, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value("\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(" ", "_") + suite_path_part = os.path.join(*suite_id.split(".")) + output_dir = self.output_dir + self.file_path = os.path.join( + output_dir, suite_path_part, "setup.info.json" + ) + self.data = dict() + self.data["version"] = Constants.MODEL_VERSION + self.data["start_time"] = start_time + self.data["suite_name"] = suite_name + self.data["suite_documentation"] = BuiltIn().get_variable_value( + "\\${SUITE_DOCUMENTATION}" + ) + # "end_time" and "duration" are added on flush. + self.data["hosts"] = set() + self.data["telemetry"] = list() + + def start_test_export(self): + """Set new file path, initialize data to minimal tree for the test case. + + It is assumed Robot variables DUT_TYPE and DUT_VERSION + are already set (in suite setup) to correct values. + + This function has to be called explicitly at the start of test setup, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite and test. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value("\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(" ", "_") + suite_path_part = os.path.join(*suite_id.split(".")) + test_name = BuiltIn().get_variable_value("\\${TEST_NAME}") + self.file_path = os.path.join( + self.output_dir, suite_path_part, + test_name.lower().replace(" ", "_") + ".info.json" + ) + self.data = dict() + self.data["version"] = Constants.MODEL_VERSION + self.data["start_time"] = start_time + self.data["suite_name"] = suite_name + self.data["test_name"] = test_name + test_doc = BuiltIn().get_variable_value("\\${TEST_DOCUMENTATION}", "") + self.data["test_documentation"] = test_doc + # "test_type" is added on flush. + # "tags" is detected and added on flush. + # "end_time" and "duration" is added on flush. + # Robot status and message are added on flush. + self.data["result"] = dict(type="unknown") + self.data["hosts"] = BuiltIn().get_variable_value("\\${hosts}") + self.data["telemetry"] = list() + export_dut_type_and_version() + export_tg_type_and_version() + + def start_suite_teardown_export(self): + """Set new file path, initialize data for the suite teardown. + + This has to be called explicitly at start of suite teardown, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value("\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(" ", "_") + suite_path_part = os.path.join(*suite_id.split(".")) + self.file_path = os.path.join( + self.output_dir, suite_path_part, "teardown.info.json" + ) + self.data = dict() + self.data["version"] = Constants.MODEL_VERSION + self.data["start_time"] = start_time + self.data["suite_name"] = suite_name + # "end_time" and "duration" is added on flush. + self.data["hosts"] = BuiltIn().get_variable_value("\\${hosts}") + self.data["telemetry"] = list() + + def finalize_suite_setup_export(self): + """Add the missing fields to data. Do not write yet. + + Should be run at the end of suite setup. + The write is done at next start (or at the end of global teardown). + """ + end_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.data["hosts"] = BuiltIn().get_variable_value("\\${hosts}") + self.data["end_time"] = end_time + self.export_pending_data() + + def finalize_test_export(self): + """Add the missing fields to data. Do not write yet. + + Should be at the end of test teardown, as the implementation + reads various Robot variables, some of them only available at teardown. + + The write is done at next start (or at the end of global teardown). + """ + end_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + message = BuiltIn().get_variable_value("\\${TEST_MESSAGE}") + test_tags = BuiltIn().get_variable_value("\\${TEST_TAGS}") + self.data["end_time"] = end_time + start_float = parse(self.data["start_time"]).timestamp() + end_float = parse(self.data["end_time"]).timestamp() + self.data["duration"] = end_float - start_float + self.data["tags"] = list(test_tags) + self.data["message"] = message + self.process_passed() + self.process_test_name() + self.process_results() + self.export_pending_data() + + def finalize_suite_teardown_export(self): + """Add the missing fields to data. Do not write yet. + + Should be run at the end of suite teardown + (but before the explicit write in the global suite teardown). + The write is done at next start (or explicitly for global teardown). + """ + end_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.data["end_time"] = end_time + self.export_pending_data() + + def process_test_name(self): + """Replace raw test name with short and long test name and set + test_type. + + Perform in-place edits on the data dictionary. + Remove raw suite_name and test_name, they are not published. + Return early if the data is not for test case. + Insert test ID and long and short test name into the data. + Besides suite_name and test_name, also test tags are read. + + Short test name is basically a suite tag, but with NIC driver prefix, + if the NIC driver used is not the default one (drv_vfio_pci for VPP + tests). + + Long test name has the following form: + {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} + Lookup in test tags is needed to get the threads value. + The threads_and_cores part may be empty, e.g. for TRex tests. + + Test ID has form {suite_name}.{test_name} where the two names come from + Robot variables, converted to lower case and spaces replaces by + undescores. + + Test type is set in an internal function. + + :raises RuntimeError: If the data does not contain expected values. + """ + suite_part = self.data.pop("suite_name").lower().replace(" ", "_") + if "test_name" not in self.data: + # There will be no test_id, provide suite_id instead. + self.data["suite_id"] = suite_part + return + test_part = self.data.pop("test_name").lower().replace(" ", "_") + self.data["test_id"] = f"{suite_part}.{test_part}" + tags = self.data["tags"] + # Test name does not contain thread count. + subparts = test_part.split("-") + if any("tg" in s for s in subparts) and subparts[1] == "": + # Physical core count not detected, assume it is a TRex test. + if "--" not in test_part: + raise RuntimeError(f"Invalid TG test name for: {subparts}") + short_name = test_part.split("--", 1)[1] + else: + short_name = "-".join(subparts[2:]) + # Add threads to test_part. + core_part = subparts[1] + tag = list(filter(lambda t: subparts[1].upper() in t, tags))[0] + test_part = test_part.replace(f"-{core_part}-", f"-{tag.lower()}-") + # For long name we need NIC model, which is only in suite name. + last_suite_part = suite_part.split(".")[-1] + # Short name happens to be the suffix we want to ignore. + prefix_part = last_suite_part.split(short_name)[0] + # Also remove the trailing dash. + prefix_part = prefix_part[:-1] + # Throw away possible link prefix such as "1n1l-". + nic_code = prefix_part.split("-", 1)[-1] + nic_short = Constants.NIC_CODE_TO_SHORT_NAME[nic_code] + long_name = f"{nic_short}-{test_part}" + # Set test type. + test_type = self._detect_test_type() + self.data["test_type"] = test_type + # Remove trailing test type from names (if present). + short_name = short_name.split(f"-{test_type}")[0] + long_name = long_name.split(f"-{test_type}")[0] + # Store names. + self.data["test_name_short"] = short_name + self.data["test_name_long"] = long_name + + def process_passed(self): + """Process the test status information as boolean. + + Boolean is used to make post processing more efficient. + In case the test status is PASS, we will truncate the test message. + """ + status = BuiltIn().get_variable_value("\\${TEST_STATUS}") + if status is not None: + self.data["passed"] = (status == "PASS") + if self.data["passed"]: + # Also truncate success test messages. + self.data["message"] = "" + + def process_results(self): + """Process measured results. + + Results are used to avoid future post processing, making it more + efficient to consume. + """ + if self.data["telemetry"]: + telemetry_encode = "\n".join(self.data["telemetry"]).encode() + telemetry_compress = compress(telemetry_encode, level=9) + telemetry_base64 = b2a_base64(telemetry_compress, newline=False) + self.data["telemetry"] = [telemetry_base64.decode()] + if "result" not in self.data: + return + result_node = self.data["result"] + result_type = result_node["type"] + if result_type == "unknown": + # Device or something else not supported. + return + + # Compute avg and stdev for mrr (rate and bandwidth). + if result_type == "mrr": + for node_name in ("rate", "bandwidth"): + node = result_node["receive_rate"].get(node_name, None) + if node is not None: + stats = AvgStdevStats.for_runs(node["values"]) + node["avg"] = stats.avg + node["stdev"] = stats.stdev + return + + # Multiple processing steps for ndrpdr. + if result_type != "ndrpdr": + return + # Filter out invalid latencies. + for which_key in ("latency_forward", "latency_reverse"): + if which_key not in result_node: + # Probably just an unidir test. + continue + for load in ("pdr_0", "pdr_10", "pdr_50", "pdr_90"): + if result_node[which_key][load]["max"] <= 0: + # One invalid number is enough to remove all loads. + break + else: + # No break means all numbers are ok, nothing to do here. + continue + # Break happened, something is invalid, remove all loads. + result_node.pop(which_key) + return diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py new file mode 100644 index 0000000000..f155848913 --- /dev/null +++ b/resources/libraries/python/model/ExportResult.py @@ -0,0 +1,316 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module with keywords that publish parts of result structure.""" + +from robot.libraries.BuiltIn import BuiltIn + +from resources.libraries.python.model.util import descend, get_export_data + + +def export_dut_type_and_version(dut_type="unknown", dut_version="unknown"): + """Export the arguments as dut type and version. + + Robot tends to convert "none" into None, hence the unusual default values. + + If either argument is missing, the value from robot variable is used. + If argument is present, the value is also stored to robot suite variable. + + :param dut_type: DUT type, e.g. VPP or DPDK. + :param dut_version: DUT version as determined by the caller. + :type dut_type: Optional[str] + :type dut_version: Optiona[str] + :raises RuntimeError: If value is neither in argument not robot variable. + """ + if dut_type == "unknown": + dut_type = BuiltIn().get_variable_value("\\${DUT_TYPE}", "unknown") + if dut_type == "unknown": + raise RuntimeError("Dut type not provided.") + else: + # We want to set a variable in higher level suite setup + # to be available to test setup several levels lower. + BuiltIn().set_suite_variable( + "\\${DUT_TYPE}", dut_type, "children=True" + ) + if dut_version == "unknown": + dut_version = BuiltIn().get_variable_value( + "\\${DUT_VERSION}", "unknown" + ) + if dut_type == "unknown": + raise RuntimeError("Dut version not provided.") + else: + BuiltIn().set_suite_variable( + "\\${DUT_VERSION}", dut_version, "children=True" + ) + data = get_export_data() + data["dut_type"] = dut_type.lower() + data["dut_version"] = dut_version + + +def export_tg_type_and_version(tg_type="unknown", tg_version="unknown"): + """Export the arguments as tg type and version. + + Robot tends to convert "none" into None, hence the unusual default values. + + If either argument is missing, the value from robot variable is used. + If argument is present, the value is also stored to robot suite variable. + + :param tg_type: TG type, e.g. TREX. + :param tg_version: TG version as determined by the caller. + :type tg_type: Optional[str] + :type tg_version: Optiona[str] + :raises RuntimeError: If value is neither in argument not robot variable. + """ + if tg_type == "unknown": + tg_type = BuiltIn().get_variable_value("\\${TG_TYPE}", "unknown") + if tg_type == "unknown": + raise RuntimeError("TG type not provided!") + else: + # We want to set a variable in higher level suite setup + # to be available to test setup several levels lower. + BuiltIn().set_suite_variable( + "\\${TG_TYPE}", tg_type, "children=True" + ) + if tg_version == "unknown": + tg_version = BuiltIn().get_variable_value( + "\\${TG_VERSION}", "unknown" + ) + if tg_type == "unknown": + raise RuntimeError("TG version not provided!") + else: + BuiltIn().set_suite_variable( + "\\${TG_VERSION}", tg_version, "children=True" + ) + data = get_export_data() + data["tg_type"] = tg_type.lower() + data["tg_version"] = tg_version + + +def append_mrr_value(mrr_value, mrr_unit, bandwidth_value=None, + bandwidth_unit="bps"): + """Store mrr value to proper place so it is dumped into json. + + The value is appended only when unit is not empty. + + :param mrr_value: Forwarding rate from MRR trial. + :param mrr_unit: Unit of measurement for the rate. + :param bandwidth_value: The same value recomputed into L1 bits per second. + :type mrr_value: float + :type mrr_unit: str + :type bandwidth_value: Optional[float] + :type bandwidth_unit: Optional[str] + """ + if not mrr_unit: + return + data = get_export_data() + data["result"]["type"] = "mrr" + + for node_val, node_unit, node_name in ((mrr_value, mrr_unit, "rate"), + (bandwidth_value, bandwidth_unit, "bandwidth")): + if node_val is not None: + node = descend(descend(data["result"], "receive_rate"), node_name) + node["unit"] = str(node_unit) + values_list = descend(node, "values", list) + values_list.append(float(node_val)) + + +def export_search_bound(text, value, unit, bandwidth=None): + """Store bound value and unit. + + This function works for both NDRPDR and SOAK, decided by text. + + If a node does not exist, it is created. + If a previous value exists, it is overwritten silently. + Result type is set (overwritten) to ndrpdr (or soak). + + Text is used to determine whether it is ndr or pdr, upper or lower bound, + as the Robot caller has the information only there. + + :param text: Info from Robot caller to determime bound type. + :param value: The bound value in packets (or connections) per second. + :param unit: Rate unit the bound is measured (or estimated) in. + :param bandwidth: The same value recomputed into L1 bits per second. + :type text: str + :type value: float + :type unit: str + :type bandwidth: Optional[float] + """ + value = float(value) + text = str(text).lower() + result_type = "soak" if "plrsearch" in text else "ndrpdr" + upper_or_lower = "upper" if "upper" in text else "lower" + ndr_or_pdr = "ndr" if "ndr" in text else "pdr" + + result_node = get_export_data()["result"] + result_node["type"] = result_type + rate_item = dict(rate=dict(value=value, unit=unit)) + if bandwidth: + rate_item["bandwidth"] = dict(value=float(bandwidth), unit="bps") + if result_type == "soak": + descend(result_node, "critical_rate")[upper_or_lower] = rate_item + return + descend(result_node, ndr_or_pdr)[upper_or_lower] = rate_item + + +def _add_latency(result_node, percent, whichward, latency_string): + """Descend to a corresponding node and add values from latency string. + + This is an internal block, moved out from export_ndrpdr_latency, + as it can be called up to 4 times. + + :param result_node: UTI tree node to descend from. + :param percent: Percent value to use in node key (90, 50, 10, 0). + :param whichward: "forward" or "reverse". + :param latency_item: Unidir output from TRex utility, min/avg/max/hdrh. + :type result_node: dict + :type percent: int + :type whichward: str + :latency_string: str + """ + l_min, l_avg, l_max, l_hdrh = latency_string.split("/", 3) + whichward_node = descend(result_node, f"latency_{whichward}") + percent_node = descend(whichward_node, f"pdr_{percent}") + percent_node["min"] = int(l_min) + percent_node["avg"] = int(l_avg) + percent_node["max"] = int(l_max) + percent_node["hdrh"] = l_hdrh + percent_node["unit"] = "us" + + +def export_ndrpdr_latency(text, latency): + """Store NDRPDR hdrh latency data. + + If "latency" node does not exist, it is created. + If a previous value exists, it is overwritten silently. + + Text is used to determine what percentage of PDR is the load, + as the Robot caller has the information only there. + + Reverse data may be missing, we assume the test was unidirectional. + + :param text: Info from Robot caller to determime load. + :param latency: Output from TRex utility, min/avg/max/hdrh. + :type text: str + :type latency: 1-tuple or 2-tuple of str + """ + result_node = get_export_data()["result"] + percent = 0 + if "90" in text: + percent = 90 + elif "50" in text: + percent = 50 + elif "10" in text: + percent = 10 + _add_latency(result_node, percent, "forward", latency[0]) + # Else TRex does not support latency measurement for this traffic profile. + if len(latency) < 2: + return + _add_latency(result_node, percent, "reverse", latency[1]) + + +def export_reconf_result(packet_rate, packet_loss, bandwidth): + """Export the RECONF type results. + + Result type is set to reconf. + + :param packet_rate: Aggregate offered load in packets per second. + :param packet_loss: How many of the packets were dropped or unsent. + :param bandwidth: The offered load recomputed into L1 bits per second. + :type packet_rate: float + :type packet_loss: int + :type bandwidth: float + """ + result_node = get_export_data()["result"] + result_node["type"] = "reconf" + + time_loss = int(packet_loss) / float(packet_rate) + result_node["aggregate_rate"] = dict( + bandwidth=dict( + unit="bps", + value=float(bandwidth) + ), + rate=dict( + unit="pps", + value=float(packet_rate) + ) + ) + result_node["loss"] = dict( + packet=dict( + unit="packets", + value=int(packet_loss) + ), + time=dict( + unit="s", + value=time_loss + ) + ) + + +def export_hoststack_results( + bandwidth, rate=None, rate_unit=None, latency=None, + failed_requests=None, completed_requests=None, retransmits=None, + duration=None +): + """Export the HOSTSTACK type results. + + Result type is set to hoststack. + + :param bandwidth: Measured transfer rate using bps as a unit. + :param rate: Resulting rate measured by the test. [Optional] + :param rate_unit: CPS or RPS. [Optional] + :param latency: Measure latency. [Optional] + :param failed_requests: Number of failed requests. [Optional] + :param completed_requests: Number of completed requests. [Optional] + :param retransmits: Retransmitted TCP packets. [Optional] + :param duration: Measurment duration. [Optional] + :type bandwidth: float + :type rate: float + :type rate_unit: str + :type latency: float + :type failed_requests: int + :type completed_requests: int + :type retransmits: int + :type duration: float + """ + result_node = get_export_data()["result"] + result_node["type"] = "hoststack" + + result_node["bandwidth"] = dict(unit="bps", value=bandwidth) + if rate is not None: + result_node["rate"] = \ + dict(unit=rate_unit, value=rate) + if latency is not None: + result_node["latency"] = \ + dict(unit="ms", value=latency) + if failed_requests is not None: + result_node["failed_requests"] = \ + dict(unit="requests", value=failed_requests) + if completed_requests is not None: + result_node["completed_requests"] = \ + dict(unit="requests", value=completed_requests) + if retransmits is not None: + result_node["retransmits"] = \ + dict(unit="packets", value=retransmits) + if duration is not None: + result_node["duration"] = \ + dict(unit="s", value=duration) + + +def append_telemetry(telemetry_item): + """Append telemetry entry to proper place so it is dumped into json. + + :param telemetry_item: Telemetry entry. + :type telemetry_item: str + """ + data = get_export_data() + data["telemetry"].append(telemetry_item) diff --git a/resources/libraries/python/model/MemDump.py b/resources/libraries/python/model/MemDump.py new file mode 100644 index 0000000000..b391569286 --- /dev/null +++ b/resources/libraries/python/model/MemDump.py @@ -0,0 +1,194 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for converting in-memory data into JSON output. + +CSIT and VPP PAPI are using custom data types that are not directly serializable +into JSON. + +Thus, before writing the output onto disk, the data is recursively converted to +equivalent serializable types, in extreme cases replaced by string +representation. + +Validation is outside the scope of this module, as it should use the JSON data +read from disk. +""" + +import json +import os + +from collections.abc import Iterable, Mapping, Set +from enum import IntFlag +from dateutil.parser import parse + + +def _pre_serialize_recursive(data): + """Recursively sort and convert to a more serializable form. + + VPP PAPI code can give data with its own MACAddres type, + or various other enum and flag types. + The default json.JSONEncoder method raises TypeError on that. + First point of this function is to apply str() or repr() + to leaf values that need it. + + Also, PAPI responses are namedtuples, which confuses + the json.JSONEncoder method (so it does not recurse). + Dictization (see PapiExecutor) helps somewhat, but it turns namedtuple + into a UserDict, which also confuses json.JSONEncoder. + Therefore, we recursively convert any Mapping into an ordinary dict. + + We also convert iterables to list (sorted if the iterable was a set), + and prevent numbers from getting converted to strings. + + As we are doing such low level operations, + we also convert mapping keys to strings + and sort the mapping items by keys alphabetically, + except "data" field moved to the end. + + :param data: Object to make serializable, dictized when applicable. + :type data: object + :returns: Serializable equivalent of the argument. + :rtype: object + :raises ValueError: If the argument does not support string conversion. + """ + # Recursion ends at scalar values, first handle irregular ones. + if isinstance(data, IntFlag): + return repr(data) + if isinstance(data, bytes): + return data.hex() + # The regular ones are good to go. + if isinstance(data, (str, int, float, bool)): + return data + # Recurse over, convert and sort mappings. + if isinstance(data, Mapping): + # Convert and sort alphabetically. + ret = { + str(key): _pre_serialize_recursive(data[key]) + for key in sorted(data.keys()) + } + # If exists, move "data" field to the end. + if u"data" in ret: + data_value = ret.pop(u"data") + ret[u"data"] = data_value + # If exists, move "type" field at the start. + if u"type" in ret: + type_value = ret.pop(u"type") + ret_old = ret + ret = dict(type=type_value) + ret.update(ret_old) + return ret + # Recurse over and convert iterables. + if isinstance(data, Iterable): + list_data = [_pre_serialize_recursive(item) for item in data] + # Additionally, sets are exported as sorted. + if isinstance(data, Set): + list_data = sorted(list_data) + return list_data + # Unknown structure, attempt str(). + return str(data) + + +def _pre_serialize_root(data): + """Recursively convert to a more serializable form, tweak order. + + See _pre_serialize_recursive for most of changes this does. + + The logic here (outside the recursive function) only affects + field ordering in the root mapping, + to make it more human friendly. + We are moving "version" to the top, + followed by start time and end time. + and various long fields to the bottom. + + Some edits are done in-place, do not trust the argument value after calling. + + :param data: Root data to make serializable, dictized when applicable. + :type data: dict + :returns: Order-tweaked version of the argument. + :rtype: dict + :raises KeyError: If the data does not contain required fields. + :raises TypeError: If the argument is not a dict. + :raises ValueError: If the argument does not support string conversion. + """ + if not isinstance(data, dict): + raise RuntimeError(f"Root data object needs to be a dict: {data!r}") + data = _pre_serialize_recursive(data) + new_data = dict(version=data.pop(u"version")) + new_data[u"start_time"] = data.pop(u"start_time") + new_data[u"end_time"] = data.pop(u"end_time") + new_data.update(data) + return new_data + + +def _merge_into_suite_info_file(teardown_path): + """Move setup and teardown data into a singe file, remove old files. + + The caller has to confirm the argument is correct, e.g. ending in + "/teardown.info.json". + + :param teardown_path: Local filesystem path to teardown file. + :type teardown_path: str + :returns: Local filesystem path to newly created suite file. + :rtype: str + """ + # Manual right replace: https://stackoverflow.com/a/9943875 + setup_path = u"setup".join(teardown_path.rsplit(u"teardown", 1)) + with open(teardown_path, u"rt", encoding="utf-8") as file_in: + teardown_data = json.load(file_in) + # Transforming setup data into suite data. + with open(setup_path, u"rt", encoding="utf-8") as file_in: + suite_data = json.load(file_in) + + end_time = teardown_data[u"end_time"] + suite_data[u"end_time"] = end_time + start_float = parse(suite_data[u"start_time"]).timestamp() + end_float = parse(suite_data[u"end_time"]).timestamp() + suite_data[u"duration"] = end_float - start_float + setup_telemetry = suite_data.pop(u"telemetry") + suite_data[u"setup_telemetry"] = setup_telemetry + suite_data[u"teardown_telemetry"] = teardown_data[u"telemetry"] + + suite_path = u"suite".join(teardown_path.rsplit(u"teardown", 1)) + with open(suite_path, u"wt", encoding="utf-8") as file_out: + json.dump(suite_data, file_out, indent=1) + # We moved everything useful from temporary setup/teardown info files. + os.remove(setup_path) + os.remove(teardown_path) + + return suite_path + + +def write_output(file_path, data): + """Prepare data for serialization and dump into a file. + + Ancestor directories are created if needed. + + :param file_path: Local filesystem path, including the file name. + :param data: Root data to make serializable, dictized when applicable. + :type file_path: str + :type data: dict + """ + data = _pre_serialize_root(data) + + # Lets move Telemetry to the end. + telemetry = data.pop(u"telemetry") + data[u"telemetry"] = telemetry + + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, u"wt", encoding="utf-8") as file_out: + json.dump(data, file_out, indent=1) + + if file_path.endswith(u"/teardown.info.json"): + file_path = _merge_into_suite_info_file(file_path) + + return file_path diff --git a/resources/libraries/python/model/__init__.py b/resources/libraries/python/model/__init__.py new file mode 100644 index 0000000000..36e32b89c4 --- /dev/null +++ b/resources/libraries/python/model/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +__init__ file for directory resources/libraries/python/model +""" diff --git a/resources/libraries/python/model/parse.py b/resources/libraries/python/model/parse.py new file mode 100644 index 0000000000..1e0aebfe18 --- /dev/null +++ b/resources/libraries/python/model/parse.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library for parsing results from JSON back to python objects. + +This is useful for vpp-csit jobs like per-patch performance verify. +Such jobs invoke robot multiple times, each time on a different build. +Each robot invocation may execute several test cases. +How exactly are the results compared depends on the job type, +but extracting just the main results from jsons (file trees) is a common task, +so it is placed into this library. + +As such, the code in this file does not directly interact +with the code in other files in this directory +(result comparison is done outside robot invocation), +but all files share common assumptions about json structure. + +The function here expects a particular tree created on a filesystem by +a bootstrap script, including test results +exported as json files according to a current model schema. +This script extracts the results (according to result type) +and joins them mapping from test IDs to lists of floats. +Also, the result is cached into a results.json file, +so each tree is parsed only once. + +The cached result does not depend on tree placement, +so the bootstrap script may move and copy trees around +before or after parsing. +""" + +import json +import os +import pathlib + +from typing import Dict, List + + +def parse(dirpath: str, fake_value: float = 1.0) -> Dict[str, List[float]]: + """Look for test jsons, extract scalar results. + + Files other than .json are skipped, jsons without test_id are skipped. + If the test failed, four fake values are used as a fake result. + + Units are ignored, as both parent and current are tested + with the same CSIT code so the unit should be identical. + + The test results are sorted by test_id, + as the filesystem order is not deterministic enough. + + The result is also cached as results.json file. + + :param dirpath: Path to the directory tree to examine. + :param fail_value: Fake value to use for test cases that failed. + :type dirpath: str + :type fail_falue: float + :returns: Mapping from test IDs to list of measured values. + :rtype: Dict[str, List[float]] + :raises RuntimeError: On duplicate test ID or unknown test type. + """ + if not pathlib.Path(dirpath).is_dir(): + # This happens when per-patch runs out of iterations. + return {} + resultpath = pathlib.Path(f"{dirpath}/results.json") + if resultpath.is_file(): + with open(resultpath, "rt", encoding="utf8") as file_in: + return json.load(file_in) + results = {} + for root, _, files in os.walk(dirpath): + for filename in files: + if not filename.endswith(".json"): + continue + filepath = os.path.join(root, filename) + with open(filepath, "rt", encoding="utf8") as file_in: + data = json.load(file_in) + if "test_id" not in data: + continue + name = data["test_id"] + if name in results: + raise RuntimeError(f"Duplicate: {name}") + if not data["passed"]: + results[name] = [fake_value] * 4 + continue + result_object = data["result"] + result_type = result_object["type"] + if result_type == "mrr": + results[name] = result_object["receive_rate"]["rate"]["values"] + elif result_type == "ndrpdr": + results[name] = [result_object["pdr"]["lower"]["rate"]["value"]] + elif result_type == "soak": + results[name] = [ + result_object["critical_rate"]["lower"]["rate"]["value"] + ] + elif result_type == "reconf": + results[name] = [result_object["loss"]["time"]["value"]] + elif result_type == "hoststack": + results[name] = [result_object["bandwidth"]["value"]] + else: + raise RuntimeError(f"Unknown result type: {result_type}") + results = {test_id: results[test_id] for test_id in sorted(results)} + with open(resultpath, "wt", encoding="utf8") as file_out: + json.dump(results, file_out, indent=1, separators=(", ", ": ")) + return results diff --git a/resources/libraries/python/model/util.py b/resources/libraries/python/model/util.py new file mode 100644 index 0000000000..db2ef14bbb --- /dev/null +++ b/resources/libraries/python/model/util.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module hosting few utility functions useful when dealing with modelled data. + +This is for storing varied utility functions, which are too short and diverse +to be put into more descriptive modules. +""" + + +from robot.libraries.BuiltIn import BuiltIn + + +def descend(parent_node, key, default_factory=None): + """Return a sub-node, create and insert it when it does not exist. + + Without this function: + child_node = parent_node.get(key, dict()) + parent_node[key] = child_node + + With this function: + child_node = descend(parent_node, key) + + New code is shorter and avoids the need to type key and parent_node twice. + + :param parent_node: Reference to inner node of a larger structure + we want to descend from. + :param key: Key of the maybe existing child node. + :param default_factory: If the key does not exist, call this + to create a new value to be inserted under the key. + None means dict. The other popular option is list. + :type parent_node: dict + :type key: str + :type default_factory: Optional[Callable[[], object]] + :returns: The reference to (maybe just created) child node. + :rtype: object + """ + if key not in parent_node: + factory = dict if default_factory is None else default_factory + parent_node[key] = factory() + return parent_node[key] + + +def get_export_data(): + """Return data member of ExportJson library instance. + + This assumes the data has been initialized already. + Return None if Robot is not running. + + :returns: Current library instance's raw data field. + :rtype: Optional[dict] + :raises AttributeError: If library is not imported yet. + """ + instance = BuiltIn().get_library_instance( + u"resources.libraries.python.model.ExportJson" + ) + if instance is None: + return None + return instance.data diff --git a/resources/libraries/python/model/validate.py b/resources/libraries/python/model/validate.py new file mode 100644 index 0000000000..85c4b993c9 --- /dev/null +++ b/resources/libraries/python/model/validate.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for validating JSON instances against schemas. + +Short module currently, as we validate only testcase info outputs. +Structure will probably change when we start validation mode file types. +""" + +import json +import jsonschema +import yaml + + +def get_validators(): + """Return mapping from file types to validator instances. + + Uses hardcoded file types and paths to schemas on disk. + + :returns: Validators, currently just for tc_info_output. + :rtype: Mapping[str, jsonschema.validators.Validator] + :raises RuntimeError: If schemas are not readable or not valid. + """ + relative_path = "resources/model_schema/test_case.schema.yaml" + # Robot is always started when CWD is CSIT_DIR. + with open(relative_path, "rt", encoding="utf-8") as file_in: + schema = json.loads( + json.dumps(yaml.safe_load(file_in.read()), indent=2) + ) + validator_class = jsonschema.validators.validator_for(schema) + validator_class.check_schema(schema) + fmt_checker = jsonschema.FormatChecker() + validator = validator_class(schema, format_checker=fmt_checker) + + return dict(tc_info=validator) + + +def validate(file_path, validator): + """Load data from disk, use validator to validate it. + + :param file_path: Local filesystem path including the file name to load. + :param validator: Validator instance to use for validation. + :type file_path: str + :type validator: jsonschema.validators.Validator + :raises ValidationError: If schema validation fails. + """ + with open(file_path, "rt", encoding="utf-8") as file_in: + instance = json.load(file_in) + error = jsonschema.exceptions.best_match(validator.iter_errors(instance)) + if error is not None: + print(json.dumps(instance, indent=4)) + raise error |