diff options
18 files changed, 511 insertions, 813 deletions
diff --git a/docs/model/current/schema/test_case.info.schema.json b/docs/model/current/schema/test_case.info.schema.json index d99dd4766a..6aed4d255b 100644 --- a/docs/model/current/schema/test_case.info.schema.json +++ b/docs/model/current/schema/test_case.info.schema.json @@ -1,5 +1,5 @@ { - "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.0.1", + "$id": "https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0", "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Schema for info output of test case.", "allOf": [ @@ -35,9 +35,14 @@ "type": "string" } }, - "log": { - "description": "No log items are implemented in the current version, but the (empty) list is present to simplify logic in multi-version importers.", - "$ref": "#/$defs/types/empty_array" + "telemetry": { + "description": "Array of telemetry entries. Each entry represent one captured metric.", + "type": "array", + "minItems": 0, + "items": { + "description": "Telemetry entry.", + "type": "string" + } }, "message": { "description": "If passed is true, this value is empty. Otherwise, value taken directly from TEST_MESSAGE Robot variable, read at the end of test case (in test teardown, before export and validation). It contains information from the exception that caused the failure, probably with additional exceptions from teardown keywords.", @@ -198,7 +203,7 @@ "version": { "description": "CSIT model version (semver format) the exporting code adhered to.", "type": "string", - "const": "1.0.1" + "const": "1.1.0" } }, "required": [ @@ -207,7 +212,7 @@ "dut_version", "end_time", "hosts", - "log", + "telemetry", "message", "passed", "result", diff --git a/docs/model/current/schema/test_case.info.schema.yaml b/docs/model/current/schema/test_case.info.schema.yaml index 9fd105a972..9f78a41d9e 100644 --- a/docs/model/current/schema/test_case.info.schema.yaml +++ b/docs/model/current/schema/test_case.info.schema.yaml @@ -13,7 +13,7 @@ --- -$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.0.1 +$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.1.0 $schema: https://json-schema.org/draft/2020-12/schema description: >- Schema for info output of test case. @@ -55,12 +55,16 @@ allOf: description: >- Host identifier, usually numeric IPv4 address. type: string - log: + telemetry: description: >- - No log items are implemented in the current version, - but the (empty) list is present to simplify logic - in multi-version importers. - $ref: "#/$defs/types/empty_array" + Array of telemetry entries. Each entry represent one captured + metric. + type: array + minItems: 0 + items: + description: >- + Telemetry entry. + type: string message: description: >- If passed is true, this value is empty. @@ -298,14 +302,14 @@ allOf: CSIT model version (semver format) the exporting code adhered to. type: string - const: 1.0.1 + const: 1.1.0 required: - duration - dut_type - dut_version - end_time - hosts - - log + - telemetry - message - passed - result diff --git a/docs/model/current/top.rst b/docs/model/current/top.rst index ee33a29cd3..e824225698 100644 --- a/docs/model/current/top.rst +++ b/docs/model/current/top.rst @@ -22,7 +22,7 @@ especially the export side (UTI), not import side (PAL). Version ~~~~~~~ -This document is valid for CSIT model version 1.0.1. +This document is valid for CSIT model version 1.1.0. It is recommended to use semantic versioning: https://semver.org/ That means, if the new model misses a field present in the old model, @@ -54,15 +54,7 @@ If the suite name contains spaces (Robot converts underscores to spaces), they are replaced with underscores. The filesystem tree is rooted under tests/ (as suites in git are there), -and for each component (test case, suite setup, suite teardown) -two files are generated. -The "raw" variant is suitable for debugging (can contain lower level logging), -while the "info" variant is suitable for processing by PAL -(can contain derivative values so PAL does not need to compute them -on every download). -Their structure and content is mostly identical, model definition mentions -if a particular subschema is not identical in the two variants. -It is possible to convert from raw to info, but not the other way. +and for each component (test case, suite setup, suite teardown). Although we expect only ASCII text in the exported files, we manipulate files using UTF-8 encoding, diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 1d3898c2d6..b1677c39ae 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -616,29 +616,12 @@ function move_archives () { function post_process_robot_outputs () { # Generate INFO level output_info.xml by rebot. - # Archive UTI raw json outputs. # # Variables read: # - ARCHIVE_DIR - Path to post-processed files. set -exuo pipefail - # Compress raw json outputs, as they will never be post-processed. - pushd "${ARCHIVE_DIR}" || die - if [ -d "tests" ]; then - # Use deterministic order. - options+=("--sort=name") - # We are keeping info outputs where they are. - # Assuming we want to move anything but info files (and dirs). - options+=("--exclude=*.info.json") - tar czf "generated_output_raw.tar.gz" "${options[@]}" "tests" || true - # Tar can remove when archiving, but chokes (not deterministically) - # on attempting to remove dirs (not empty as info files are there). - # So we need to delete the raw files manually. - find "tests" -type f -name "*.raw.json" -delete || true - fi - popd || die - # Generate INFO level output_info.xml for post-processing. all_options=("--loglevel" "INFO") all_options+=("--log" "none") @@ -756,7 +739,6 @@ function run_pybot () { # Run pybot with options based on input variables. # Generate INFO level output_info.xml by rebot. - # Archive UTI raw json outputs. # # Variables read: # - CSIT_DIR - Path to existing root of local CSIT git repository. diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index e91cf6c713..ae1a64d2d2 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -120,7 +120,7 @@ class Constants: """Constants used in CSIT.""" # Version for CSIT data model. See docs/model/. - MODEL_VERSION = u"1.0.1" + MODEL_VERSION = u"1.1.0" # Global off-switch in case JSON export is large or slow. EXPORT_JSON = get_optimistic_bool_from_env(u"EXPORT_JSON") diff --git a/resources/libraries/python/NodePath.py b/resources/libraries/python/NodePath.py index dd68506914..5b445bc593 100644 --- a/resources/libraries/python/NodePath.py +++ b/resources/libraries/python/NodePath.py @@ -243,8 +243,11 @@ class NodePath: :raises RuntimeError: If unsupported combination of parameters. """ t_dict = dict() + t_dict[u"hosts"] = set() if topo_has_dut: duts = [key for key in nodes if u"DUT" in key] + for host in [nodes[dut][u"host"] for dut in duts]: + t_dict[u"hosts"].add(host) t_dict[u"duts"] = duts t_dict[u"duts_count"] = len(duts) t_dict[u"int"] = u"pf" @@ -259,6 +262,7 @@ class NodePath: for dut in duts: self.append_node(nodes[dut], filter_list=filter_list) if topo_has_tg: + t_dict[u"hosts"].add(nodes[u"TG"][u"host"]) if topo_has_dut: self.append_node(nodes[u"TG"]) else: diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py index 6d1332c1b7..bde018a4e0 100644 --- a/resources/libraries/python/SetupFramework.py +++ b/resources/libraries/python/SetupFramework.py @@ -14,8 +14,6 @@ """This module exists to provide setup utilities for the framework on topology nodes. All tasks required to be run before the actual tests are started is supposed to end up here. - -TODO: Figure out how to export JSON from SSH outside main Robot thread. """ from os import environ, remove @@ -108,7 +106,7 @@ def extract_tarball_at_node(tarball, node): node, cmd, message=f"Failed to extract {tarball} at node {node[u'type']} " f"host {node[u'host']}, port {node[u'port']}", - timeout=240, include_reason=True, export=False + timeout=240, include_reason=True ) logger.console( f"Extracting tarball to {con.REMOTE_FW_DIR} on {node[u'type']} " @@ -137,7 +135,7 @@ def create_env_directory_at_node(node): f"&& source env/bin/activate && ANSIBLE_SKIP_CONFLICT_CHECK=1 " \ f"pip3 install -r requirements.txt" stdout, stderr = exec_cmd_no_error( - node, cmd, timeout=300, include_reason=True, export=False, + node, cmd, timeout=300, include_reason=True, message=f"Failed install at node {node[u'type']} host {node[u'host']}, " f"port {node[u'port']}" ) @@ -217,7 +215,7 @@ def delete_framework_dir(node): node, f"sudo rm -rf {con.REMOTE_FW_DIR}", message=f"Framework delete failed at node {node[u'type']} " f"host {node[u'host']}, port {node[u'port']}", - timeout=100, include_reason=True, export=False + timeout=100, include_reason=True, ) logger.console( f"Deleting framework directory on {node[u'type']} host {node[u'host']}," diff --git a/resources/libraries/python/model/ExportJson.py b/resources/libraries/python/model/ExportJson.py new file mode 100644 index 0000000000..b0e0158295 --- /dev/null +++ b/resources/libraries/python/model/ExportJson.py @@ -0,0 +1,393 @@ +# Copyright (c) 2022 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module tracking json in-memory data and saving it to files. + +Each test case, suite setup (hierarchical) and teardown has its own file pair. + +Validation is performed for output files with available JSON schema. +Validation is performed in data deserialized from disk, +as serialization might have introduced subtle errors. +""" + +import datetime +import os.path + +from dateutil.parser import parse +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn + +from resources.libraries.python.Constants import Constants +from resources.libraries.python.jumpavg.AvgStdevStats import AvgStdevStats +from resources.libraries.python.model.ExportResult import ( + export_dut_type_and_version, export_tg_type_and_version +) +from resources.libraries.python.model.MemDump import write_output +from resources.libraries.python.model.validate import ( + get_validators, validate +) + + +class ExportJson(): + """Class handling the json data setting and export.""" + + ROBOT_LIBRARY_SCOPE = u"GLOBAL" + + def __init__(self): + """Declare required fields, cache output dir. + + Also memorize schema validator instances. + """ + self.output_dir = BuiltIn().get_variable_value(u"\\${OUTPUT_DIR}", ".") + self.file_path = None + self.data = None + self.validators = get_validators() + + def _detect_test_type(self): + """Return test_type, as inferred from robot test tags. + + :returns: The inferred test type value. + :rtype: str + :raises RuntimeError: If the test tags does not contain expected values. + """ + tags = self.data[u"tags"] + # First 5 options are specific for VPP tests. + if u"DEVICETEST" in tags: + test_type = u"device" + elif u"LDP_NGINX" in tags: + test_type = u"vsap" + elif u"HOSTSTACK" in tags: + test_type = u"hoststack" + elif u"GSO_TRUE" in tags or u"GSO_FALSE" in tags: + test_type = u"gso" + elif u"RECONF" in tags: + test_type = u"reconf" + # The remaining 3 options could also apply to DPDK and TRex tests. + elif u"SOAK" in tags: + test_type = u"soak" + elif u"NDRPDR" in tags: + test_type = u"ndrpdr" + elif u"MRR" in tags: + test_type = u"mrr" + else: + raise RuntimeError(f"Unable to infer test type from tags: {tags}") + return test_type + + def export_pending_data(self): + """Write the accumulated data to disk. + + Create missing directories. + Reset both file path and data to avoid writing multiple times. + + Functions which finalize content for given file are calling this, + so make sure each test and non-empty suite setup or teardown + is calling this as their last keyword. + + If no file path is set, do not write anything, + as that is the failsafe behavior when caller from unexpected place. + Aso do not write anything when EXPORT_JSON constant is false. + + Regardless of whether data was written, it is cleared. + """ + if not Constants.EXPORT_JSON or not self.file_path: + self.data = None + self.file_path = None + return + new_file_path = write_output(self.file_path, self.data) + # Data is going to be cleared (as a sign that export succeeded), + # so this is the last chance to detect if it was for a test case. + is_testcase = u"result" in self.data + self.data = None + # Validation for output goes here when ready. + self.file_path = None + if is_testcase: + validate(new_file_path, self.validators[u"tc_info"]) + + def warn_on_bad_export(self): + """If bad state is detected, log a warning and clean up state.""" + if self.file_path is not None or self.data is not None: + logger.warn(f"Previous export not clean, path {self.file_path}") + self.data = None + self.file_path = None + + def start_suite_setup_export(self): + """Set new file path, initialize data for the suite setup. + + This has to be called explicitly at start of suite setup, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + u"%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(u" ", u"_") + suite_path_part = os.path.join(*suite_id.split(u".")) + output_dir = self.output_dir + self.file_path = os.path.join( + output_dir, suite_path_part, u"setup.info.json" + ) + self.data = dict() + self.data[u"version"] = Constants.MODEL_VERSION + self.data[u"start_time"] = start_time + self.data[u"suite_name"] = suite_name + self.data[u"suite_documentation"] = BuiltIn().get_variable_value( + u"\\${SUITE_DOCUMENTATION}" + ) + # "end_time" and "duration" are added on flush. + self.data[u"hosts"] = set() + self.data[u"telemetry"] = list() + + def start_test_export(self): + """Set new file path, initialize data to minimal tree for the test case. + + It is assumed Robot variables DUT_TYPE and DUT_VERSION + are already set (in suite setup) to correct values. + + This function has to be called explicitly at the start of test setup, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite and test. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + u"%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(u" ", u"_") + suite_path_part = os.path.join(*suite_id.split(u".")) + test_name = BuiltIn().get_variable_value(u"\\${TEST_NAME}") + self.file_path = os.path.join( + self.output_dir, suite_path_part, + test_name.lower().replace(u" ", u"_") + u".info.json" + ) + self.data = dict() + self.data[u"version"] = Constants.MODEL_VERSION + self.data[u"start_time"] = start_time + self.data[u"suite_name"] = suite_name + self.data[u"test_name"] = test_name + test_doc = BuiltIn().get_variable_value(u"\\${TEST_DOCUMENTATION}", u"") + self.data[u"test_documentation"] = test_doc + # "test_type" is added on flush. + # "tags" is detected and added on flush. + # "end_time" and "duration" is added on flush. + # Robot status and message are added on flush. + self.data[u"result"] = dict(type=u"unknown") + self.data[u"hosts"] = BuiltIn().get_variable_value(u"\\${hosts}") + self.data[u"telemetry"] = list() + export_dut_type_and_version() + export_tg_type_and_version() + + def start_suite_teardown_export(self): + """Set new file path, initialize data for the suite teardown. + + This has to be called explicitly at start of suite teardown, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + u"%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(u" ", u"_") + suite_path_part = os.path.join(*suite_id.split(u".")) + self.file_path = os.path.join( + self.output_dir, suite_path_part, u"teardown.info.json" + ) + self.data = dict() + self.data[u"version"] = Constants.MODEL_VERSION + self.data[u"start_time"] = start_time + self.data[u"suite_name"] = suite_name + # "end_time" and "duration" is added on flush. + self.data[u"hosts"] = BuiltIn().get_variable_value(u"\\${hosts}") + self.data[u"telemetry"] = list() + + def finalize_suite_setup_export(self): + """Add the missing fields to data. Do not write yet. + + Should be run at the end of suite setup. + The write is done at next start (or at the end of global teardown). + """ + end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") + self.data[u"hosts"] = BuiltIn().get_variable_value(u"\\${hosts}") + self.data[u"end_time"] = end_time + self.export_pending_data() + + def finalize_test_export(self): + """Add the missing fields to data. Do not write yet. + + Should be at the end of test teardown, as the implementation + reads various Robot variables, some of them only available at teardown. + + The write is done at next start (or at the end of global teardown). + """ + end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") + message = BuiltIn().get_variable_value(u"\\${TEST_MESSAGE}") + test_tags = BuiltIn().get_variable_value(u"\\${TEST_TAGS}") + self.data[u"end_time"] = end_time + start_float = parse(self.data[u"start_time"]).timestamp() + end_float = parse(self.data[u"end_time"]).timestamp() + self.data[u"duration"] = end_float - start_float + self.data[u"tags"] = list(test_tags) + self.data[u"message"] = message + self.process_passed() + self.process_test_name() + self.process_results() + self.export_pending_data() + + def finalize_suite_teardown_export(self): + """Add the missing fields to data. Do not write yet. + + Should be run at the end of suite teardown + (but before the explicit write in the global suite teardown). + The write is done at next start (or explicitly for global teardown). + """ + end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") + self.data[u"end_time"] = end_time + self.export_pending_data() + + def process_test_name(self): + """Replace raw test name with short and long test name and set + test_type. + + Perform in-place edits on the data dictionary. + Remove raw suite_name and test_name, they are not published. + Return early if the data is not for test case. + Insert test ID and long and short test name into the data. + Besides suite_name and test_name, also test tags are read. + + Short test name is basically a suite tag, but with NIC driver prefix, + if the NIC driver used is not the default one (drv_vfio_pci for VPP + tests). + + Long test name has the following form: + {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} + Lookup in test tags is needed to get the threads value. + The threads_and_cores part may be empty, e.g. for TRex tests. + + Test ID has form {suite_name}.{test_name} where the two names come from + Robot variables, converted to lower case and spaces replaces by + undescores. + + Test type is set in an internal function. + + :raises RuntimeError: If the data does not contain expected values. + """ + suite_part = self.data.pop(u"suite_name").lower().replace(u" ", u"_") + if u"test_name" not in self.data: + # There will be no test_id, provide suite_id instead. + self.data[u"suite_id"] = suite_part + return + test_part = self.data.pop(u"test_name").lower().replace(u" ", u"_") + self.data[u"test_id"] = f"{suite_part}.{test_part}" + tags = self.data[u"tags"] + # Test name does not contain thread count. + subparts = test_part.split(u"c-", 1) + if len(subparts) < 2 or subparts[0][-2:-1] != u"-": + # Physical core count not detected, assume it is a TRex test. + if u"--" not in test_part: + raise RuntimeError(f"Cores not found for {subparts}") + short_name = test_part.split(u"--", 1)[1] + else: + short_name = subparts[1] + # Add threads to test_part. + core_part = subparts[0][-1] + u"c" + for tag in tags: + tag = tag.lower() + if len(tag) == 4 and core_part == tag[2:] and tag[1] == u"t": + test_part = test_part.replace(f"-{core_part}-", f"-{tag}-") + break + else: + raise RuntimeError( + f"Threads not found for {test_part} tags {tags}" + ) + # For long name we need NIC model, which is only in suite name. + last_suite_part = suite_part.split(u".")[-1] + # Short name happens to be the suffix we want to ignore. + prefix_part = last_suite_part.split(short_name)[0] + # Also remove the trailing dash. + prefix_part = prefix_part[:-1] + # Throw away possible link prefix such as "1n1l-". + nic_code = prefix_part.split(u"-", 1)[-1] + nic_short = Constants.NIC_CODE_TO_SHORT_NAME[nic_code] + long_name = f"{nic_short}-{test_part}" + # Set test type. + test_type = self._detect_test_type() + self.data[u"test_type"] = test_type + # Remove trailing test type from names (if present). + short_name = short_name.split(f"-{test_type}")[0] + long_name = long_name.split(f"-{test_type}")[0] + # Store names. + self.data[u"test_name_short"] = short_name + self.data[u"test_name_long"] = long_name + + def process_passed(self): + """Process the test status information as boolean. + + Boolean is used to make post processing more efficient. + In case the test status is PASS, we will truncate the test message. + """ + status = BuiltIn().get_variable_value(u"\\${TEST_STATUS}") + if status is not None: + self.data[u"passed"] = (status == u"PASS") + if self.data[u"passed"]: + # Also truncate success test messages. + self.data[u"message"] = u"" + + def process_results(self): + """Process measured results. + + Results are used to avoid future post processing, making it more + efficient to consume. + """ + if u"result" not in self.data: + return + result_node = self.data[u"result"] + result_type = result_node[u"type"] + if result_type == u"unknown": + # Device or something else not supported. + return + + # Compute avg and stdev for mrr. + if result_type == u"mrr": + rate_node = result_node[u"receive_rate"][u"rate"] + stats = AvgStdevStats.for_runs(rate_node[u"values"]) + rate_node[u"avg"] = stats.avg + rate_node[u"stdev"] = stats.stdev + return + + # Multiple processing steps for ndrpdr. + if result_type != u"ndrpdr": + return + # Filter out invalid latencies. + for which_key in (u"latency_forward", u"latency_reverse"): + if which_key not in result_node: + # Probably just an unidir test. + continue + for load in (u"pdr_0", u"pdr_10", u"pdr_50", u"pdr_90"): + if result_node[which_key][load][u"max"] <= 0: + # One invalid number is enough to remove all loads. + break + else: + # No break means all numbers are ok, nothing to do here. + continue + # Break happened, something is invalid, remove all loads. + result_node.pop(which_key) + return diff --git a/resources/libraries/python/model/ExportLog.py b/resources/libraries/python/model/ExportLog.py deleted file mode 100644 index e02eef63c5..0000000000 --- a/resources/libraries/python/model/ExportLog.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) 2021 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module with keywords that publish metric and other log events. -""" - -import datetime - -from resources.libraries.python.model.util import get_export_data - - -def export_ssh_command(host, port, command): - """Add a log item about SSH command execution starting. - - The log item is present only in raw output. - Result arrives in a separate log item. - Log level is always DEBUG. - - The command is stored as "data" (not "msg") as in some cases - the command can be too long to act as a message. - - The host is added to the info set of hosts. - - :param host: Node "host" attribute, usually its IPv4 address. - :param port: SSH port number to use when connecting to the host. - :param command: Serialized bash command to execute. - :type host: str - :type port: int - :type command: str - """ - timestamp = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - data = get_export_data() - ssh_record = dict( - source_type=u"host,port", - source_id=dict(host=host, port=port), - msg_type=u"ssh_command", - log_level=u"DEBUG", - timestamp=timestamp, - msg="", - data=str(command), - ) - data[u"hosts"].add(host) - data[u"log"].append(ssh_record) - - -def export_ssh_result(host, port, code, stdout, stderr, duration): - """Add a log item about ssh execution result. - - Only for raw output log. - - There is no easy way to pair with the corresponding command, - but usually there is only one SSH session for given host and port. - The duration value may give a hint if that is not the case. - - Message is empty, data has fields "rc", "stdout", "stderr" and "duration". - Log level is always DEBUG. - - The host is NOT added to the info set of hosts, as each result - comes after a command. - - TODO: Do not require duration, find preceding ssh command in log. - Reason: Pylint complains about too many arguments. - Alternative: Define type for SSH endopoint (and use that instead host+port). - - :param host: Node "host" attribute, usually its IPv4 address. - :param port: SSH port number to use when connecting to the host. - :param code: Bash return code, e.g. 0 for success. - :param stdout: Captured standard output of the command execution. - :param stderr: Captured error output of the command execution. - :param duration: How long has the command been executing, in seconds. - :type host: str - :type port: int - :type code: int - :type stdout: str - :type stderr: str - :type duration: float - """ - timestamp = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - data = get_export_data() - ssh_record = dict( - source_type=u"host,port", - source_id=dict(host=host, port=port), - msg_type=u"ssh_result", - log_level=u"DEBUG", - timestamp=timestamp, - msg=u"", - data=dict( - rc=int(code), - stdout=str(stdout), - stderr=str(stderr), - duration=float(duration), - ), - ) - data[u"log"].append(ssh_record) - - -def export_ssh_timeout(host, port, stdout, stderr, duration): - """Add a log item about ssh execution timing out. - - Only for debug log. - - There is no easy way to pair with the corresponding command, - but usually there is only one SSH session for given host and port. - - Message is empty, data has fields "stdout", "stderr" and "duration". - The duration value may give a hint if that is not the case. - Log level is always DEBUG. - - The host is NOT added to the info set of hosts, as each timeout - comes after a command. - - :param host: Node "host" attribute, usually its IPv4 address. - :param port: SSH port number to use when connecting to the host. - :param stdout: Captured standard output of the command execution so far. - :param stderr: Captured error output of the command execution so far. - :param duration: How long has the command been executing, in seconds. - :type host: str - :type port: int - :type stdout: str - :type stderr: str - :type duration: float - """ - timestamp = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - data = get_export_data() - ssh_record = dict( - source_type=u"host,port", - source_id=dict(host=host, port=port), - msg_type=u"ssh_timeout", - log_level=u"DEBUG", - timestamp=timestamp, - msg=u"", - data=dict( - stdout=str(stdout), - stderr=str(stderr), - duration=float(duration), - ), - ) - data[u"log"].append(ssh_record) diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py index 16c6b89fb3..dbe2914565 100644 --- a/resources/libraries/python/model/ExportResult.py +++ b/resources/libraries/python/model/ExportResult.py @@ -114,7 +114,6 @@ def append_mrr_value(mrr_value, unit): rate_node[u"unit"] = str(unit) values_list = descend(rate_node, u"values", list) values_list.append(float(mrr_value)) - # TODO: Fill in the bandwidth part for pps? def export_search_bound(text, value, unit, bandwidth=None): diff --git a/resources/libraries/python/model/mem2raw.py b/resources/libraries/python/model/MemDump.py index 543ee935e2..bf8835244b 100644 --- a/resources/libraries/python/model/mem2raw.py +++ b/resources/libraries/python/model/MemDump.py @@ -11,17 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module for converting in-memory data into raw JSON output. +"""Module for converting in-memory data into JSON output. -CSIT and VPP PAPI are using custom data types -that are not directly serializable into JSON. +CSIT and VPP PAPI are using custom data types that are not directly serializable +into JSON. -Thus, before writing the raw outpt onto disk, -the data is recursively converted to equivalent serializable types, -in extreme cases replaced by string representation. +Thus, before writing the output onto disk, the data is recursively converted to +equivalent serializable types, in extreme cases replaced by string +representation. -Validation is outside the scope of this module, -as it should use the JSON data read from disk. +Validation is outside the scope of this module, as it should use the JSON data +read from disk. """ import json @@ -29,6 +29,7 @@ import os from collections.abc import Iterable, Mapping, Set from enum import IntFlag +from dateutil.parser import parse def _pre_serialize_recursive(data): @@ -107,7 +108,7 @@ def _pre_serialize_root(data): to make it more human friendly. We are moving "version" to the top, followed by start time and end time. - and various long fields (such as "log") to the bottom. + and various long fields to the bottom. Some edits are done in-place, do not trust the argument value after calling. @@ -122,24 +123,72 @@ def _pre_serialize_root(data): if not isinstance(data, dict): raise RuntimeError(f"Root data object needs to be a dict: {data!r}") data = _pre_serialize_recursive(data) - log = data.pop(u"log") new_data = dict(version=data.pop(u"version")) new_data[u"start_time"] = data.pop(u"start_time") new_data[u"end_time"] = data.pop(u"end_time") new_data.update(data) - new_data[u"log"] = log return new_data -def write_raw_output(raw_file_path, raw_data): +def _merge_into_suite_info_file(teardown_path): + """Move setup and teardown data into a singe file, remove old files. + + The caller has to confirm the argument is correct, e.g. ending in + "/teardown.info.json". + + :param teardown_path: Local filesystem path to teardown file. + :type teardown_path: str + :returns: Local filesystem path to newly created suite file. + :rtype: str + """ + # Manual right replace: https://stackoverflow.com/a/9943875 + setup_path = u"setup".join(teardown_path.rsplit(u"teardown", 1)) + with open(teardown_path, u"rt", encoding="utf-8") as file_in: + teardown_data = json.load(file_in) + # Transforming setup data into suite data. + with open(setup_path, u"rt", encoding="utf-8") as file_in: + suite_data = json.load(file_in) + + end_time = teardown_data[u"end_time"] + suite_data[u"end_time"] = end_time + start_float = parse(suite_data[u"start_time"]).timestamp() + end_float = parse(suite_data[u"end_time"]).timestamp() + suite_data[u"duration"] = end_float - start_float + setup_telemetry = suite_data.pop(u"telemetry") + suite_data[u"setup_telemetry"] = setup_telemetry + suite_data[u"teardown_telemetry"] = teardown_data[u"telemetry"] + + suite_path = u"suite".join(teardown_path.rsplit(u"teardown", 1)) + with open(suite_path, u"wt", encoding="utf-8") as file_out: + json.dump(suite_data, file_out, indent=1) + # We moved everything useful from temporary setup/teardown info files. + os.remove(setup_path) + os.remove(teardown_path) + + return suite_path + + +def write_output(file_path, data): """Prepare data for serialization and dump into a file. Ancestor directories are created if needed. - :param to_raw_path: Local filesystem path, including the file name. - :type to_raw_path: str + :param file_path: Local filesystem path, including the file name. + :param data: Root data to make serializable, dictized when applicable. + :type file_path: str + :type data: dict """ - raw_data = _pre_serialize_root(raw_data) - os.makedirs(os.path.dirname(raw_file_path), exist_ok=True) - with open(raw_file_path, u"wt", encoding="utf-8") as file_out: - json.dump(raw_data, file_out, indent=1) + data = _pre_serialize_root(data) + + # Lets move Telemetry to the end. + telemetry = data.pop(u"telemetry") + data[u"telemetry"] = telemetry + + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, u"wt", encoding="utf-8") as file_out: + json.dump(data, file_out, indent=1) + + if file_path.endswith(u"/teardown.info.json"): + file_path = _merge_into_suite_info_file(file_path) + + return file_path diff --git a/resources/libraries/python/model/export_json.py b/resources/libraries/python/model/export_json.py deleted file mode 100644 index 840c49fa70..0000000000 --- a/resources/libraries/python/model/export_json.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (c) 2022 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module tracking json in-memory data and saving it to files. - -The current implementation tracks data for raw output, -and info output is created from raw output on disk (see raw2info module). -Raw file contains all log items but no derived quantities, -info file contains only important log items but also derived quantities. -The overlap between two files is big. - -Each test case, suite setup (hierarchical) and teardown has its own file pair. - -Validation is performed for output files with available JSON schema. -Validation is performed in data deserialized from disk, -as serialization might have introduced subtle errors. -""" - -import datetime -import os.path - -from robot.api import logger -from robot.libraries.BuiltIn import BuiltIn - -from resources.libraries.python.Constants import Constants -from resources.libraries.python.model.ExportResult import ( - export_dut_type_and_version, export_tg_type_and_version -) -from resources.libraries.python.model.mem2raw import write_raw_output -from resources.libraries.python.model.raw2info import convert_content_to_info -from resources.libraries.python.model.validate import (get_validators, validate) - - -class export_json(): - """Class handling the json data setting and export.""" - - ROBOT_LIBRARY_SCOPE = u"GLOBAL" - - def __init__(self): - """Declare required fields, cache output dir. - - Also memorize schema validator instances. - """ - self.output_dir = BuiltIn().get_variable_value(u"\\${OUTPUT_DIR}", ".") - self.raw_file_path = None - self.raw_data = None - self.validators = get_validators() - - def export_pending_data(self): - """Write the accumulated data to disk. - - Create missing directories. - Reset both file path and data to avoid writing multiple times. - - Functions which finalize content for given file are calling this, - so make sure each test and non-empty suite setup or teardown - is calling this as their last keyword. - - If no file path is set, do not write anything, - as that is the failsafe behavior when caller from unexpected place. - Aso do not write anything when EXPORT_JSON constant is false. - - Regardless of whether data was written, it is cleared. - """ - if not Constants.EXPORT_JSON or not self.raw_file_path: - self.raw_data = None - self.raw_file_path = None - return - write_raw_output(self.raw_file_path, self.raw_data) - # Raw data is going to be cleared (as a sign that raw export succeeded), - # so this is the last chance to detect if it was for a test case. - is_testcase = u"result" in self.raw_data - self.raw_data = None - # Validation for raw output goes here when ready. - info_file_path = convert_content_to_info(self.raw_file_path) - self.raw_file_path = None - # If "result" is missing from info content, - # it could be a bug in conversion from raw test case content, - # so instead of that we use the flag detected earlier. - if is_testcase: - validate(info_file_path, self.validators[u"tc_info"]) - - def warn_on_bad_export(self): - """If bad state is detected, log a warning and clean up state.""" - if self.raw_file_path is not None or self.raw_data is not None: - logger.warn(f"Previous export not clean, path {self.raw_file_path}") - self.raw_data = None - self.raw_file_path = None - - def start_suite_setup_export(self): - """Set new file path, initialize data for the suite setup. - - This has to be called explicitly at start of suite setup, - otherwise Robot likes to postpone initialization - until first call by a data-adding keyword. - - File path is set based on suite. - """ - self.warn_on_bad_export() - start_time = datetime.datetime.utcnow().strftime( - u"%Y-%m-%dT%H:%M:%S.%fZ" - ) - suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") - suite_id = suite_name.lower().replace(u" ", u"_") - suite_path_part = os.path.join(*suite_id.split(u".")) - output_dir = self.output_dir - self.raw_file_path = os.path.join( - output_dir, suite_path_part, u"setup.raw.json" - ) - self.raw_data = dict() - self.raw_data[u"version"] = Constants.MODEL_VERSION - self.raw_data[u"start_time"] = start_time - self.raw_data[u"suite_name"] = suite_name - self.raw_data[u"suite_documentation"] = BuiltIn().get_variable_value( - u"\\${SUITE_DOCUMENTATION}" - ) - # "end_time" and "duration" is added on flush. - self.raw_data[u"hosts"] = set() - self.raw_data[u"log"] = list() - - def start_test_export(self): - """Set new file path, initialize data to minimal tree for the test case. - - It is assumed Robot variables DUT_TYPE and DUT_VERSION - are already set (in suite setup) to correct values. - - This function has to be called explicitly at the start of test setup, - otherwise Robot likes to postpone initialization - until first call by a data-adding keyword. - - File path is set based on suite and test. - """ - self.warn_on_bad_export() - start_time = datetime.datetime.utcnow().strftime( - u"%Y-%m-%dT%H:%M:%S.%fZ" - ) - suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") - suite_id = suite_name.lower().replace(u" ", u"_") - suite_path_part = os.path.join(*suite_id.split(u".")) - test_name = BuiltIn().get_variable_value(u"\\${TEST_NAME}") - self.raw_file_path = os.path.join( - self.output_dir, suite_path_part, - test_name.lower().replace(u" ", u"_") + u".raw.json" - ) - self.raw_data = dict() - self.raw_data[u"version"] = Constants.MODEL_VERSION - self.raw_data[u"start_time"] = start_time - self.raw_data[u"suite_name"] = suite_name - self.raw_data[u"test_name"] = test_name - test_doc = BuiltIn().get_variable_value(u"\\${TEST_DOCUMENTATION}", u"") - self.raw_data[u"test_documentation"] = test_doc - # "test_type" is added when converting to info. - # "tags" is detected and added on flush. - # "end_time" and "duration" is added on flush. - # Robot status and message are added on flush. - self.raw_data[u"result"] = dict(type=u"unknown") - self.raw_data[u"hosts"] = set() - self.raw_data[u"log"] = list() - export_dut_type_and_version() - export_tg_type_and_version() - - def start_suite_teardown_export(self): - """Set new file path, initialize data for the suite teardown. - - This has to be called explicitly at start of suite teardown, - otherwise Robot likes to postpone initialization - until first call by a data-adding keyword. - - File path is set based on suite. - """ - self.warn_on_bad_export() - start_time = datetime.datetime.utcnow().strftime( - u"%Y-%m-%dT%H:%M:%S.%fZ" - ) - suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") - suite_id = suite_name.lower().replace(u" ", u"_") - suite_path_part = os.path.join(*suite_id.split(u".")) - self.raw_file_path = os.path.join( - self.output_dir, suite_path_part, u"teardown.raw.json" - ) - self.raw_data = dict() - self.raw_data[u"version"] = Constants.MODEL_VERSION - self.raw_data[u"start_time"] = start_time - self.raw_data[u"suite_name"] = suite_name - # "end_time" and "duration" is added on flush. - self.raw_data[u"hosts"] = set() - self.raw_data[u"log"] = list() - - def finalize_suite_setup_export(self): - """Add the missing fields to data. Do not write yet. - - Should be run at the end of suite setup. - The write is done at next start (or at the end of global teardown). - """ - end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - self.raw_data[u"end_time"] = end_time - self.export_pending_data() - - def finalize_test_export(self): - """Add the missing fields to data. Do not write yet. - - Should be at the end of test teardown, as the implementation - reads various Robot variables, some of them only available at teardown. - - The write is done at next start (or at the end of global teardown). - """ - end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - message = BuiltIn().get_variable_value(u"\\${TEST_MESSAGE}") - status = BuiltIn().get_variable_value(u"\\${TEST_STATUS}") - test_tags = BuiltIn().get_variable_value(u"\\${TEST_TAGS}") - self.raw_data[u"end_time"] = end_time - self.raw_data[u"tags"] = list(test_tags) - self.raw_data[u"status"] = status - self.raw_data[u"message"] = message - self.export_pending_data() - - def finalize_suite_teardown_export(self): - """Add the missing fields to data. Do not write yet. - - Should be run at the end of suite teardown - (but before the explicit write in the global suite teardown). - The write is done at next start (or explicitly for global teardown). - """ - end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") - self.raw_data[u"end_time"] = end_time - self.export_pending_data() diff --git a/resources/libraries/python/model/raw2info.py b/resources/libraries/python/model/raw2info.py deleted file mode 100644 index bd7d0e3cf1..0000000000 --- a/resources/libraries/python/model/raw2info.py +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright (c) 2022 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module facilitating conversion from raw outputs into info outputs.""" - -import copy -import json -import os - -import dateutil.parser - -from resources.libraries.python.Constants import Constants -from resources.libraries.python.jumpavg.AvgStdevStats import AvgStdevStats - - -def _raw_to_info_path(raw_path): - """Compute path for info output corresponding to given raw output. - - :param raw_path: Local filesystem path to read raw JSON data from. - :type raw_path: str - :returns: Local filesystem path to write info JSON content to. - :rtype: str - :raises RuntimeError: If the input path does not meet all expectations. - """ - raw_extension = u".raw.json" - tmp_parts = raw_path.split(raw_extension) - if len(tmp_parts) != 2 or tmp_parts[1] != u"": - raise RuntimeError(f"Not good extension {raw_extension}: {raw_path}") - info_path = tmp_parts[0] + u".info.json" - return info_path - - -def _process_test_name(data): - """Replace raw test name with short and long test name and set test_type. - - Perform in-place edits on the data dictionary. - Remove raw suite_name and test_name, they are not part of info schema. - Return early if the data is not for test case. - Inserttest ID and long and short test name into the data. - Besides suite_name and test_name, also test tags are read. - - Short test name is basically a suite tag, but with NIC driver prefix, - if the NIC driver used is not the default one (drv_vfio_pci for VPP tests). - - Long test name has the following form: - {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} - Lookup in test tags is needed to get the threads value. - The threads_and_cores part may be empty, e.g. for TRex tests. - - Test ID has form {suite_name}.{test_name} where the two names come from - Robot variables, converted to lower case and spaces replaces by undescores. - - Test type is set in an internal function. - - :param data: Raw data, perhaps some fields converted into info data already. - :type data: dict - :raises RuntimeError: If the raw data does not contain expected values. - """ - suite_part = data.pop(u"suite_name").lower().replace(u" ", u"_") - if u"test_name" not in data: - # There will be no test_id, provide suite_id instead. - data[u"suite_id"] = suite_part - return - test_part = data.pop(u"test_name").lower().replace(u" ", u"_") - data[u"test_id"] = f"{suite_part}.{test_part}" - tags = data[u"tags"] - # Test name does not contain thread count. - subparts = test_part.split(u"c-", 1) - if len(subparts) < 2 or subparts[0][-2:-1] != u"-": - # Physical core count not detected, assume it is a TRex test. - if u"--" not in test_part: - raise RuntimeError(f"Cores not found for {subparts}") - short_name = test_part.split(u"--", 1)[1] - else: - short_name = subparts[1] - # Add threads to test_part. - core_part = subparts[0][-1] + u"c" - for tag in tags: - tag = tag.lower() - if len(tag) == 4 and core_part == tag[2:] and tag[1] == u"t": - test_part = test_part.replace(f"-{core_part}-", f"-{tag}-") - break - else: - raise RuntimeError(f"Threads not found for {test_part} tags {tags}") - # For long name we need NIC model, which is only in suite name. - last_suite_part = suite_part.split(u".")[-1] - # Short name happens to be the suffix we want to ignore. - prefix_part = last_suite_part.split(short_name)[0] - # Also remove the trailing dash. - prefix_part = prefix_part[:-1] - # Throw away possible link prefix such as "1n1l-". - nic_code = prefix_part.split(u"-", 1)[-1] - nic_short = Constants.NIC_CODE_TO_SHORT_NAME[nic_code] - long_name = f"{nic_short}-{test_part}" - # Set test type. - test_type = _detect_test_type(data) - data[u"test_type"] = test_type - # Remove trailing test type from names (if present). - short_name = short_name.split(f"-{test_type}")[0] - long_name = long_name.split(f"-{test_type}")[0] - # Store names. - data[u"test_name_short"] = short_name - data[u"test_name_long"] = long_name - - -def _detect_test_type(data): - """Return test_type, as inferred from robot test tags. - - :param data: Raw data, perhaps some fields converted into info data already. - :type data: dict - :returns: The inferred test type value. - :rtype: str - :raises RuntimeError: If the test tags does not contain expected values. - """ - tags = data[u"tags"] - # First 5 options are specific for VPP tests. - if u"DEVICETEST" in tags: - test_type = u"device" - elif u"LDP_NGINX" in tags: - test_type = u"vsap" - elif u"HOSTSTACK" in tags: - test_type = u"hoststack" - elif u"GSO_TRUE" in tags or u"GSO_FALSE" in tags: - test_type = u"gso" - elif u"RECONF" in tags: - test_type = u"reconf" - # The remaining 3 options could also apply to DPDK and TRex tests. - elif u"SOAK" in tags: - test_type = u"soak" - elif u"NDRPDR" in tags: - test_type = u"ndrpdr" - elif u"MRR" in tags: - test_type = u"mrr" - else: - raise RuntimeError(f"Unable to infer test type from tags: {tags}") - return test_type - - -def _convert_to_info_in_memory(data): - """Perform all changes needed for processing of data, return new data. - - Data is assumed to be valid for raw schema, so no exceptions are expected. - The original argument object is not edited, - a new copy is created for edits and returned, - because there is no easy way to sort keys in-place. - - :param data: The whole composite object to filter and enhance. - :type data: dict - :returns: New object with the edited content. - :rtype: dict - """ - data = copy.deepcopy(data) - - # Drop any SSH log items. - data[u"log"] = list() - - # Duration is computed for every file. - start_float = dateutil.parser.parse(data[u"start_time"]).timestamp() - end_float = dateutil.parser.parse(data[u"end_time"]).timestamp() - data[u"duration"] = end_float - start_float - - # Reorder impotant fields to the top. - sorted_data = dict(version=data.pop(u"version")) - sorted_data[u"duration"] = data.pop(u"duration") - sorted_data[u"start_time"] = data.pop(u"start_time") - sorted_data[u"end_time"] = data.pop(u"end_time") - sorted_data.update(data) - data = sorted_data - # TODO: Do we care about the order of subsequently added fields? - - # Convert status into a boolean. - status = data.pop(u"status", None) - if status is not None: - data[u"passed"] = (status == u"PASS") - if data[u"passed"]: - # Also truncate success test messages. - data[u"message"] = u"" - - # Replace raw names with processed ones, set test_id and test_type. - _process_test_name(data) - - # The rest is only relevant for test case outputs. - if u"result" not in data: - return data - result_node = data[u"result"] - result_type = result_node[u"type"] - if result_type == u"unknown": - # Device or something else not supported. - return data - - # More processing depending on result type. TODO: Separate functions? - - # Compute avg and stdev for mrr. - if result_type == u"mrr": - rate_node = result_node[u"receive_rate"][u"rate"] - stats = AvgStdevStats.for_runs(rate_node[u"values"]) - rate_node[u"avg"] = stats.avg - rate_node[u"stdev"] = stats.stdev - - # Multiple processing steps for ndrpdr. - if result_type != u"ndrpdr": - return data - # Filter out invalid latencies. - for which_key in (u"latency_forward", u"latency_reverse"): - if which_key not in result_node: - # Probably just an unidir test. - continue - for load in (u"pdr_0", u"pdr_10", u"pdr_50", u"pdr_90"): - if result_node[which_key][load][u"max"] <= 0: - # One invalid number is enough to remove all loads. - break - else: - # No break means all numbers are ok, nothing to do here. - continue - # Break happened, something is invalid, remove all loads. - result_node.pop(which_key) - - return data - - -def _merge_into_suite_info_file(teardown_info_path): - """Move setup and teardown data into a singe file, remove old files. - - The caller has to confirm the argument is correct, e.g. ending in - "/teardown.info.json". - - :param teardown_info_path: Local filesystem path to teardown info file. - :type teardown_info_path: str - :returns: Local filesystem path to newly created suite info file. - :rtype: str - """ - # Manual right replace: https://stackoverflow.com/a/9943875 - setup_info_path = u"setup".join(teardown_info_path.rsplit(u"teardown", 1)) - with open(teardown_info_path, u"rt", encoding="utf-8") as file_in: - teardown_data = json.load(file_in) - # Transforming setup data into suite data. - with open(setup_info_path, u"rt", encoding="utf-8") as file_in: - suite_data = json.load(file_in) - - end_time = teardown_data[u"end_time"] - suite_data[u"end_time"] = end_time - start_float = dateutil.parser.parse(suite_data[u"start_time"]).timestamp() - end_float = dateutil.parser.parse(suite_data[u"end_time"]).timestamp() - suite_data[u"duration"] = end_float - start_float - setup_log = suite_data.pop(u"log") - suite_data[u"setup_log"] = setup_log - suite_data[u"teardown_log"] = teardown_data[u"log"] - - suite_info_path = u"suite".join(teardown_info_path.rsplit(u"teardown", 1)) - with open(suite_info_path, u"wt", encoding="utf-8") as file_out: - json.dump(suite_data, file_out, indent=1) - # We moved everything useful from temporary setup/teardown info files. - os.remove(setup_info_path) - os.remove(teardown_info_path) - - return suite_info_path - - -def convert_content_to_info(from_raw_path): - """Read raw output, perform filtering, add derivatves, write info output. - - Directory path is created if missing. - - When processing teardown, create also suite output using setup info. - - :param from_raw_path: Local filesystem path to read raw JSON data from. - :type from_raw_path: str - :returns: Local filesystem path to written info JSON file. - :rtype: str - :raises RuntimeError: If path or content do not match expectations. - """ - to_info_path = _raw_to_info_path(from_raw_path) - with open(from_raw_path, u"rt", encoding="utf-8") as file_in: - data = json.load(file_in) - - data = _convert_to_info_in_memory(data) - - with open(to_info_path, u"wt", encoding="utf-8") as file_out: - json.dump(data, file_out, indent=1) - if to_info_path.endswith(u"/teardown.info.json"): - to_info_path = _merge_into_suite_info_file(to_info_path) - # TODO: Return both paths for validation? - - return to_info_path diff --git a/resources/libraries/python/model/util.py b/resources/libraries/python/model/util.py index 879f1f28b1..ff5042fa6c 100644 --- a/resources/libraries/python/model/util.py +++ b/resources/libraries/python/model/util.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2022 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -52,7 +52,7 @@ def descend(parent_node, key, default_factory=None): def get_export_data(): - """Return raw_data member of export_json library instance. + """Return data member of ExportJson library instance. This assumes the data has been initialized already. Return None if Robot is not running. @@ -62,8 +62,8 @@ def get_export_data(): :raises AttributeError: If library is not imported yet. """ instance = BuiltIn().get_library_instance( - u"resources.libraries.python.model.export_json" + u"resources.libraries.python.model.ExportJson" ) if instance is None: return None - return instance.raw_data + return instance.data diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py index e47272f4db..437b1ad3e6 100644 --- a/resources/libraries/python/ssh.py +++ b/resources/libraries/python/ssh.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2022 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -25,9 +25,6 @@ from robot.api import logger from scp import SCPClient, SCPException from resources.libraries.python.OptionString import OptionString -from resources.libraries.python.model.ExportLog import ( - export_ssh_command, export_ssh_result, export_ssh_timeout -) __all__ = [ u"exec_cmd", u"exec_cmd_no_error", u"SSH", u"SSHTimeout", u"scp_node" @@ -145,7 +142,7 @@ class SSH: f"Reconnecting peer done: {node[u'host']}, {node[u'port']}" ) - def exec_command(self, cmd, timeout=10, log_stdout_err=True, export=True): + def exec_command(self, cmd, timeout=10, log_stdout_err=True): """Execute SSH command on a new channel on the connected Node. :param cmd: Command to run on the Node. @@ -154,12 +151,10 @@ class SSH: :param log_stdout_err: If True, stdout and stderr are logged. stdout and stderr are logged also if the return code is not zero independently of the value of log_stdout_err. - :param export: If false, do not attempt JSON export. Needed for calls outside Robot (e.g. from reservation script). :type cmd: str or OptionString :type timeout: int :type log_stdout_err: bool - :type export: bool :returns: return_code, stdout, stderr :rtype: tuple(int, str, str) :raises SSHTimeout: If command is not finished in timeout time. @@ -180,8 +175,6 @@ class SSH: logger.trace(f"exec_command on {peer} with timeout {timeout}: {cmd}") - if export: - export_ssh_command(self._node[u"host"], self._node[u"port"], cmd) start = monotonic() chan.exec_command(cmd) while not chan.exit_status_ready() and timeout is not None: @@ -197,14 +190,6 @@ class SSH: duration = monotonic() - start if duration > timeout: - if export: - export_ssh_timeout( - host=self._node[u"host"], - port=self._node[u"port"], - stdout=stdout, - stderr=stderr, - duration=duration, - ) raise SSHTimeout( f"Timeout exception during execution of command: {cmd}\n" f"Current contents of stdout buffer: " @@ -237,33 +222,21 @@ class SSH: logger.trace( f"return STDERR {stderr}" ) - if export: - export_ssh_result( - host=self._node[u"host"], - port=self._node[u"port"], - code=return_code, - stdout=stdout, - stderr=stderr, - duration=duration, - ) return return_code, stdout, stderr def exec_command_sudo( - self, cmd, cmd_input=None, timeout=30, log_stdout_err=True, - export=True): + self, cmd, cmd_input=None, timeout=30, log_stdout_err=True): """Execute SSH command with sudo on a new channel on the connected Node. :param cmd: Command to be executed. :param cmd_input: Input redirected to the command. :param timeout: Timeout. :param log_stdout_err: If True, stdout and stderr are logged. - :param export: If false, do not attempt JSON export. Needed for calls outside Robot (e.g. from reservation script). :type cmd: str :type cmd_input: str :type timeout: int :type log_stdout_err: bool - :type export: bool :returns: return_code, stdout, stderr :rtype: tuple(int, str, str) @@ -284,7 +257,7 @@ class SSH: else: command = f"sudo -E -S {cmd} <<< \"{cmd_input}\"" return self.exec_command( - command, timeout, log_stdout_err=log_stdout_err, export=export + command, timeout, log_stdout_err=log_stdout_err ) def exec_command_lxc( @@ -442,7 +415,7 @@ class SSH: def exec_cmd( node, cmd, timeout=600, sudo=False, disconnect=False, - log_stdout_err=True, export=True + log_stdout_err=True ): """Convenience function to ssh/exec/return rc, out & err. @@ -456,7 +429,6 @@ def exec_cmd( :param log_stdout_err: If True, stdout and stderr are logged. stdout and stderr are logged also if the return code is not zero independently of the value of log_stdout_err. - :param export: If false, do not attempt JSON export. Needed for calls outside Robot (e.g. from reservation script). :type node: dict :type cmd: str or OptionString @@ -464,7 +436,6 @@ def exec_cmd( :type sudo: bool :type disconnect: bool :type log_stdout_err: bool - :type export: bool :returns: RC, Stdout, Stderr. :rtype: Tuple[int, str, str] """ @@ -486,13 +457,11 @@ def exec_cmd( try: if not sudo: ret_code, stdout, stderr = ssh.exec_command( - cmd, timeout=timeout, log_stdout_err=log_stdout_err, - export=export + cmd, timeout=timeout, log_stdout_err=log_stdout_err ) else: ret_code, stdout, stderr = ssh.exec_command_sudo( - cmd, timeout=timeout, log_stdout_err=log_stdout_err, - export=export + cmd, timeout=timeout, log_stdout_err=log_stdout_err ) except SSHException as err: logger.error(repr(err)) @@ -506,7 +475,7 @@ def exec_cmd( def exec_cmd_no_error( node, cmd, timeout=600, sudo=False, message=None, disconnect=False, - retries=0, include_reason=False, log_stdout_err=True, export=True + retries=0, include_reason=False, log_stdout_err=True ): """Convenience function to ssh/exec/return out & err. @@ -526,7 +495,6 @@ def exec_cmd_no_error( :param log_stdout_err: If True, stdout and stderr are logged. stdout and stderr are logged also if the return code is not zero independently of the value of log_stdout_err. - :param export: If false, do not attempt JSON export. Needed for calls outside Robot thread (e.g. parallel framework setup). :type node: dict :type cmd: str or OptionString @@ -537,7 +505,6 @@ def exec_cmd_no_error( :type retries: int :type include_reason: bool :type log_stdout_err: bool - :type export: bool :returns: Stdout, Stderr. :rtype: tuple(str, str) :raises RuntimeError: If bash return code is not 0. @@ -545,7 +512,7 @@ def exec_cmd_no_error( for _ in range(retries + 1): ret_code, stdout, stderr = exec_cmd( node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect, - log_stdout_err=log_stdout_err, export=export + log_stdout_err=log_stdout_err ) if ret_code == 0: break diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot index 08646d9054..c28fc4f820 100644 --- a/resources/libraries/robot/shared/default.robot +++ b/resources/libraries/robot/shared/default.robot @@ -31,7 +31,7 @@ | Library | resources.libraries.python.IPUtil | Library | resources.libraries.python.IPv6Util | Library | resources.libraries.python.IrqUtil -| Library | resources.libraries.python.model.export_json +| Library | resources.libraries.python.model.ExportJson | Library | resources.libraries.python.NodePath | Library | resources.libraries.python.Namespaces | Library | resources.libraries.python.PapiHistory diff --git a/resources/tools/scripts/topo_reservation.py b/resources/tools/scripts/topo_reservation.py index 0016ebc0cc..f2d18bcafd 100755 --- a/resources/tools/scripts/topo_reservation.py +++ b/resources/tools/scripts/topo_reservation.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2022 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -24,30 +24,13 @@ import argparse import sys import yaml -from resources.libraries.python.ssh import exec_cmd as _exec_cmd +from resources.libraries.python.ssh import exec_cmd RESERVATION_DIR = u"/tmp/reservation_dir" RESERVATION_NODE = u"TG" -def exec_cmd(node, cmd): - """A wrapper around ssh.exec_cmd with disabled JSON export. - - Using this, maintainers can use "exec_cmd" without worrying - about interaction with json export. - - TODO: Instead this, divide ssh module into reusable and robot-bound parts. - - :param node: Node object as parsed from topology file to execute cmd on. - :param cmd: Command to execute. - :type node: dict - :type cmd: str - :returns: RC, Stdout, Stderr. - :rtype: Tuple[int, str, str] - """ - return _exec_cmd(node, cmd, export=False) - def diag_cmd(node, cmd): """Execute cmd, print cmd and stdout, ignore stderr and rc; return None. diff --git a/tests/__init__.robot b/tests/__init__.robot index 46f3125845..b610a511a2 100644 --- a/tests/__init__.robot +++ b/tests/__init__.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2022 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -12,7 +12,7 @@ # limitations under the License. *** Settings *** -| Library | resources.libraries.python.model.export_json +| Library | resources.libraries.python.model.ExportJson | | Suite Setup | Global Suite Setup | Suite Teardown | Global Suite Teardown |