From 01d8f262afc567c3d49a23c3cb2cdeaced8a6887 Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Wed, 15 Dec 2021 17:14:36 +0100 Subject: UTI: Export results + Model version 1.0.0. - Only some result types are exported. + MRR, NDRPDR and SOAK. - Other result types to be added later. + In contrast, all test types are detected. + Convert custom classes to JSON-serializable equivalents. + Sort dict keys before converting to JSON. + Override the order for some known keys. + Export sets as sorted arrays. + Convert to info content from serialized raw content. + Also export outputs for suite setups and teardowns. + Info files for setup/teardown exist only temporarily. + The data is merged into suite.info.json file. + This simplifies presentation of total suite duration. + Define model via JSON schema: - Just test case, suite setup/teardown/suite to be added later. - Just info, raw to be added later. + Proper descriptions. + Json is generated from yaml. + This is a convenience for maintainers. + The officially used schema is the .json one. + TODOs written into a separate .txt file. + Validate exported instance against the schema. + Include format checking. + Update CSIT requirements for validation dependencies. + This needs python-dateutil==2.8.2, only a patch bump. + Compute bandwidth also for soak tests. + This unifies with NDRPDR to simplify schema definition. - PAL may need an update for parsing soak test message. + Include SSH log items, raw output only. + Generate all outputs in a single filesystem tree. + Move raw outputs into test_output_raw.tar.xz. + Rename existing tar with suites to generated_robot_files.tar.xz. Change-Id: I69ff7b330ed1a14dc435fd0ef008e753c0d7f78c Signed-off-by: Vratko Polak --- resources/libraries/python/model/export_json.py | 238 ++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 resources/libraries/python/model/export_json.py (limited to 'resources/libraries/python/model/export_json.py') diff --git a/resources/libraries/python/model/export_json.py b/resources/libraries/python/model/export_json.py new file mode 100644 index 0000000000..4f1b86dbf4 --- /dev/null +++ b/resources/libraries/python/model/export_json.py @@ -0,0 +1,238 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module tracking json in-memory data and saving it to files. + +The current implementation tracks data for raw output, +and info output is created from raw output on disk (see raw2info module). +Raw file contains all log items but no derived quantities, +info file contains only important log items but also derived quantities. +The overlap between two files is big. + +Each test case, suite setup (hierarchical) and teardown has its own file pair. + +Validation is performed for output files with available JSON schema. +Validation is performed in data deserialized from disk, +as serialization might have introduced subtle errors. +""" + +import datetime +import os.path + +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn + +from resources.libraries.python.Constants import Constants +from resources.libraries.python.model.ExportResult import ( + export_dut_type_and_version +) +from resources.libraries.python.model.mem2raw import write_raw_output +from resources.libraries.python.model.raw2info import convert_content_to_info +from resources.libraries.python.model.validate import (get_validators, validate) + + +class export_json(): + """Class handling the json data setting and export.""" + + ROBOT_LIBRARY_SCOPE = u"GLOBAL" + + def __init__(self): + """Declare required fields, cache output dir. + + Also memorize schema validator instances. + """ + self.output_dir = BuiltIn().get_variable_value(u"\\${OUTPUT_DIR}", ".") + self.raw_file_path = None + self.raw_data = None + self.validators = get_validators() + + def export_pending_data(self): + """Write the accumulated data to disk. + + Create missing directories. + Reset both file path and data to avoid writing multiple times. + + Functions which finalize content for given file are calling this, + so make sure each test and non-empty suite setup or teardown + is calling this as their last keyword. + + If no file path is set, do not write anything, + as that is the failsafe behavior when caller from unexpected place. + Aso do not write anything when EXPORT_JSON constant is false. + + Regardless of whether data was written, it is cleared. + """ + if not Constants.EXPORT_JSON or not self.raw_file_path: + self.raw_data = None + self.raw_file_path = None + return + write_raw_output(self.raw_file_path, self.raw_data) + # Raw data is going to be cleared (as a sign that raw export succeeded), + # so this is the last chance to detect if it was for a test case. + is_testcase = u"result" in self.raw_data + self.raw_data = None + # Validation for raw output goes here when ready. + info_file_path = convert_content_to_info(self.raw_file_path) + self.raw_file_path = None + # If "result" is missing from info content, + # it could be a bug in conversion from raw test case content, + # so instead of that we use the flag detected earlier. + if is_testcase: + validate(info_file_path, self.validators[u"tc_info"]) + + def warn_on_bad_export(self): + """If bad state is detected, log a warning and clean up state.""" + if self.raw_file_path is not None or self.raw_data is not None: + logger.warn( + f"Previous export not clean, path {self.raw_file_path}\n" + f"data: {self.raw_data}" + ) + self.raw_data = None + self.raw_file_path = None + + def start_suite_setup_export(self): + """Set new file path, initialize data for the suite setup. + + This has to be called explicitly at start of suite setup, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + u"%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(u" ", u"_") + suite_path_part = os.path.join(*suite_id.split(u".")) + output_dir = self.output_dir + self.raw_file_path = os.path.join( + output_dir, suite_path_part, u"setup.raw.json" + ) + self.raw_data = dict() + self.raw_data[u"version"] = Constants.MODEL_VERSION + self.raw_data[u"start_time"] = start_time + self.raw_data[u"suite_name"] = suite_name + self.raw_data[u"suite_documentation"] = BuiltIn().get_variable_value( + u"\\${SUITE_DOCUMENTATION}" + ) + # "end_time" and "duration" is added on flush. + self.raw_data[u"hosts"] = set() + self.raw_data[u"log"] = list() + + def start_test_export(self): + """Set new file path, initialize data to minimal tree for the test case. + + It is assumed Robot variables DUT_TYPE and DUT_VERSION + are already set (in suite setup) to correct values. + + This function has to be called explicitly at the start of test setup, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite and test. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + u"%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(u" ", u"_") + suite_path_part = os.path.join(*suite_id.split(u".")) + test_name = BuiltIn().get_variable_value(u"\\${TEST_NAME}") + self.raw_file_path = os.path.join( + self.output_dir, suite_path_part, + test_name.lower().replace(u" ", u"_") + u".raw.json" + ) + self.raw_data = dict() + self.raw_data[u"version"] = Constants.MODEL_VERSION + self.raw_data[u"start_time"] = start_time + self.raw_data[u"suite_name"] = suite_name + self.raw_data[u"test_name"] = test_name + test_doc = BuiltIn().get_variable_value(u"\\${TEST_DOCUMENTATION}", u"") + self.raw_data[u"test_documentation"] = test_doc + # "test_type" is added when converting to info. + # "tags" is detected and added on flush. + # "end_time" and "duration" is added on flush. + # Robot status and message are added on flush. + self.raw_data[u"result"] = dict(type=u"unknown") + self.raw_data[u"hosts"] = set() + self.raw_data[u"log"] = list() + export_dut_type_and_version() + + def start_suite_teardown_export(self): + """Set new file path, initialize data for the suite teardown. + + This has to be called explicitly at start of suite teardown, + otherwise Robot likes to postpone initialization + until first call by a data-adding keyword. + + File path is set based on suite. + """ + self.warn_on_bad_export() + start_time = datetime.datetime.utcnow().strftime( + u"%Y-%m-%dT%H:%M:%S.%fZ" + ) + suite_name = BuiltIn().get_variable_value(u"\\${SUITE_NAME}") + suite_id = suite_name.lower().replace(u" ", u"_") + suite_path_part = os.path.join(*suite_id.split(u".")) + self.raw_file_path = os.path.join( + self.output_dir, suite_path_part, u"teardown.raw.json" + ) + self.raw_data = dict() + self.raw_data[u"version"] = Constants.MODEL_VERSION + self.raw_data[u"start_time"] = start_time + self.raw_data[u"suite_name"] = suite_name + # "end_time" and "duration" is added on flush. + self.raw_data[u"hosts"] = set() + self.raw_data[u"log"] = list() + + def finalize_suite_setup_export(self): + """Add the missing fields to data. Do not write yet. + + Should be run at the end of suite setup. + The write is done at next start (or at the end of global teardown). + """ + end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") + self.raw_data[u"end_time"] = end_time + self.export_pending_data() + + def finalize_test_export(self): + """Add the missing fields to data. Do not write yet. + + Should be at the end of test teardown, as the implementation + reads various Robot variables, some of them only available at teardown. + + The write is done at next start (or at the end of global teardown). + """ + end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") + message = BuiltIn().get_variable_value(u"\\${TEST_MESSAGE}") + status = BuiltIn().get_variable_value(u"\\${TEST_STATUS}") + test_tags = BuiltIn().get_variable_value(u"\\${TEST_TAGS}") + self.raw_data[u"end_time"] = end_time + self.raw_data[u"tags"] = list(test_tags) + self.raw_data[u"status"] = status + self.raw_data[u"message"] = message + self.export_pending_data() + + def finalize_suite_teardown_export(self): + """Add the missing fields to data. Do not write yet. + + Should be run at the end of suite teardown + (but before the explicit write in the global suite teardown). + The write is done at next start (or explicitly for global teardown). + """ + end_time = datetime.datetime.utcnow().strftime(u"%Y-%m-%dT%H:%M:%S.%fZ") + self.raw_data[u"end_time"] = end_time + self.export_pending_data() -- cgit 1.2.3-korg