diff options
Diffstat (limited to 'resources/tools/presentation/convert_xml_json.py')
| -rw-r--r-- | resources/tools/presentation/convert_xml_json.py | 475 |
1 files changed, 475 insertions, 0 deletions
diff --git a/resources/tools/presentation/convert_xml_json.py b/resources/tools/presentation/convert_xml_json.py new file mode 100644 index 0000000000..e9ccca0b63 --- /dev/null +++ b/resources/tools/presentation/convert_xml_json.py @@ -0,0 +1,475 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Convert output_info.xml files into JSON structures. + +Version: 0.1.0 +Date: 8th June 2021 + +The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992 +""" + +import os +import re +import json +import logging +import gzip + +from os.path import join +from shutil import rmtree +from copy import deepcopy + +from pal_utils import get_files + + +class JSONData: + """A Class storing and manipulating data from tests. + """ + + def __init__(self, template=None): + """Initialization. + + :param template: JSON formatted template used to store data. It can + include default values. + :type template: dict + """ + + self._template = deepcopy(template) + self._data = self._template if self._template else dict() + + def __str__(self): + """Return a string with human readable data. + + :returns: Readable description. + :rtype: str + """ + return str(self._data) + + def __repr__(self): + """Return a string executable as Python constructor call. + + :returns: Executable constructor call. + :rtype: str + """ + return f"JSONData(template={self._template!r})" + + @property + def data(self): + """Getter + + :return: Data stored in the object. + :rtype: dict + """ + return self._data + + def add_element(self, value, path_to_value): + """Add an element to the json structure. + + :param value: Element value. + :param path_to_value: List of tuples where the first item is the element + on the path and the second one is its type. + :type value: dict, list, str, int, float, bool + :type path_to_value: list + :raises: IndexError if the path is empty. + :raises: TypeError if the val is of not supported type. + """ + + def _add_element(val, path, structure): + """Add an element to the given path. + + :param val: Element value. + :param path: List of tuples where the first item is the element + on the path and the second one is its type. + :param structure: The structure where the element is added. + :type val: dict, list, str, int, float, bool + :type path: list + :type structure: dict + :raises TypeError if there is a wrong type in the path. + """ + if len(path) == 1: + if isinstance(structure, dict): + if path[0][1] is dict: + if path[0][0] not in structure: + structure[path[0][0]] = dict() + structure[path[0][0]].update(val) + elif path[0][1] is list: + if path[0][0] not in structure: + structure[path[0][0]] = list() + if isinstance(val, list): + structure[path[0][0]].extend(val) + else: + structure[path[0][0]].append(val) + else: + structure[path[0][0]] = val + elif isinstance(structure, list): + if path[0][0] == -1 or path[0][0] >= len(structure): + if isinstance(val, list): + structure.extend(val) + else: + structure.append(val) + else: + structure[path[0][0]] = val + return + + if isinstance(structure, dict): + if path[0][1] is dict: + if path[0][0] not in structure: + structure[path[0][0]] = dict() + elif path[0][1] is list: + if path[0][0] not in structure: + structure[path[0][0]] = list() + elif isinstance(structure, list): + if path[0][0] == -1 or path[0][0] >= len(structure): + if path[0][1] is list: + structure.append(list()) + elif path[0][1] is dict: + structure.append(dict()) + else: + structure.append(0) + path[0][0] = len(structure) - 1 + else: + raise TypeError( + u"Only the last item in the path can be different type " + u"then list or dictionary." + ) + _add_element(val, path[1:], structure[path[0][0]]) + + if not isinstance(value, (dict, list, str, int, float, bool)): + raise TypeError( + u"The value must be one of these types: dict, list, str, int, " + u"float, bool.\n" + f"Value: {value}\n" + f"Path: {path_to_value}" + ) + _add_element(deepcopy(value), path_to_value, self._data) + + def get_element(self, path): + """Get the element specified by the path. + + :param path: List of keys and indices to the requested element or + sub-tree. + :type path: list + :returns: Element specified by the path. + :rtype: any + """ + raise NotImplementedError + + def dump(self, file_out, indent=None): + """Write JSON data to a file. + + :param file_out: Path to the output JSON file. + :param indent: Indentation of items in JSON string. It is directly + passed to json.dump method. + :type file_out: str + :type indent: str + """ + try: + with open(file_out, u"w") as file_handler: + json.dump(self._data, file_handler, indent=indent) + except OSError as err: + logging.warning(f"{repr(err)} Skipping") + + def load(self, file_in): + """Load JSON data from a file. + + :param file_in: Path to the input JSON file. + :type file_in: str + :raises: ValueError if the data being deserialized is not a valid + JSON document. + :raises: IOError if the file is not found or corrupted. + """ + with open(file_in, u"r") as file_handler: + self._data = json.load(file_handler) + + +def _export_test_from_xml_to_json(tid, in_data, out, template, metadata): + """Export data from a test to a json structure. + + :param tid: Test ID. + :param in_data: Test data. + :param out: Path to output json file. + :param template: JSON template with optional default values. + :param metadata: Data which are not stored in XML structure. + :type tid: str + :type in_data: dict + :type out: str + :type template: dict + :type metadata: dict + """ + + p_metadata = [(u"metadata", dict), ] + p_test = [(u"test", dict), ] + p_log = [(u"log", list), (-1, list)] + + data = JSONData(template=template) + + data.add_element({u"suite-id": metadata.pop(u"suite-id", u"")}, p_metadata) + data.add_element( + {u"suite-doc": metadata.pop(u"suite-doc", u"")}, p_metadata + ) + data.add_element({u"testbed": metadata.pop(u"testbed", u"")}, p_metadata) + data.add_element( + {u"sut-version": metadata.pop(u"sut-version", u"")}, p_metadata + ) + + data.add_element({u"test-id": tid}, p_test) + t_type = in_data.get(u"type", u"") + t_type = u"NDRPDR" if t_type == u"CPS" else t_type # It is NDRPDR + data.add_element({u"test-type": t_type}, p_test) + tags = in_data.get(u"tags", list()) + data.add_element({u"tags": tags}, p_test) + data.add_element( + {u"documentation": in_data.get(u"documentation", u"")}, p_test + ) + data.add_element({u"message": in_data.get(u"msg", u"")}, p_test) + execution = { + u"start_time": in_data.get(u"starttime", u""), + u"end_time": in_data.get(u"endtime", u""), + u"status": in_data.get(u"status", u"FAILED"), + } + execution.update(metadata) + data.add_element({u"execution": execution}, p_test) + + log_item = { + u"source": { + u"type": u"node", + u"id": "" + }, + u"msg-type": u"", + u"log-level": u"INFO", + u"timestamp": in_data.get(u"starttime", u""), # replacement + u"msg": u"", + u"data": [] + } + + # Process configuration history: + in_papi = deepcopy(in_data.get(u"conf-history", None)) + if in_papi: + regex_dut = re.compile(r'\*\*DUT(\d):\*\*') + node_id = u"dut1" + for line in in_papi.split(u"\n"): + if not line: + continue + groups = re.search(regex_dut, line) + if groups: + node_id = f"dut{groups.group(1)}" + else: + log_item[u"source"][u"id"] = node_id + log_item[u"msg-type"] = u"papi" + log_item[u"msg"] = line + data.add_element(log_item, p_log) + + # Process show runtime: + in_sh_run = deepcopy(in_data.get(u"show-run", None)) + if in_sh_run: + # Transform to openMetrics format + for key, val in in_sh_run.items(): + log_item[u"source"][u"id"] = key + log_item[u"msg-type"] = u"metric" + log_item[u"msg"] = u"show-runtime" + log_item[u"data"] = list() + for item in val.get(u"runtime", list()): + for metric, m_data in item.items(): + if metric == u"name": + continue + for idx, m_item in enumerate(m_data): + log_item[u"data"].append( + { + u"name": metric, + u"value": m_item, + u"labels": { + u"host": val.get(u"host", u""), + u"socket": val.get(u"socket", u""), + u"graph-node": item.get(u"name", u""), + u"thread-id": str(idx) + } + } + ) + data.add_element(log_item, p_log) + + # Process results: + results = dict() + if t_type == u"DEVICETEST": + pass # Nothing to add. + elif t_type == u"NDRPDR": + results = { + u"throughput": { + u"unit": + u"cps" if u"TCP_CPS" in tags or u"UDP_CPS" in tags + else u"pps", + u"ndr": { + u"value": { + u"lower": in_data.get(u"throughput", dict()). + get(u"NDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"throughput", dict()). + get(u"NDR", dict()).get(u"UPPER", u"NaN") + }, + u"value_gbps": { + u"lower": in_data.get(u"gbps", dict()). + get(u"NDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"gbps", dict()). + get(u"NDR", dict()).get(u"UPPER", u"NaN") + } + }, + u"pdr": { + u"value": { + u"lower": in_data.get(u"throughput", dict()). + get(u"PDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"throughput", dict()). + get(u"PDR", dict()).get(u"UPPER", u"NaN") + }, + u"value_gbps": { + u"lower": in_data.get(u"gbps", dict()). + get(u"PDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"gbps", dict()). + get(u"PDR", dict()).get(u"UPPER", u"NaN") + } + } + }, + u"latency": { + u"forward": { + u"pdr-90": in_data.get(u"latency", dict()). + get(u"PDR90", dict()).get(u"direction1", u"NaN"), + u"pdr-50": in_data.get(u"latency", dict()). + get(u"PDR50", dict()).get(u"direction1", u"NaN"), + u"pdr-10": in_data.get(u"latency", dict()). + get(u"PDR10", dict()).get(u"direction1", u"NaN"), + u"pdr-0": in_data.get(u"latency", dict()). + get(u"LAT0", dict()).get(u"direction1", u"NaN") + }, + u"reverse": { + u"pdr-90": in_data.get(u"latency", dict()). + get(u"PDR90", dict()).get(u"direction2", u"NaN"), + u"pdr-50": in_data.get(u"latency", dict()). + get(u"PDR50", dict()).get(u"direction2", u"NaN"), + u"pdr-10": in_data.get(u"latency", dict()). + get(u"PDR10", dict()).get(u"direction2", u"NaN"), + u"pdr-0": in_data.get(u"latency", dict()). + get(u"LAT0", dict()).get(u"direction2", u"NaN") + } + } + } + elif t_type == "MRR": + results = { + u"unit": u"pps", # Old data use only pps + u"samples": in_data.get(u"result", dict()).get(u"samples", list()), + u"avg": in_data.get(u"result", dict()).get(u"receive-rate", u"NaN"), + u"stdev": in_data.get(u"result", dict()). + get(u"receive-stdev", u"NaN") + } + elif t_type == "SOAK": + results = { + u"critical-rate": { + u"lower": in_data.get(u"throughput", dict()). + get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"throughput", dict()). + get(u"UPPER", u"NaN"), + } + } + elif t_type == "HOSTSTACK": + results = in_data.get(u"result", dict()) + # elif t_type == "TCP": # Not used ??? + # results = in_data.get(u"result", u"NaN") + elif t_type == "RECONF": + results = { + u"loss": in_data.get(u"result", dict()).get(u"loss", u"NaN"), + u"time": in_data.get(u"result", dict()).get(u"time", u"NaN") + } + else: + pass + data.add_element({u"results": results}, p_test) + + data.dump(out, indent=u" ") + + +def convert_xml_to_json(spec, data): + """Convert downloaded XML files into JSON. + + Procedure: + - create one json file for each test, + - gzip all json files one by one, + - delete json files. + + :param spec: Specification read from the specification files. + :param data: Input data parsed from output.xml files. + :type spec: Specification + :type data: InputData + """ + + logging.info(u"Converting downloaded XML files to JSON ...") + + template_name = spec.output.get(u"use-template", None) + structure = spec.output.get(u"structure", u"tree") + if template_name: + with open(template_name, u"r") as file_handler: + template = json.load(file_handler) + else: + template = None + + build_dir = spec.environment[u"paths"][u"DIR[BUILD,JSON]"] + try: + rmtree(build_dir) + except FileNotFoundError: + pass # It does not exist + + os.mkdir(build_dir) + + for job, builds in data.data.items(): + logging.info(f" Processing job {job}") + if structure == "tree": + os.makedirs(join(build_dir, job), exist_ok=True) + for build_nr, build in builds.items(): + logging.info(f" Processing build {build_nr}") + if structure == "tree": + os.makedirs(join(build_dir, job, build_nr), exist_ok=True) + for test_id, test_data in build[u"tests"].items(): + groups = re.search(re.compile(r'-(\d+[tT](\d+[cC]))-'), test_id) + if groups: + test_id = test_id.replace(groups.group(1), groups.group(2)) + logging.info(f" Processing test {test_id}") + if structure == "tree": + dirs = test_id.split(u".")[:-1] + name = test_id.split(u".")[-1] + os.makedirs( + join(build_dir, job, build_nr, *dirs), exist_ok=True + ) + file_name = \ + f"{join(build_dir, job, build_nr, *dirs, name)}.json" + else: + file_name = join( + build_dir, + u'.'.join((job, build_nr, test_id, u'json')) + ) + suite_id = test_id.rsplit(u".", 1)[0].replace(u" ", u"_") + _export_test_from_xml_to_json( + test_id, test_data, file_name, template, + { + u"ci": u"jenkins.fd.io", + u"job": job, + u"build": build_nr, + u"suite-id": suite_id, + u"suite-doc": build[u"suites"].get(suite_id, dict()). + get(u"doc", u""), + u"testbed": build[u"metadata"].get(u"testbed", u""), + u"sut-version": build[u"metadata"].get(u"version", u"") + } + ) + + # gzip the json files: + for file in get_files(build_dir, u"json"): + with open(file, u"rb") as src: + with gzip.open(f"{file}.gz", u"wb") as dst: + dst.writelines(src) + os.remove(file) + + logging.info(u"Done.") |