diff options
Diffstat (limited to 'resources/tools')
16 files changed, 798 insertions, 194 deletions
diff --git a/resources/tools/presentation/convert_xml_json.py b/resources/tools/presentation/convert_xml_json.py new file mode 100644 index 0000000000..e9ccca0b63 --- /dev/null +++ b/resources/tools/presentation/convert_xml_json.py @@ -0,0 +1,475 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Convert output_info.xml files into JSON structures. + +Version: 0.1.0 +Date: 8th June 2021 + +The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992 +""" + +import os +import re +import json +import logging +import gzip + +from os.path import join +from shutil import rmtree +from copy import deepcopy + +from pal_utils import get_files + + +class JSONData: + """A Class storing and manipulating data from tests. + """ + + def __init__(self, template=None): + """Initialization. + + :param template: JSON formatted template used to store data. It can + include default values. + :type template: dict + """ + + self._template = deepcopy(template) + self._data = self._template if self._template else dict() + + def __str__(self): + """Return a string with human readable data. + + :returns: Readable description. + :rtype: str + """ + return str(self._data) + + def __repr__(self): + """Return a string executable as Python constructor call. + + :returns: Executable constructor call. + :rtype: str + """ + return f"JSONData(template={self._template!r})" + + @property + def data(self): + """Getter + + :return: Data stored in the object. + :rtype: dict + """ + return self._data + + def add_element(self, value, path_to_value): + """Add an element to the json structure. + + :param value: Element value. + :param path_to_value: List of tuples where the first item is the element + on the path and the second one is its type. + :type value: dict, list, str, int, float, bool + :type path_to_value: list + :raises: IndexError if the path is empty. + :raises: TypeError if the val is of not supported type. + """ + + def _add_element(val, path, structure): + """Add an element to the given path. + + :param val: Element value. + :param path: List of tuples where the first item is the element + on the path and the second one is its type. + :param structure: The structure where the element is added. + :type val: dict, list, str, int, float, bool + :type path: list + :type structure: dict + :raises TypeError if there is a wrong type in the path. + """ + if len(path) == 1: + if isinstance(structure, dict): + if path[0][1] is dict: + if path[0][0] not in structure: + structure[path[0][0]] = dict() + structure[path[0][0]].update(val) + elif path[0][1] is list: + if path[0][0] not in structure: + structure[path[0][0]] = list() + if isinstance(val, list): + structure[path[0][0]].extend(val) + else: + structure[path[0][0]].append(val) + else: + structure[path[0][0]] = val + elif isinstance(structure, list): + if path[0][0] == -1 or path[0][0] >= len(structure): + if isinstance(val, list): + structure.extend(val) + else: + structure.append(val) + else: + structure[path[0][0]] = val + return + + if isinstance(structure, dict): + if path[0][1] is dict: + if path[0][0] not in structure: + structure[path[0][0]] = dict() + elif path[0][1] is list: + if path[0][0] not in structure: + structure[path[0][0]] = list() + elif isinstance(structure, list): + if path[0][0] == -1 or path[0][0] >= len(structure): + if path[0][1] is list: + structure.append(list()) + elif path[0][1] is dict: + structure.append(dict()) + else: + structure.append(0) + path[0][0] = len(structure) - 1 + else: + raise TypeError( + u"Only the last item in the path can be different type " + u"then list or dictionary." + ) + _add_element(val, path[1:], structure[path[0][0]]) + + if not isinstance(value, (dict, list, str, int, float, bool)): + raise TypeError( + u"The value must be one of these types: dict, list, str, int, " + u"float, bool.\n" + f"Value: {value}\n" + f"Path: {path_to_value}" + ) + _add_element(deepcopy(value), path_to_value, self._data) + + def get_element(self, path): + """Get the element specified by the path. + + :param path: List of keys and indices to the requested element or + sub-tree. + :type path: list + :returns: Element specified by the path. + :rtype: any + """ + raise NotImplementedError + + def dump(self, file_out, indent=None): + """Write JSON data to a file. + + :param file_out: Path to the output JSON file. + :param indent: Indentation of items in JSON string. It is directly + passed to json.dump method. + :type file_out: str + :type indent: str + """ + try: + with open(file_out, u"w") as file_handler: + json.dump(self._data, file_handler, indent=indent) + except OSError as err: + logging.warning(f"{repr(err)} Skipping") + + def load(self, file_in): + """Load JSON data from a file. + + :param file_in: Path to the input JSON file. + :type file_in: str + :raises: ValueError if the data being deserialized is not a valid + JSON document. + :raises: IOError if the file is not found or corrupted. + """ + with open(file_in, u"r") as file_handler: + self._data = json.load(file_handler) + + +def _export_test_from_xml_to_json(tid, in_data, out, template, metadata): + """Export data from a test to a json structure. + + :param tid: Test ID. + :param in_data: Test data. + :param out: Path to output json file. + :param template: JSON template with optional default values. + :param metadata: Data which are not stored in XML structure. + :type tid: str + :type in_data: dict + :type out: str + :type template: dict + :type metadata: dict + """ + + p_metadata = [(u"metadata", dict), ] + p_test = [(u"test", dict), ] + p_log = [(u"log", list), (-1, list)] + + data = JSONData(template=template) + + data.add_element({u"suite-id": metadata.pop(u"suite-id", u"")}, p_metadata) + data.add_element( + {u"suite-doc": metadata.pop(u"suite-doc", u"")}, p_metadata + ) + data.add_element({u"testbed": metadata.pop(u"testbed", u"")}, p_metadata) + data.add_element( + {u"sut-version": metadata.pop(u"sut-version", u"")}, p_metadata + ) + + data.add_element({u"test-id": tid}, p_test) + t_type = in_data.get(u"type", u"") + t_type = u"NDRPDR" if t_type == u"CPS" else t_type # It is NDRPDR + data.add_element({u"test-type": t_type}, p_test) + tags = in_data.get(u"tags", list()) + data.add_element({u"tags": tags}, p_test) + data.add_element( + {u"documentation": in_data.get(u"documentation", u"")}, p_test + ) + data.add_element({u"message": in_data.get(u"msg", u"")}, p_test) + execution = { + u"start_time": in_data.get(u"starttime", u""), + u"end_time": in_data.get(u"endtime", u""), + u"status": in_data.get(u"status", u"FAILED"), + } + execution.update(metadata) + data.add_element({u"execution": execution}, p_test) + + log_item = { + u"source": { + u"type": u"node", + u"id": "" + }, + u"msg-type": u"", + u"log-level": u"INFO", + u"timestamp": in_data.get(u"starttime", u""), # replacement + u"msg": u"", + u"data": [] + } + + # Process configuration history: + in_papi = deepcopy(in_data.get(u"conf-history", None)) + if in_papi: + regex_dut = re.compile(r'\*\*DUT(\d):\*\*') + node_id = u"dut1" + for line in in_papi.split(u"\n"): + if not line: + continue + groups = re.search(regex_dut, line) + if groups: + node_id = f"dut{groups.group(1)}" + else: + log_item[u"source"][u"id"] = node_id + log_item[u"msg-type"] = u"papi" + log_item[u"msg"] = line + data.add_element(log_item, p_log) + + # Process show runtime: + in_sh_run = deepcopy(in_data.get(u"show-run", None)) + if in_sh_run: + # Transform to openMetrics format + for key, val in in_sh_run.items(): + log_item[u"source"][u"id"] = key + log_item[u"msg-type"] = u"metric" + log_item[u"msg"] = u"show-runtime" + log_item[u"data"] = list() + for item in val.get(u"runtime", list()): + for metric, m_data in item.items(): + if metric == u"name": + continue + for idx, m_item in enumerate(m_data): + log_item[u"data"].append( + { + u"name": metric, + u"value": m_item, + u"labels": { + u"host": val.get(u"host", u""), + u"socket": val.get(u"socket", u""), + u"graph-node": item.get(u"name", u""), + u"thread-id": str(idx) + } + } + ) + data.add_element(log_item, p_log) + + # Process results: + results = dict() + if t_type == u"DEVICETEST": + pass # Nothing to add. + elif t_type == u"NDRPDR": + results = { + u"throughput": { + u"unit": + u"cps" if u"TCP_CPS" in tags or u"UDP_CPS" in tags + else u"pps", + u"ndr": { + u"value": { + u"lower": in_data.get(u"throughput", dict()). + get(u"NDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"throughput", dict()). + get(u"NDR", dict()).get(u"UPPER", u"NaN") + }, + u"value_gbps": { + u"lower": in_data.get(u"gbps", dict()). + get(u"NDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"gbps", dict()). + get(u"NDR", dict()).get(u"UPPER", u"NaN") + } + }, + u"pdr": { + u"value": { + u"lower": in_data.get(u"throughput", dict()). + get(u"PDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"throughput", dict()). + get(u"PDR", dict()).get(u"UPPER", u"NaN") + }, + u"value_gbps": { + u"lower": in_data.get(u"gbps", dict()). + get(u"PDR", dict()).get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"gbps", dict()). + get(u"PDR", dict()).get(u"UPPER", u"NaN") + } + } + }, + u"latency": { + u"forward": { + u"pdr-90": in_data.get(u"latency", dict()). + get(u"PDR90", dict()).get(u"direction1", u"NaN"), + u"pdr-50": in_data.get(u"latency", dict()). + get(u"PDR50", dict()).get(u"direction1", u"NaN"), + u"pdr-10": in_data.get(u"latency", dict()). + get(u"PDR10", dict()).get(u"direction1", u"NaN"), + u"pdr-0": in_data.get(u"latency", dict()). + get(u"LAT0", dict()).get(u"direction1", u"NaN") + }, + u"reverse": { + u"pdr-90": in_data.get(u"latency", dict()). + get(u"PDR90", dict()).get(u"direction2", u"NaN"), + u"pdr-50": in_data.get(u"latency", dict()). + get(u"PDR50", dict()).get(u"direction2", u"NaN"), + u"pdr-10": in_data.get(u"latency", dict()). + get(u"PDR10", dict()).get(u"direction2", u"NaN"), + u"pdr-0": in_data.get(u"latency", dict()). + get(u"LAT0", dict()).get(u"direction2", u"NaN") + } + } + } + elif t_type == "MRR": + results = { + u"unit": u"pps", # Old data use only pps + u"samples": in_data.get(u"result", dict()).get(u"samples", list()), + u"avg": in_data.get(u"result", dict()).get(u"receive-rate", u"NaN"), + u"stdev": in_data.get(u"result", dict()). + get(u"receive-stdev", u"NaN") + } + elif t_type == "SOAK": + results = { + u"critical-rate": { + u"lower": in_data.get(u"throughput", dict()). + get(u"LOWER", u"NaN"), + u"upper": in_data.get(u"throughput", dict()). + get(u"UPPER", u"NaN"), + } + } + elif t_type == "HOSTSTACK": + results = in_data.get(u"result", dict()) + # elif t_type == "TCP": # Not used ??? + # results = in_data.get(u"result", u"NaN") + elif t_type == "RECONF": + results = { + u"loss": in_data.get(u"result", dict()).get(u"loss", u"NaN"), + u"time": in_data.get(u"result", dict()).get(u"time", u"NaN") + } + else: + pass + data.add_element({u"results": results}, p_test) + + data.dump(out, indent=u" ") + + +def convert_xml_to_json(spec, data): + """Convert downloaded XML files into JSON. + + Procedure: + - create one json file for each test, + - gzip all json files one by one, + - delete json files. + + :param spec: Specification read from the specification files. + :param data: Input data parsed from output.xml files. + :type spec: Specification + :type data: InputData + """ + + logging.info(u"Converting downloaded XML files to JSON ...") + + template_name = spec.output.get(u"use-template", None) + structure = spec.output.get(u"structure", u"tree") + if template_name: + with open(template_name, u"r") as file_handler: + template = json.load(file_handler) + else: + template = None + + build_dir = spec.environment[u"paths"][u"DIR[BUILD,JSON]"] + try: + rmtree(build_dir) + except FileNotFoundError: + pass # It does not exist + + os.mkdir(build_dir) + + for job, builds in data.data.items(): + logging.info(f" Processing job {job}") + if structure == "tree": + os.makedirs(join(build_dir, job), exist_ok=True) + for build_nr, build in builds.items(): + logging.info(f" Processing build {build_nr}") + if structure == "tree": + os.makedirs(join(build_dir, job, build_nr), exist_ok=True) + for test_id, test_data in build[u"tests"].items(): + groups = re.search(re.compile(r'-(\d+[tT](\d+[cC]))-'), test_id) + if groups: + test_id = test_id.replace(groups.group(1), groups.group(2)) + logging.info(f" Processing test {test_id}") + if structure == "tree": + dirs = test_id.split(u".")[:-1] + name = test_id.split(u".")[-1] + os.makedirs( + join(build_dir, job, build_nr, *dirs), exist_ok=True + ) + file_name = \ + f"{join(build_dir, job, build_nr, *dirs, name)}.json" + else: + file_name = join( + build_dir, + u'.'.join((job, build_nr, test_id, u'json')) + ) + suite_id = test_id.rsplit(u".", 1)[0].replace(u" ", u"_") + _export_test_from_xml_to_json( + test_id, test_data, file_name, template, + { + u"ci": u"jenkins.fd.io", + u"job": job, + u"build": build_nr, + u"suite-id": suite_id, + u"suite-doc": build[u"suites"].get(suite_id, dict()). + get(u"doc", u""), + u"testbed": build[u"metadata"].get(u"testbed", u""), + u"sut-version": build[u"metadata"].get(u"version", u"") + } + ) + + # gzip the json files: + for file in get_files(build_dir, u"json"): + with open(file, u"rb") as src: + with gzip.open(f"{file}.gz", u"wb") as dst: + dst.writelines(src) + os.remove(file) + + logging.info(u"Done.") diff --git a/resources/tools/presentation/generator_files.py b/resources/tools/presentation/generator_files.py index 11ed9b0337..aa4392e473 100644 --- a/resources/tools/presentation/generator_files.py +++ b/resources/tools/presentation/generator_files.py @@ -205,7 +205,12 @@ def file_details_split(file_spec, input_data, frmt=u"rst"): chapters[chapter_l1][chapter_l2][nic][u"tables"].append( ( table_lst.pop(idx), - suite[u"doc"].replace(u'|br|', u'\n\n -') + suite[u"doc"].replace(u'"', u"'"). + replace(u'\n', u' '). + replace(u'\r', u''). + replace(u'*[', u'\n\n - *['). + replace(u"*", u"**"). + replace(u'\n\n - *[', u' - *[', 1) ) ) break diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py index 1d6bbaabf5..fb1b4734cf 100644 --- a/resources/tools/presentation/generator_plots.py +++ b/resources/tools/presentation/generator_plots.py @@ -18,16 +18,16 @@ import re import logging +from collections import OrderedDict +from copy import deepcopy +from math import log + import hdrh.histogram import hdrh.codec import pandas as pd import plotly.offline as ploff import plotly.graph_objs as plgo -from collections import OrderedDict -from copy import deepcopy -from math import log - from plotly.exceptions import PlotlyError from pal_utils import mean, stdev @@ -200,7 +200,8 @@ def plot_hdrh_lat_by_percentile(plot, input_data): hovertext.append( f"<b>{desc[graph]}</b><br>" f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>" - f"Percentile: {previous_x:.5f}-{percentile:.5f}%<br>" + f"Percentile: " + f"{previous_x:.5f}-{percentile:.5f}%<br>" f"Latency: {item.value_iterated_to}uSec" ) xaxis.append(percentile) @@ -208,7 +209,8 @@ def plot_hdrh_lat_by_percentile(plot, input_data): hovertext.append( f"<b>{desc[graph]}</b><br>" f"Direction: {(u'W-E', u'E-W')[idx % 2]}<br>" - f"Percentile: {previous_x:.5f}-{percentile:.5f}%<br>" + f"Percentile: " + f"{previous_x:.5f}-{percentile:.5f}%<br>" f"Latency: {item.value_iterated_to}uSec" ) previous_x = percentile @@ -351,7 +353,7 @@ def plot_hdrh_lat_by_percentile_x_log(plot, input_data): decoded = hdrh.histogram.HdrHistogram.decode( test[u"latency"][graph][direction][u"hdrh"] ) - except hdrh.codec.HdrLengthException: + except (hdrh.codec.HdrLengthException, TypeError): logging.warning( f"No data for direction {(u'W-E', u'E-W')[idx % 2]}" ) @@ -855,10 +857,10 @@ def plot_mrr_box_name(plot, input_data): # Add plot traces traces = list() - for idx in range(len(data_x)): + for idx, x_item in enumerate(data_x): traces.append( plgo.Box( - x=[data_x[idx], ] * len(data_y[idx]), + x=[x_item, ] * len(data_y[idx]), y=data_y[idx], name=data_names[idx], hoverinfo=u"y+name" @@ -988,7 +990,7 @@ def plot_tsa_name(plot, input_data): REGEX_NIC, u"", test_name.replace(u'-ndrpdr', u''). - replace(u'2n1l-', u'') + replace(u'2n1l-', u'') ) vals[name] = OrderedDict() y_val_1 = test_vals[u"1"][0] / 1e6 diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index b03261c6d8..bb962890d0 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -323,7 +323,8 @@ def table_merged_details(table, input_data): suite_name = suite[u"name"] table_lst = list() for test in data.keys(): - if data[test][u"parent"] not in suite_name: + if data[test][u"status"] != u"PASS" or \ + data[test][u"parent"] not in suite_name: continue row_lst = list() for column in table[u"columns"]: @@ -351,10 +352,12 @@ def table_merged_details(table, input_data): col_data = col_data.split(u" |br| ", 1)[1] except IndexError: pass + col_data = col_data.replace(u'\n', u' |br| ').\ + replace(u'\r', u'').replace(u'"', u"'") col_data = f" |prein| {col_data} |preout| " elif column[u"data"].split(u" ")[1] in \ (u"conf-history", u"show-run"): - col_data = col_data.replace(u" |br| ", u"", 1) + col_data = col_data.replace(u'\n', u' |br| ') col_data = f" |prein| {col_data[:-5]} |preout| " row_lst.append(f'"{col_data}"') except KeyError: @@ -386,12 +389,7 @@ def _tpc_modify_test_name(test_name, ignore_nic=False): :rtype: str """ test_name_mod = test_name.\ - replace(u"-ndrpdrdisc", u""). \ replace(u"-ndrpdr", u"").\ - replace(u"-pdrdisc", u""). \ - replace(u"-ndrdisc", u"").\ - replace(u"-pdr", u""). \ - replace(u"-ndr", u""). \ replace(u"1t1c", u"1c").\ replace(u"2t1c", u"1c"). \ replace(u"2t2c", u"2c").\ @@ -425,7 +423,7 @@ def _tpc_insert_data(target, src, include_tests): """Insert src data to the target structure. :param target: Target structure where the data is placed. - :param src: Source data to be placed into the target stucture. + :param src: Source data to be placed into the target structure. :param include_tests: Which results will be included (MRR, NDR, PDR). :type target: list :type src: dict @@ -1252,8 +1250,8 @@ def table_perf_trending_dash_html(table, input_data): u"a", attrib=dict( href=f"{lnk_dir}" - f"{_generate_url(table.get(u'testbed', ''), item)}" - f"{lnk_sufix}" + f"{_generate_url(table.get(u'testbed', ''), item)}" + f"{lnk_sufix}" ) ) ref.text = item diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index fc629bc218..5bd6af42d6 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -181,22 +181,6 @@ def _unzip_file(spec, build, pid): return False -def _download_json(source, job, build, w_dir, arch): - """ - - :param source: - :param job: - :param build: - :param w_dir: Path to working directory - :param arch: - :return: - """ - success = False - downloaded_name = u"" - - return success, downloaded_name - - def _download_xml(source, job, build, w_dir, arch): """ @@ -219,10 +203,9 @@ def _download_xml(source, job, build, w_dir, arch): job=job, build=build[u'build'], filename=file_name ) ) - verify = False if u"nginx" in url else True logging.info(f" Trying to download {url}") success, downloaded_name = _download_file( - url, new_name, arch=arch, verify=verify, repeat=3 + url, new_name, arch=arch, verify=(u"nginx" not in url), repeat=3 ) return success, downloaded_name @@ -286,7 +269,6 @@ def download_and_unzip_data_file(spec, job, build, pid): """ download = { - "json": _download_json, "xml": _download_xml, "xml-docs": _download_xml_docs } @@ -302,12 +284,12 @@ def download_and_unzip_data_file(spec, job, build, pid): if not download_type: continue success, downloaded_name = download[download_type]( - source, - job, - build, - spec.environment[u"paths"][u"DIR[WORKING,DATA]"], - arch - ) + source, + job, + build, + spec.environment[u"paths"][u"DIR[WORKING,DATA]"], + arch + ) if success: source[u"successful-downloads"] += 1 build[u"source"] = source[u"type"] diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index e1db03660d..d108d09e84 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -346,8 +346,6 @@ class ExecutionChecker(ResultVisitor): u"timestamp": self._get_timestamp, u"vpp-version": self._get_vpp_version, u"dpdk-version": self._get_dpdk_version, - # TODO: Remove when not needed: - u"teardown-vat-history": self._get_vat_history, u"teardown-papi-history": self._get_papi_history, u"test-show-runtime": self._get_show_run, u"testbed": self._get_testbed @@ -608,32 +606,6 @@ class ExecutionChecker(ResultVisitor): self._data[u"metadata"][u"generated"] = self._timestamp self._msg_type = None - def _get_vat_history(self, msg): - """Called when extraction of VAT command history is required. - - TODO: Remove when not needed. - - :param msg: Message to process. - :type msg: Message - :returns: Nothing. - """ - if msg.message.count(u"VAT command history:"): - self._conf_history_lookup_nr += 1 - if self._conf_history_lookup_nr == 1: - self._data[u"tests"][self._test_id][u"conf-history"] = str() - else: - self._msg_type = None - text = re.sub( - r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} VAT command history:", - u"", - msg.message, - count=1 - ).replace(u'\n', u' |br| ').replace(u'"', u"'") - - self._data[u"tests"][self._test_id][u"conf-history"] += ( - f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}" - ) - def _get_papi_history(self, msg): """Called when extraction of PAPI command history is required. @@ -652,9 +624,9 @@ class ExecutionChecker(ResultVisitor): u"", msg.message, count=1 - ).replace(u'\n', u' |br| ').replace(u'"', u"'") + ).replace(u'"', u"'") self._data[u"tests"][self._test_id][u"conf-history"] += ( - f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}" + f"**DUT{str(self._conf_history_lookup_nr)}:** {text}" ) def _get_show_run(self, msg): @@ -697,12 +669,13 @@ class ExecutionChecker(ResultVisitor): except (IndexError, KeyError): return - dut = u"DUT{nr}".format( + dut = u"dut{nr}".format( nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1) oper = { u"host": host, u"socket": sock, + u"runtime": runtime, u"threads": OrderedDict({idx: list() for idx in range(threads_nr)}) } @@ -917,38 +890,6 @@ class ExecutionChecker(ResultVisitor): except (IndexError, ValueError): pass - # TODO: Remove when not needed - latency[u"NDR10"] = { - u"direction1": copy.copy(latency_default), - u"direction2": copy.copy(latency_default) - } - latency[u"NDR50"] = { - u"direction1": copy.copy(latency_default), - u"direction2": copy.copy(latency_default) - } - latency[u"NDR90"] = { - u"direction1": copy.copy(latency_default), - u"direction2": copy.copy(latency_default) - } - try: - latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5)) - latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6)) - latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7)) - latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8)) - latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9)) - latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10)) - latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11)) - latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12)) - latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13)) - latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14)) - latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15)) - latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16)) - latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17)) - latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18)) - return latency, u"PASS" - except (IndexError, ValueError): - pass - return latency, u"FAIL" @staticmethod @@ -1010,19 +951,11 @@ class ExecutionChecker(ResultVisitor): except AttributeError: return - doc_str = suite.doc.\ - replace(u'"', u"'").\ - replace(u'\n', u' ').\ - replace(u'\r', u'').\ - replace(u'*[', u' |br| *[').\ - replace(u"*", u"**").\ - replace(u' |br| *[', u'*[', 1) - self._data[u"suites"][suite.longname.lower(). replace(u'"', u"'"). replace(u" ", u"_")] = { u"name": suite.name.lower(), - u"doc": doc_str, + u"doc": suite.doc, u"parent": parent_name, u"level": len(suite.longname.split(u".")) } @@ -1080,49 +1013,36 @@ class ExecutionChecker(ResultVisitor): name = test.name.lower() # Remove TC number from the TC long name (backward compatibility): - self._test_id = re.sub( - self.REGEX_TC_NUMBER, u"", longname.replace(u"snat", u"nat") - ) + self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname) # Remove TC number from the TC name (not needed): - test_result[u"name"] = re.sub( - self.REGEX_TC_NUMBER, "", name.replace(u"snat", u"nat") - ) + test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name) - test_result[u"parent"] = test.parent.name.lower().\ - replace(u"snat", u"nat") + test_result[u"parent"] = test.parent.name.lower() test_result[u"tags"] = tags - test_result["doc"] = test.doc.\ - replace(u'"', u"'").\ - replace(u'\n', u' ').\ - replace(u'\r', u'').\ - replace(u'[', u' |br| [').\ - replace(u' |br| [', u'[', 1) - test_result[u"type"] = u"FUNC" + test_result["doc"] = test.doc + test_result[u"type"] = u"" test_result[u"status"] = test.status + test_result[u"starttime"] = test.starttime + test_result[u"endtime"] = test.endtime if test.status == u"PASS": if u"NDRPDR" in tags: if u"TCP_PPS" in tags or u"UDP_PPS" in tags: test_result[u"msg"] = self._get_data_from_pps_test_msg( - test.message).replace(u'\n', u' |br| '). \ - replace(u'\r', u'').replace(u'"', u"'") + test.message) elif u"TCP_CPS" in tags or u"UDP_CPS" in tags: test_result[u"msg"] = self._get_data_from_cps_test_msg( - test.message).replace(u'\n', u' |br| '). \ - replace(u'\r', u'').replace(u'"', u"'") + test.message) else: test_result[u"msg"] = self._get_data_from_perf_test_msg( - test.message).replace(u'\n', u' |br| ').\ - replace(u'\r', u'').replace(u'"', u"'") + test.message) elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags: test_result[u"msg"] = self._get_data_from_mrr_test_msg( - test.message).replace(u'\n', u' |br| ').\ - replace(u'\r', u'').replace(u'"', u"'") + test.message) else: - test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\ - replace(u'\r', u'').replace(u'"', u"'") + test_result[u"msg"] = test.message else: - test_result[u"msg"] = u"Test Failed." + test_result[u"msg"] = test.message if u"PERFTEST" in tags: # Replace info about cores (e.g. -1c-) with the info about threads @@ -1157,26 +1077,26 @@ class ExecutionChecker(ResultVisitor): ) return - if test.status == u"PASS": - if u"DEVICETEST" in tags: - test_result[u"type"] = u"DEVICETEST" - elif u"NDRPDR" in tags: - if u"TCP_CPS" in tags or u"UDP_CPS" in tags: - test_result[u"type"] = u"CPS" - else: - test_result[u"type"] = u"NDRPDR" + if u"DEVICETEST" in tags: + test_result[u"type"] = u"DEVICETEST" + elif u"NDRPDR" in tags: + if u"TCP_CPS" in tags or u"UDP_CPS" in tags: + test_result[u"type"] = u"CPS" + else: + test_result[u"type"] = u"NDRPDR" + if test.status == u"PASS": test_result[u"throughput"], test_result[u"status"] = \ self._get_ndrpdr_throughput(test.message) test_result[u"gbps"], test_result[u"status"] = \ self._get_ndrpdr_throughput_gbps(test.message) test_result[u"latency"], test_result[u"status"] = \ self._get_ndrpdr_latency(test.message) - elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags: - if u"MRR" in tags: - test_result[u"type"] = u"MRR" - else: - test_result[u"type"] = u"BMRR" - + elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags: + if u"MRR" in tags: + test_result[u"type"] = u"MRR" + else: + test_result[u"type"] = u"BMRR" + if test.status == u"PASS": test_result[u"result"] = dict() groups = re.search(self.REGEX_BMRR, test.message) if groups is not None: @@ -1194,20 +1114,24 @@ class ExecutionChecker(ResultVisitor): groups = re.search(self.REGEX_MRR, test.message) test_result[u"result"][u"receive-rate"] = \ float(groups.group(3)) / float(groups.group(1)) - elif u"SOAK" in tags: - test_result[u"type"] = u"SOAK" + elif u"SOAK" in tags: + test_result[u"type"] = u"SOAK" + if test.status == u"PASS": test_result[u"throughput"], test_result[u"status"] = \ self._get_plr_throughput(test.message) - elif u"HOSTSTACK" in tags: - test_result[u"type"] = u"HOSTSTACK" + elif u"HOSTSTACK" in tags: + test_result[u"type"] = u"HOSTSTACK" + if test.status == u"PASS": test_result[u"result"], test_result[u"status"] = \ self._get_hoststack_data(test.message, tags) - elif u"TCP" in tags: - test_result[u"type"] = u"TCP" - groups = re.search(self.REGEX_TCP, test.message) - test_result[u"result"] = int(groups.group(2)) - elif u"RECONF" in tags: - test_result[u"type"] = u"RECONF" + # elif u"TCP" in tags: # This might be not used + # test_result[u"type"] = u"TCP" + # if test.status == u"PASS": + # groups = re.search(self.REGEX_TCP, test.message) + # test_result[u"result"] = int(groups.group(2)) + elif u"RECONF" in tags: + test_result[u"type"] = u"RECONF" + if test.status == u"PASS": test_result[u"result"] = None try: grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message) @@ -1218,10 +1142,8 @@ class ExecutionChecker(ResultVisitor): } except (AttributeError, IndexError, ValueError, TypeError): test_result[u"status"] = u"FAIL" - else: - test_result[u"status"] = u"FAIL" - self._data[u"tests"][self._test_id] = test_result - return + else: + test_result[u"status"] = u"FAIL" self._data[u"tests"][self._test_id] = test_result @@ -1370,13 +1292,7 @@ class ExecutionChecker(ResultVisitor): :type teardown_kw: Keyword :returns: Nothing. """ - - if teardown_kw.name.count(u"Show Vat History On All Duts"): - # TODO: Remove when not needed: - self._conf_history_lookup_nr = 0 - self._msg_type = u"teardown-vat-history" - teardown_kw.messages.visit(self) - elif teardown_kw.name.count(u"Show Papi History On All Duts"): + if teardown_kw.name.count(u"Show Papi History On All Duts"): self._conf_history_lookup_nr = 0 self._msg_type = u"teardown-papi-history" teardown_kw.messages.visit(self) @@ -1876,7 +1792,7 @@ class InputData: if params is None: params = element.get(u"parameters", None) if params: - params.append(u"type") + params.extend((u"type", u"status")) data_to_filter = data if data else element[u"data"] data = pd.Series() diff --git a/resources/tools/presentation/json/template_0.1.0.json b/resources/tools/presentation/json/template_0.1.0.json new file mode 100644 index 0000000000..dd9fed7360 --- /dev/null +++ b/resources/tools/presentation/json/template_0.1.0.json @@ -0,0 +1,25 @@ +{ + "version": "0.1.0", + "test": { + "test-id": "", + "test-type": "", + "tags": [], + "documentation": "", + "message": "", + "execution": { + "ci": "", + "job": "", + "build": "", + "csit-commit": "", + "csit-gerrit-change": "", + "start_time": "", + "end_time": "", + "status": "" + }, + "results": {} + }, + "metadata": {}, + "resource": [], + "network": [], + "log": [] +} diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py index 5bbea297ef..7e2d9a8dbd 100644 --- a/resources/tools/presentation/pal.py +++ b/resources/tools/presentation/pal.py @@ -29,9 +29,10 @@ from generator_files import generate_files from generator_report import generate_report from generator_cpta import generate_cpta from generator_alerts import Alerting, AlertingError +from convert_xml_json import convert_xml_to_json -OUTPUTS = (u"none", u"report", u"trending", u"convert_to_json") +OUTPUTS = (u"none", u"report", u"trending", u"convert-xml-to-json") def parse_args(): @@ -131,6 +132,7 @@ def main(): spec.read_specification() except PresentationError as err: logging.critical(u"Finished with error.") + logging.critical(repr(err)) return 1 if spec.output[u"output"] not in OUTPUTS: @@ -170,6 +172,8 @@ def main(): alert.generate_alerts() except AlertingError as err: logging.warning(repr(err)) + elif spec.output[u"output"] == u"convert-xml-to-json": + convert_xml_to_json(spec, data) else: logging.info("No output will be generated.") diff --git a/resources/tools/presentation/run_convert.sh b/resources/tools/presentation/run_convert.sh new file mode 100755 index 0000000000..814fab3a28 --- /dev/null +++ b/resources/tools/presentation/run_convert.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -x + +# set default values in config array +typeset -A CFG +typeset -A DIR + +DIR[WORKING]=_tmp + +# Create working directories +mkdir ${DIR[WORKING]} + +# Create virtual environment +virtualenv -p $(which python3) ${DIR[WORKING]}/env +source ${DIR[WORKING]}/env/bin/activate + +# FIXME: s3 config (until migrated to vault, then account will be reset) +mkdir -p ${HOME}/.aws +echo "[nomad-s3]" >> ${HOME}/.aws/config +echo "[nomad-s3] +aws_access_key_id = csit +aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials + +# Install python dependencies: +pip3 install -r requirements.txt + +export PYTHONPATH=`pwd`:`pwd`/../../../ + +python pal.py \ + --specification specifications/converter \ + --logging INFO \ + +RETURN_STATUS=$(echo $?) +exit ${RETURN_STATUS} diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh index 8d3dd269a7..842339f7f5 100755 --- a/resources/tools/presentation/run_cpta.sh +++ b/resources/tools/presentation/run_cpta.sh @@ -24,7 +24,7 @@ aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials # Install python dependencies: pip3 install -r requirements.txt -export PYTHONPATH=`pwd`:`pwd`/../../../:`pwd`/../../libraries/python +export PYTHONPATH=`pwd`:`pwd`/../../../ STATUS=$(python pal.py \ --specification specifications/trending \ diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh index 9cc33542e0..2a14da1b62 100755 --- a/resources/tools/presentation/run_report.sh +++ b/resources/tools/presentation/run_report.sh @@ -27,7 +27,7 @@ aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials # Install python dependencies: pip3 install -r requirements.txt -export PYTHONPATH=`pwd`:`pwd`/../../../:`pwd`/../../libraries/python +export PYTHONPATH=`pwd`:`pwd`/../../../ python pal.py \ --specification specifications/report \ diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index 4110bfff9b..a94d09f3fa 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -192,7 +192,7 @@ class Specification: :returns: List of specifications of tables to be generated. :rtype: list """ - return self._specification[u"tables"] + return self._specification.get(u"tables", list()) @property def plots(self): @@ -201,7 +201,7 @@ class Specification: :returns: List of specifications of plots to be generated. :rtype: list """ - return self._specification[u"plots"] + return self._specification.get(u"plots", list()) @property def files(self): @@ -210,7 +210,7 @@ class Specification: :returns: List of specifications of files to be generated. :rtype: list """ - return self._specification[u"files"] + return self._specification.get(u"files", list()) @property def cpta(self): @@ -614,6 +614,8 @@ class Specification: idx = self._get_type_index(u"static") if idx is None: logging.warning(u"No static content specified.") + self._specification[u"static"] = dict() + return for key, value in self._cfg_yaml[idx].items(): if isinstance(value, str): @@ -816,10 +818,26 @@ class Specification: logging.info(u"Parsing specification: INPUT") - for data_set in self.data_sets.values(): - if data_set == "data-sets": - continue - for job, builds in data_set.items(): + idx = self._get_type_index(u"input") + if idx is None: + logging.info(u"Creating the list of inputs from data sets.") + for data_set in self.data_sets.values(): + if data_set == "data-sets": + continue + for job, builds in data_set.items(): + for build in builds: + self.add_build( + job, + { + u"build": build, + u"status": None, + u"file-name": None, + u"source": None + } + ) + else: + logging.info(u"Reading pre-defined inputs.") + for job, builds in self._cfg_yaml[idx][u"builds"].items(): for build in builds: self.add_build( job, diff --git a/resources/tools/presentation/specifications/converter/environment.yaml b/resources/tools/presentation/specifications/converter/environment.yaml new file mode 100644 index 0000000000..1f57445638 --- /dev/null +++ b/resources/tools/presentation/specifications/converter/environment.yaml @@ -0,0 +1,124 @@ +################################################################################ +### E N V I R O N M E N T ### +################################################################################ + +- type: "environment" + + spec-files: + - "specifications/converter/input.yaml" # Only for converter XML --> JSON + + paths: + # Top level directories: + ## Working directory + DIR[WORKING]: "_tmp" + ## Build directories + DIR[BUILD,JSON]: "_build" + + # Working directories + ## Input data files (.zip, .xml) + DIR[WORKING,DATA]: "{DIR[WORKING]}/data" + + # Data sources are used in this order: + data-sources: + # JSON from S3 + - type: "json" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output.json.gz" + file-format: ".gz" + enabled: False + # XML + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + # XML from docs.nexus + - type: "xml-docs" + url: "https://docs.fd.io/csit" + path: "report/_static/archive" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml-docs" + url: "https://docs.fd.io/csit" + path: "report/_static/archive" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml-docs" + url: "https://docs.fd.io/csit" + path: "report/_static/archive" + file-name: "robot-plugin.zip" + file-format: ".zip" + enabled: True + + make-dirs: + # List the directories which are created while preparing the environment. + # All directories MUST be defined in "paths" section. + - "DIR[WORKING,DATA]" + + remove-dirs: + # List the directories which are deleted while cleaning the environment. + # All directories MUST be defined in "paths" section. + - "DIR[WORKING,DATA]" + + build-dirs: + # List the directories where the results (build) is stored. + # All directories MUST be defined in "paths" section. + - "DIR[BUILD,JSON]" + +################################################################################ +### O U T P U T ### +################################################################################ + +- type: "output" + output: "convert-xml-to-json" + # type: flat | structured + # - flat - all .gz files in one directory + # - structured - .gz files in directories structured as job/build/*.gz + structure: "tree" # Use flat or tree + use-template: "json/template_0.1.0.json" diff --git a/resources/tools/presentation/specifications/converter/input.yaml b/resources/tools/presentation/specifications/converter/input.yaml new file mode 100644 index 0000000000..0cf765030d --- /dev/null +++ b/resources/tools/presentation/specifications/converter/input.yaml @@ -0,0 +1,21 @@ +################################################################################ +### I N P U T X M L F I L E S ### +################################################################################ + +# This is only an example for converter XML --> JSON + +- type: "input" + + # 3n-hsw + + builds: + csit-vpp-perf-report-iterative-2101-3n-hsw: + - 65 # rls2101.rel NDRPDR reconf iter env 6 + - 69 # rls2101.rel Hoststack iter env 6 + - 64 # rls2101.rel NDRPDR iter env 6 + - 63 # rls2101.rel MRR iter env 6 + csit-vpp-perf-report-iterative-2101-2n-skx: + - 94 # rls2101.rel NDRPDR iter env 6 + - 68 # rls2101.rel soak env 6 + csit-vpp-device-2101-ubuntu1804-1n-skx: + - 358 # rls2101.rel VPP DEV env 6 diff --git a/resources/tools/presentation/specifications/report/environment.yaml b/resources/tools/presentation/specifications/report/environment.yaml index 0e946046b4..10d61f56e4 100644 --- a/resources/tools/presentation/specifications/report/environment.yaml +++ b/resources/tools/presentation/specifications/report/environment.yaml @@ -193,7 +193,7 @@ file-format: ".zip" enabled: True - archive-inputs: True + archive-inputs: False mapping-file: "" @@ -224,7 +224,6 @@ reverse-input: False # Needed for trending, not important for the report - # TODO: Change in code needed, it was in type: "configuration" limits: nic: x520: 24460000 @@ -258,7 +257,7 @@ ################################################################################ - type: "output" - arch-file-format: # moved from input, TODO: change it in the code + arch-file-format: - ".gz" - ".zip" output: "report" diff --git a/resources/tools/presentation/specifications/trending/environment.yaml b/resources/tools/presentation/specifications/trending/environment.yaml index dfa9f680c2..95eaa7b606 100644 --- a/resources/tools/presentation/specifications/trending/environment.yaml +++ b/resources/tools/presentation/specifications/trending/environment.yaml @@ -256,7 +256,7 @@ ################################################################################ - type: "output" - arch-file-format: # moved from input, TODO: change it in the code + arch-file-format: - ".gz" - ".zip" output: "trending" |