diff options
Diffstat (limited to 'resources/tools/presentation/generator_cpta.py')
| -rw-r--r-- | resources/tools/presentation/generator_cpta.py | 635 |
1 files changed, 635 insertions, 0 deletions
diff --git a/resources/tools/presentation/generator_cpta.py b/resources/tools/presentation/generator_cpta.py new file mode 100644 index 0000000000..511800a988 --- /dev/null +++ b/resources/tools/presentation/generator_cpta.py @@ -0,0 +1,635 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generation of Continuous Performance Trending and Analysis. +""" + +import logging +import csv + +from collections import OrderedDict +from datetime import datetime +from copy import deepcopy + +import prettytable +import plotly.offline as ploff +import plotly.graph_objs as plgo +import plotly.exceptions as plerr + +from pal_utils import archive_input_data, execute_command, classify_anomalies + + +# Command to build the html format of the report +HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \ + u'-b html -E ' \ + u'-t html ' \ + u'-D version="{date}" ' \ + u'{working_dir} ' \ + u'{build_dir}/' + +# .css file for the html format of the report +THEME_OVERRIDES = u"""/* override table width restrictions */ +.wy-nav-content { + max-width: 1200px !important; +} +.rst-content blockquote { + margin-left: 0px; + line-height: 18px; + margin-bottom: 0px; +} +.wy-menu-vertical a { + display: inline-block; + line-height: 18px; + padding: 0 2em; + display: block; + position: relative; + font-size: 90%; + color: #d9d9d9 +} +.wy-menu-vertical li.current a { + color: gray; + border-right: solid 1px #c9c9c9; + padding: 0 3em; +} +.wy-menu-vertical li.toctree-l2.current > a { + background: #c9c9c9; + padding: 0 3em; +} +.wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a { + display: block; + background: #c9c9c9; + padding: 0 4em; +} +.wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a { + display: block; + background: #bdbdbd; + padding: 0 5em; +} +.wy-menu-vertical li.on a, .wy-menu-vertical li.current > a { + color: #404040; + padding: 0 2em; + font-weight: bold; + position: relative; + background: #fcfcfc; + border: none; + border-top-width: medium; + border-bottom-width: medium; + border-top-style: none; + border-bottom-style: none; + border-top-color: currentcolor; + border-bottom-color: currentcolor; + padding-left: 2em -4px; +} +""" + +COLORS = [ + u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink", + u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black", + u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson", + u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod", + u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon", + u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey", + u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink", + u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black", + u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson", + u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod", + u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon", + u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey" +] + + +def generate_cpta(spec, data): + """Generate all formats and versions of the Continuous Performance Trending + and Analysis. + + :param spec: Specification read from the specification file. + :param data: Full data set. + :type spec: Specification + :type data: InputData + """ + + logging.info(u"Generating the Continuous Performance Trending and Analysis " + u"...") + + ret_code = _generate_all_charts(spec, data) + + cmd = HTML_BUILDER.format( + date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'), + working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'], + build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]']) + execute_command(cmd) + + with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \ + css_file: + css_file.write(THEME_OVERRIDES) + + with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \ + css_file: + css_file.write(THEME_OVERRIDES) + + if spec.configuration.get(u"archive-inputs", True): + archive_input_data(spec) + + logging.info(u"Done.") + + return ret_code + + +def _generate_trending_traces(in_data, job_name, build_info, + show_trend_line=True, name=u"", color=u""): + """Generate the trending traces: + - samples, + - outliers, regress, progress + - average of normal samples (trending line) + + :param in_data: Full data set. + :param job_name: The name of job which generated the data. + :param build_info: Information about the builds. + :param show_trend_line: Show moving median (trending plot). + :param name: Name of the plot + :param color: Name of the color for the plot. + :type in_data: OrderedDict + :type job_name: str + :type build_info: dict + :type show_trend_line: bool + :type name: str + :type color: str + :returns: Generated traces (list) and the evaluated result. + :rtype: tuple(traces, result) + """ + + data_x = list(in_data.keys()) + data_y_pps = list() + data_y_mpps = list() + data_y_stdev = list() + for item in in_data.values(): + data_y_pps.append(float(item[u"receive-rate"])) + data_y_stdev.append(float(item[u"receive-stdev"]) / 1e6) + data_y_mpps.append(float(item[u"receive-rate"]) / 1e6) + + hover_text = list() + xaxis = list() + for index, key in enumerate(data_x): + str_key = str(key) + date = build_info[job_name][str_key][0] + hover_str = (u"date: {date}<br>" + u"value [Mpps]: {value:.3f}<br>" + u"stdev [Mpps]: {stdev:.3f}<br>" + u"{sut}-ref: {build}<br>" + u"csit-ref: mrr-{period}-build-{build_nr}<br>" + u"testbed: {testbed}") + if u"dpdk" in job_name: + hover_text.append(hover_str.format( + date=date, + value=data_y_mpps[index], + stdev=data_y_stdev[index], + sut=u"dpdk", + build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0], + period=u"weekly", + build_nr=str_key, + testbed=build_info[job_name][str_key][2])) + elif u"vpp" in job_name: + hover_text.append(hover_str.format( + date=date, + value=data_y_mpps[index], + stdev=data_y_stdev[index], + sut=u"vpp", + build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0], + period=u"daily", + build_nr=str_key, + testbed=build_info[job_name][str_key][2])) + + xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]), + int(date[9:11]), int(date[12:]))) + + data_pd = OrderedDict() + for key, value in zip(xaxis, data_y_pps): + data_pd[key] = value + + anomaly_classification, avgs_pps = classify_anomalies(data_pd) + avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps] + + anomalies = OrderedDict() + anomalies_colors = list() + anomalies_avgs = list() + anomaly_color = { + u"regression": 0.0, + u"normal": 0.5, + u"progression": 1.0 + } + if anomaly_classification: + for index, (key, value) in enumerate(data_pd.items()): + if anomaly_classification[index] in (u"regression", u"progression"): + anomalies[key] = value / 1e6 + anomalies_colors.append( + anomaly_color[anomaly_classification[index]]) + anomalies_avgs.append(avgs_mpps[index]) + anomalies_colors.extend([0.0, 0.5, 1.0]) + + # Create traces + + trace_samples = plgo.Scatter( + x=xaxis, + y=data_y_mpps, + mode=u"markers", + line={ + u"width": 1 + }, + showlegend=True, + legendgroup=name, + name=f"{name}", + marker={ + u"size": 5, + u"color": color, + u"symbol": u"circle", + }, + text=hover_text, + hoverinfo=u"text+name" + ) + traces = [trace_samples, ] + + if show_trend_line: + trace_trend = plgo.Scatter( + x=xaxis, + y=avgs_mpps, + mode=u"lines", + line={ + u"shape": u"linear", + u"width": 1, + u"color": color, + }, + showlegend=False, + legendgroup=name, + name=f"{name}", + text=[f"trend [Mpps]: {avg:.3f}" for avg in avgs_mpps], + hoverinfo=u"text+name" + ) + traces.append(trace_trend) + + trace_anomalies = plgo.Scatter( + x=list(anomalies.keys()), + y=anomalies_avgs, + mode=u"markers", + hoverinfo=u"none", + showlegend=False, + legendgroup=name, + name=f"{name}-anomalies", + marker={ + u"size": 15, + u"symbol": u"circle-open", + u"color": anomalies_colors, + u"colorscale": [ + [0.00, u"red"], + [0.33, u"red"], + [0.33, u"white"], + [0.66, u"white"], + [0.66, u"green"], + [1.00, u"green"] + ], + u"showscale": True, + u"line": { + u"width": 2 + }, + u"colorbar": { + u"y": 0.5, + u"len": 0.8, + u"title": u"Circles Marking Data Classification", + u"titleside": u"right", + u"titlefont": { + u"size": 14 + }, + u"tickmode": u"array", + u"tickvals": [0.167, 0.500, 0.833], + u"ticktext": [u"Regression", u"Normal", u"Progression"], + u"ticks": u"", + u"ticklen": 0, + u"tickangle": -90, + u"thickness": 10 + } + } + ) + traces.append(trace_anomalies) + + if anomaly_classification: + return traces, anomaly_classification[-1] + + return traces, None + + +def _generate_all_charts(spec, input_data): + """Generate all charts specified in the specification file. + + :param spec: Specification. + :param input_data: Full data set. + :type spec: Specification + :type input_data: InputData + """ + + def _generate_chart(graph): + """Generates the chart. + + :param graph: The graph to be generated + :type graph: dict + :returns: Dictionary with the job name, csv table with results and + list of tests classification results. + :rtype: dict + """ + + logging.info(f" Generating the chart {graph.get(u'title', u'')} ...") + + job_name = list(graph[u"data"].keys())[0] + + csv_tbl = list() + res = dict() + + # Transform the data + logging.info( + f" Creating the data set for the {graph.get(u'type', u'')} " + f"{graph.get(u'title', u'')}." + ) + + if graph.get(u"include", None): + data = input_data.filter_tests_by_name( + graph, + params=[u"type", u"result", u"tags"], + continue_on_error=True + ) + else: + data = input_data.filter_data( + graph, + params=[u"type", u"result", u"tags"], + continue_on_error=True) + + if data is None or data.empty: + logging.error(u"No data.") + return dict() + + chart_data = dict() + chart_tags = dict() + for job, job_data in data.items(): + if job != job_name: + continue + for index, bld in job_data.items(): + for test_name, test in bld.items(): + if chart_data.get(test_name, None) is None: + chart_data[test_name] = OrderedDict() + try: + chart_data[test_name][int(index)] = { + u"receive-rate": test[u"result"][u"receive-rate"], + u"receive-stdev": test[u"result"][u"receive-stdev"] + } + chart_tags[test_name] = test.get(u"tags", None) + except (KeyError, TypeError): + pass + + # Add items to the csv table: + for tst_name, tst_data in chart_data.items(): + tst_lst = list() + for bld in builds_dict[job_name]: + itm = tst_data.get(int(bld), dict()) + # CSIT-1180: Itm will be list, compute stats. + try: + tst_lst.append(str(itm.get(u"receive-rate", u""))) + except AttributeError: + tst_lst.append(u"") + csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n') + + # Generate traces: + traces = list() + index = 0 + groups = graph.get(u"groups", None) + visibility = list() + + if groups: + for group in groups: + visible = list() + for tag in group: + for tst_name, test_data in chart_data.items(): + if not test_data: + logging.warning(f"No data for the test {tst_name}") + continue + if tag not in chart_tags[tst_name]: + continue + try: + trace, rslt = _generate_trending_traces( + test_data, + job_name=job_name, + build_info=build_info, + name=u'-'.join(tst_name.split(u'.')[-1]. + split(u'-')[2:-1]), + color=COLORS[index]) + except IndexError: + logging.error(f"Out of colors: index: " + f"{index}, test: {tst_name}") + index += 1 + continue + traces.extend(trace) + visible.extend([True for _ in range(len(trace))]) + res[tst_name] = rslt + index += 1 + break + visibility.append(visible) + else: + for tst_name, test_data in chart_data.items(): + if not test_data: + logging.warning(f"No data for the test {tst_name}") + continue + try: + trace, rslt = _generate_trending_traces( + test_data, + job_name=job_name, + build_info=build_info, + name=u'-'.join( + tst_name.split(u'.')[-1].split(u'-')[2:-1]), + color=COLORS[index]) + except IndexError: + logging.error( + f"Out of colors: index: {index}, test: {tst_name}" + ) + index += 1 + continue + traces.extend(trace) + res[tst_name] = rslt + index += 1 + + if traces: + # Generate the chart: + try: + layout = deepcopy(graph[u"layout"]) + except KeyError as err: + logging.error(u"Finished with error: No layout defined") + logging.error(repr(err)) + return dict() + if groups: + show = list() + for i in range(len(visibility)): + visible = list() + for vis_idx, _ in enumerate(visibility): + for _ in range(len(visibility[vis_idx])): + visible.append(i == vis_idx) + show.append(visible) + + buttons = list() + buttons.append(dict( + label=u"All", + method=u"update", + args=[{u"visible": [True for _ in range(len(show[0]))]}, ] + )) + for i in range(len(groups)): + try: + label = graph[u"group-names"][i] + except (IndexError, KeyError): + label = f"Group {i + 1}" + buttons.append(dict( + label=label, + method=u"update", + args=[{u"visible": show[i]}, ] + )) + + layout[u"updatemenus"] = list([ + dict( + active=0, + type=u"dropdown", + direction=u"down", + xanchor=u"left", + yanchor=u"bottom", + x=-0.12, + y=1.0, + buttons=buttons + ) + ]) + + name_file = ( + f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}" + f"{spec.cpta[u'output-file-type']}") + + logging.info(f" Writing the file {name_file} ...") + plpl = plgo.Figure(data=traces, layout=layout) + try: + ploff.plot(plpl, show_link=False, auto_open=False, + filename=name_file) + except plerr.PlotlyEmptyDataError: + logging.warning(u"No data for the plot. Skipped.") + + return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res} + + builds_dict = dict() + for job in spec.input[u"builds"].keys(): + if builds_dict.get(job, None) is None: + builds_dict[job] = list() + for build in spec.input[u"builds"][job]: + status = build[u"status"] + if status not in (u"failed", u"not found", u"removed", None): + builds_dict[job].append(str(build[u"build"])) + + # Create "build ID": "date" dict: + build_info = dict() + tb_tbl = spec.environment.get(u"testbeds", None) + for job_name, job_data in builds_dict.items(): + if build_info.get(job_name, None) is None: + build_info[job_name] = OrderedDict() + for build in job_data: + testbed = u"" + tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"") + if tb_ip and tb_tbl: + testbed = tb_tbl.get(tb_ip, u"") + build_info[job_name][build] = ( + input_data.metadata(job_name, build).get(u"generated", u""), + input_data.metadata(job_name, build).get(u"version", u""), + testbed + ) + + anomaly_classifications = dict() + + # Create the table header: + csv_tables = dict() + for job_name in builds_dict: + if csv_tables.get(job_name, None) is None: + csv_tables[job_name] = list() + header = f"Build Number:,{u','.join(builds_dict[job_name])}\n" + csv_tables[job_name].append(header) + build_dates = [x[0] for x in build_info[job_name].values()] + header = f"Build Date:,{u','.join(build_dates)}\n" + csv_tables[job_name].append(header) + versions = [x[1] for x in build_info[job_name].values()] + header = f"Version:,{u','.join(versions)}\n" + csv_tables[job_name].append(header) + + for chart in spec.cpta[u"plots"]: + result = _generate_chart(chart) + if not result: + continue + + csv_tables[result[u"job_name"]].extend(result[u"csv_table"]) + + if anomaly_classifications.get(result[u"job_name"], None) is None: + anomaly_classifications[result[u"job_name"]] = dict() + anomaly_classifications[result[u"job_name"]].update(result[u"results"]) + + # Write the tables: + for job_name, csv_table in csv_tables.items(): + file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending" + with open(f"{file_name}.csv", u"wt") as file_handler: + file_handler.writelines(csv_table) + + txt_table = None + with open(f"{file_name}.csv", u"rt") as csv_file: + csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"') + line_nr = 0 + for row in csv_content: + if txt_table is None: + txt_table = prettytable.PrettyTable(row) + else: + if line_nr > 1: + for idx, item in enumerate(row): + try: + row[idx] = str(round(float(item) / 1000000, 2)) + except ValueError: + pass + try: + txt_table.add_row(row) + # PrettyTable raises Exception + except Exception as err: + logging.warning( + f"Error occurred while generating TXT table:\n{err}" + ) + line_nr += 1 + txt_table.align[u"Build Number:"] = u"l" + with open(f"{file_name}.txt", u"wt") as txt_file: + txt_file.write(str(txt_table)) + + # Evaluate result: + if anomaly_classifications: + result = u"PASS" + for job_name, job_data in anomaly_classifications.items(): + file_name = \ + f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt" + with open(file_name, u'w') as txt_file: + for test_name, classification in job_data.items(): + if classification == u"regression": + txt_file.write(test_name + u'\n') + if classification in (u"regression", u"outlier"): + result = u"FAIL" + file_name = \ + f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt" + with open(file_name, u'w') as txt_file: + for test_name, classification in job_data.items(): + if classification == u"progression": + txt_file.write(test_name + u'\n') + else: + result = u"FAIL" + + logging.info(f"Partial results: {anomaly_classifications}") + logging.info(f"Result: {result}") + + return result |