diff options
Diffstat (limited to 'resources/tools/presentation/generator_tables.py')
| -rw-r--r-- | resources/tools/presentation/generator_tables.py | 526 |
1 files changed, 502 insertions, 24 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index a667fffb16..8791ae5804 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -18,11 +18,16 @@ import logging import csv import prettytable +import pandas as pd from string import replace +from collections import OrderedDict +from numpy import nan, isnan +from xml.etree import ElementTree as ET from errors import PresentationError -from utils import mean, stdev, relative_change, remove_outliers +from utils import mean, stdev, relative_change, remove_outliers,\ + split_outliers, classify_anomalies def generate_tables(spec, data): @@ -58,6 +63,8 @@ def table_details(table, input_data): format(table.get("title", ""))) # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) data = input_data.filter_data(table) # Prepare the header of the tables @@ -124,10 +131,14 @@ def table_merged_details(table, input_data): format(table.get("title", ""))) # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) data = input_data.filter_data(table) data = input_data.merge_data(data) data.sort_index(inplace=True) + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) suites = input_data.filter_data(table, data_set="suites") suites = input_data.merge_data(suites) @@ -221,6 +232,8 @@ def table_performance_improvements(table, input_data): return None # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) data = input_data.filter_data(table) # Prepare the header of the tables @@ -352,16 +365,26 @@ def table_performance_comparison(table, input_data): format(table.get("title", ""))) # Transform the data - data = input_data.filter_data(table) + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables try: - header = ["Test case", - "{0} Throughput [Mpps]".format(table["reference"]["title"]), - "{0} stdev [Mpps]".format(table["reference"]["title"]), - "{0} Throughput [Mpps]".format(table["compare"]["title"]), - "{0} stdev [Mpps]".format(table["compare"]["title"]), - "Change [%]"] + header = ["Test case", ] + + history = table.get("history", None) + if history: + for item in history: + header.extend( + ["{0} Throughput [Mpps]".format(item["title"]), + "{0} Stdev [Mpps]".format(item["title"])]) + header.extend( + ["{0} Throughput [Mpps]".format(table["reference"]["title"]), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} Throughput [Mpps]".format(table["compare"]["title"]), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Change [%]"]) header_str = ",".join(header) + "\n" except (AttributeError, KeyError) as err: logging.error("The model is invalid, missing parameter: {0}". @@ -396,27 +419,68 @@ def table_performance_comparison(table, input_data): pass except TypeError: tbl_dict.pop(tst_name, None) + if history: + for item in history: + for job, builds in item["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if tbl_dict.get(tst_name, None) is None: + continue + if tbl_dict[tst_name].get("history", None) is None: + tbl_dict[tst_name]["history"] = OrderedDict() + if tbl_dict[tst_name]["history"].get(item["title"], + None) is None: + tbl_dict[tst_name]["history"][item["title"]] = \ + list() + try: + tbl_dict[tst_name]["history"][item["title"]].\ + append(tst_data["throughput"]["value"]) + except (TypeError, KeyError): + pass tbl_lst = list() for tst_name in tbl_dict.keys(): item = [tbl_dict[tst_name]["name"], ] + if history: + if tbl_dict[tst_name].get("history", None) is not None: + for hist_data in tbl_dict[tst_name]["history"].values(): + if hist_data: + data_t = remove_outliers( + hist_data, outlier_const=table["outlier-const"]) + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + else: + item.extend([None, None]) + else: + item.extend([None, None]) if tbl_dict[tst_name]["ref-data"]: data_t = remove_outliers(tbl_dict[tst_name]["ref-data"], - table["outlier-const"]) - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + outlier_const=table["outlier-const"]) + # TODO: Specify window size. + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) else: item.extend([None, None]) if tbl_dict[tst_name]["cmp-data"]: data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"], - table["outlier-const"]) - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + outlier_const=table["outlier-const"]) + # TODO: Specify window size. + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) else: item.extend([None, None]) - if item[1] is not None and item[3] is not None: - item.append(int(relative_change(float(item[1]), float(item[3])))) - if len(item) == 6: + if item[-4] is not None and item[-2] is not None and item[-4] != 0: + item.append(int(relative_change(float(item[-4]), float(item[-2])))) + if len(item) == len(header): tbl_lst.append(item) # Sort the table according to the relative change @@ -438,7 +502,7 @@ def table_performance_comparison(table, input_data): table["output-file-ext"]) ] for file_name in tbl_names: - logging.info(" Writing file: '{}'".format(file_name)) + logging.info(" Writing file: '{0}'".format(file_name)) with open(file_name, "w") as file_handler: file_handler.write(header_str) for test in tbl_lst: @@ -459,7 +523,7 @@ def table_performance_comparison(table, input_data): for i, txt_name in enumerate(tbl_names_txt): txt_table = None - logging.info(" Writing file: '{}'".format(txt_name)) + logging.info(" Writing file: '{0}'".format(txt_name)) with open(tbl_names[i], 'rb') as csv_file: csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') for row in csv_content: @@ -481,7 +545,7 @@ def table_performance_comparison(table, input_data): output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{}'".format(output_file)) + logging.info(" Writing file: '{0}'".format(output_file)) with open(output_file, "w") as out_file: out_file.write(header_str) for i, line in enumerate(lines[1:]): @@ -491,7 +555,7 @@ def table_performance_comparison(table, input_data): output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{}'".format(output_file)) + logging.info(" Writing file: '{0}'".format(output_file)) with open(output_file, "w") as out_file: out_file.write(header_str) for i, line in enumerate(lines[-1:0:-1]): @@ -508,7 +572,7 @@ def table_performance_comparison(table, input_data): output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{}'".format(output_file)) + logging.info(" Writing file: '{0}'".format(output_file)) with open(output_file, "w") as out_file: out_file.write(header_str) for i, line in enumerate(lines[1:]): @@ -518,10 +582,424 @@ def table_performance_comparison(table, input_data): output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{}'".format(output_file)) + logging.info(" Writing file: '{0}'".format(output_file)) with open(output_file, "w") as out_file: out_file.write(header_str) for i, line in enumerate(lines[-1:0:-1]): if i == table["nr-of-tests-shown"]: break out_file.write(line) + + +def table_performance_comparison_mrr(table, input_data): + """Generate the table(s) with algorithm: table_performance_comparison_mrr + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + # Prepare the header of the tables + try: + header = ["Test case", + "{0} Throughput [Mpps]".format(table["reference"]["title"]), + "{0} stdev [Mpps]".format(table["reference"]["title"]), + "{0} Throughput [Mpps]".format(table["compare"]["title"]), + "{0} stdev [Mpps]".format(table["compare"]["title"]), + "Change [%]"] + header_str = ",".join(header) + "\n" + except (AttributeError, KeyError) as err: + logging.error("The model is invalid, missing parameter: {0}". + format(err)) + return + + # Prepare data to the table: + tbl_dict = dict() + for job, builds in table["reference"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if tbl_dict.get(tst_name, None) is None: + name = "{0}-{1}".format(tst_data["parent"].split("-")[0], + "-".join(tst_data["name"]. + split("-")[1:])) + tbl_dict[tst_name] = {"name": name, + "ref-data": list(), + "cmp-data": list()} + try: + tbl_dict[tst_name]["ref-data"].\ + append(tst_data["result"]["throughput"]) + except TypeError: + pass # No data in output.xml for this test + + for job, builds in table["compare"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + try: + tbl_dict[tst_name]["cmp-data"].\ + append(tst_data["result"]["throughput"]) + except KeyError: + pass + except TypeError: + tbl_dict.pop(tst_name, None) + + tbl_lst = list() + for tst_name in tbl_dict.keys(): + item = [tbl_dict[tst_name]["name"], ] + if tbl_dict[tst_name]["ref-data"]: + data_t = remove_outliers(tbl_dict[tst_name]["ref-data"], + outlier_const=table["outlier-const"]) + # TODO: Specify window size. + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + else: + item.extend([None, None]) + if tbl_dict[tst_name]["cmp-data"]: + data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"], + outlier_const=table["outlier-const"]) + # TODO: Specify window size. + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + else: + item.extend([None, None]) + if item[1] is not None and item[3] is not None and item[1] != 0: + item.append(int(relative_change(float(item[1]), float(item[3])))) + if len(item) == 6: + tbl_lst.append(item) + + # Sort the table according to the relative change + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + + # Generate tables: + # All tests in csv: + tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"], + table["output-file-ext"]), + "{0}-2t2c-full{1}".format(table["output-file"], + table["output-file-ext"]), + "{0}-4t4c-full{1}".format(table["output-file"], + table["output-file-ext"]) + ] + for file_name in tbl_names: + logging.info(" Writing file: '{0}'".format(file_name)) + with open(file_name, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + if file_name.split("-")[-2] in test[0]: # cores + test[0] = "-".join(test[0].split("-")[:-1]) + file_handler.write(",".join([str(item) for item in test]) + + "\n") + + # All tests in txt: + tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]), + "{0}-2t2c-full.txt".format(table["output-file"]), + "{0}-4t4c-full.txt".format(table["output-file"]) + ] + + for i, txt_name in enumerate(tbl_names_txt): + txt_table = None + logging.info(" Writing file: '{0}'".format(txt_name)) + with open(tbl_names[i], 'rb') as csv_file: + csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') + for row in csv_content: + if txt_table is None: + txt_table = prettytable.PrettyTable(row) + else: + txt_table.add_row(row) + txt_table.align["Test case"] = "l" + with open(txt_name, "w") as txt_file: + txt_file.write(str(txt_table)) + + +def table_performance_trending_dashboard(table, input_data): + """Generate the table(s) with algorithm: table_performance_comparison + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + # Prepare the header of the tables + header = ["Test Case", + "Trend [Mpps]", + "Short-Term Change [%]", + "Long-Term Change [%]", + "Regressions [#]", + "Progressions [#]", + "Outliers [#]" + ] + header_str = ",".join(header) + "\n" + + # Prepare data to the table: + tbl_dict = dict() + for job, builds in table["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if tst_name.lower() in table["ignore-list"]: + continue + if tbl_dict.get(tst_name, None) is None: + name = "{0}-{1}".format(tst_data["parent"].split("-")[0], + "-".join(tst_data["name"]. + split("-")[1:])) + tbl_dict[tst_name] = {"name": name, + "data": OrderedDict()} + try: + tbl_dict[tst_name]["data"][str(build)] = \ + tst_data["result"]["throughput"] + except (TypeError, KeyError): + pass # No data in output.xml for this test + + tbl_lst = list() + for tst_name in tbl_dict.keys(): + if len(tbl_dict[tst_name]["data"]) < 3: + continue + + pd_data = pd.Series(tbl_dict[tst_name]["data"]) + data_t, _ = split_outliers(pd_data, outlier_const=1.5, + window=table["window"]) + last_key = data_t.keys()[-1] + win_size = min(data_t.size, table["window"]) + win_first_idx = data_t.size - win_size + key_14 = data_t.keys()[win_first_idx] + long_win_size = min(data_t.size, table["long-trend-window"]) + median_t = data_t.rolling(window=win_size, min_periods=2).median() + median_first_idx = median_t.size - long_win_size + try: + max_median = max( + [x for x in median_t.values[median_first_idx:-win_size] + if not isnan(x)]) + except ValueError: + max_median = nan + try: + last_median_t = median_t[last_key] + except KeyError: + last_median_t = nan + try: + median_t_14 = median_t[key_14] + except KeyError: + median_t_14 = nan + + if isnan(last_median_t) or isnan(median_t_14) or median_t_14 == 0.0: + rel_change_last = nan + else: + rel_change_last = round( + ((last_median_t - median_t_14) / median_t_14) * 100, 2) + + if isnan(max_median) or isnan(last_median_t) or max_median == 0.0: + rel_change_long = nan + else: + rel_change_long = round( + ((last_median_t - max_median) / max_median) * 100, 2) + + # Classification list: + classification_lst = classify_anomalies(data_t, window=14) + + if classification_lst: + tbl_lst.append( + [tbl_dict[tst_name]["name"], + '-' if isnan(last_median_t) else + round(last_median_t / 1000000, 2), + '-' if isnan(rel_change_last) else rel_change_last, + '-' if isnan(rel_change_long) else rel_change_long, + classification_lst[win_first_idx:].count("regression"), + classification_lst[win_first_idx:].count("progression"), + classification_lst[win_first_idx:].count("outlier")]) + + tbl_lst.sort(key=lambda rel: rel[0]) + + tbl_sorted = list() + for nrr in range(table["window"], -1, -1): + tbl_reg = [item for item in tbl_lst if item[4] == nrr] + for nrp in range(table["window"], -1, -1): + tbl_pro = [item for item in tbl_reg if item[5] == nrp] + for nro in range(table["window"], -1, -1): + tbl_out = [item for item in tbl_pro if item[6] == nro] + tbl_out.sort(key=lambda rel: rel[2]) + tbl_sorted.extend(tbl_out) + + file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) + + logging.info(" Writing file: '{0}'".format(file_name)) + with open(file_name, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_sorted: + file_handler.write(",".join([str(item) for item in test]) + '\n') + + txt_file_name = "{0}.txt".format(table["output-file"]) + txt_table = None + logging.info(" Writing file: '{0}'".format(txt_file_name)) + with open(file_name, 'rb') as csv_file: + csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') + for row in csv_content: + if txt_table is None: + txt_table = prettytable.PrettyTable(row) + else: + txt_table.add_row(row) + txt_table.align["Test case"] = "l" + with open(txt_file_name, "w") as txt_file: + txt_file.write(str(txt_table)) + + +def table_performance_trending_dashboard_html(table, input_data): + """Generate the table(s) with algorithm: + table_performance_trending_dashboard_html specified in the specification + file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + try: + with open(table["input-file"], 'rb') as csv_file: + csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') + csv_lst = [item for item in csv_content] + except KeyError: + logging.warning("The input file is not defined.") + return + except csv.Error as err: + logging.warning("Not possible to process the file '{0}'.\n{1}". + format(table["input-file"], err)) + return + + # Table: + dashboard = ET.Element("table", attrib=dict(width="100%", border='0')) + + # Table header: + tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7")) + for idx, item in enumerate(csv_lst[0]): + alignment = "left" if idx == 0 else "center" + th = ET.SubElement(tr, "th", attrib=dict(align=alignment)) + th.text = item + + # Rows: + colors = {"regression": ("#ffcccc", "#ff9999"), + "progression": ("#c6ecc6", "#9fdf9f"), + "outlier": ("#e6e6e6", "#cccccc"), + "normal": ("#e9f1fb", "#d4e4f7")} + for r_idx, row in enumerate(csv_lst[1:]): + if int(row[4]): + color = "regression" + elif int(row[5]): + color = "progression" + elif int(row[6]): + color = "outlier" + else: + color = "normal" + background = colors[color][r_idx % 2] + tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background)) + + # Columns: + for c_idx, item in enumerate(row): + alignment = "left" if c_idx == 0 else "center" + td = ET.SubElement(tr, "td", attrib=dict(align=alignment)) + # Name: + url = "../trending/" + file_name = "" + anchor = "#" + feature = "" + if c_idx == 0: + if "memif" in item: + file_name = "container_memif.html" + + elif "srv6" in item: + file_name = "srv6.html" + + elif "vhost" in item: + if "l2xcbase" in item or "l2bdbasemaclrn" in item: + file_name = "vm_vhost_l2.html" + elif "ip4base" in item: + file_name = "vm_vhost_ip4.html" + + elif "ipsec" in item: + file_name = "ipsec.html" + + elif "ethip4lispip" in item or "ethip4vxlan" in item: + file_name = "ip4_tunnels.html" + + elif "ip4base" in item or "ip4scale" in item: + file_name = "ip4.html" + if "iacl" in item or "snat" in item or "cop" in item: + feature = "-features" + + elif "ip6base" in item or "ip6scale" in item: + file_name = "ip6.html" + + elif "l2xcbase" in item or "l2xcscale" in item \ + or "l2bdbasemaclrn" in item or "l2bdscale" in item \ + or "l2dbbasemaclrn" in item or "l2dbscale" in item: + file_name = "l2.html" + if "iacl" in item: + feature = "-features" + + if "x520" in item: + anchor += "x520-" + elif "x710" in item: + anchor += "x710-" + elif "xl710" in item: + anchor += "xl710-" + + if "64b" in item: + anchor += "64b-" + elif "78b" in item: + anchor += "78b-" + elif "imix" in item: + anchor += "imix-" + elif "9000b" in item: + anchor += "9000b-" + elif "1518" in item: + anchor += "1518b-" + + if "1t1c" in item: + anchor += "1t1c" + elif "2t2c" in item: + anchor += "2t2c" + elif "4t4c" in item: + anchor += "4t4c" + + url = url + file_name + anchor + feature + + ref = ET.SubElement(td, "a", attrib=dict(href=url)) + ref.text = item + + if c_idx > 0: + td.text = item + + try: + with open(table["output-file"], 'w') as html_file: + logging.info(" Writing file: '{0}'". + format(table["output-file"])) + html_file.write(".. raw:: html\n\n\t") + html_file.write(ET.tostring(dashboard)) + html_file.write("\n\t<p><br><br></p>\n") + except KeyError: + logging.warning("The output file is not defined.") + return |