diff options
Diffstat (limited to 'resources/tools/presentation/generator_tables.py')
| -rw-r--r-- | resources/tools/presentation/generator_tables.py | 981 |
1 files changed, 15 insertions, 966 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 154e7e7885..a72551f15a 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -50,9 +50,6 @@ def generate_tables(spec, data): generator = { u"table_merged_details": table_merged_details, - u"table_perf_comparison": table_perf_comparison, - u"table_perf_comparison_nic": table_perf_comparison_nic, - u"table_nics_comparison": table_nics_comparison, u"table_soak_vs_ndr": table_soak_vs_ndr, u"table_perf_trending_dash": table_perf_trending_dash, u"table_perf_trending_dash_html": table_perf_trending_dash_html, @@ -436,62 +433,16 @@ def _tpc_insert_data(target, src, include_tests): """ try: if include_tests == u"MRR": - target.append( - ( - src[u"result"][u"receive-rate"], - src[u"result"][u"receive-stdev"] - ) - ) + target[u"mean"] = src[u"result"][u"receive-rate"] + target[u"stdev"] = src[u"result"][u"receive-stdev"] elif include_tests == u"PDR": - target.append(src[u"throughput"][u"PDR"][u"LOWER"]) + target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"]) elif include_tests == u"NDR": - target.append(src[u"throughput"][u"NDR"][u"LOWER"]) + target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"]) except (KeyError, TypeError): pass -def _tpc_sort_table(table): - """Sort the table this way: - - 1. Put "New in CSIT-XXXX" at the first place. - 2. Put "See footnote" at the second place. - 3. Sort the rest by "Delta". - - :param table: Table to sort. - :type table: list - :returns: Sorted table. - :rtype: list - """ - - tbl_new = list() - tbl_see = list() - tbl_delta = list() - for item in table: - if isinstance(item[-1], str): - if u"New in CSIT" in item[-1]: - tbl_new.append(item) - elif u"See footnote" in item[-1]: - tbl_see.append(item) - else: - tbl_delta.append(item) - - # Sort the tables: - tbl_new.sort(key=lambda rel: rel[0], reverse=False) - tbl_see.sort(key=lambda rel: rel[0], reverse=False) - tbl_see.sort(key=lambda rel: rel[-2], reverse=False) - tbl_delta.sort(key=lambda rel: rel[0], reverse=False) - tbl_delta.sort(key=lambda rel: rel[-2], reverse=True) - - # Put the tables together: - table = list() - # We do not want "New in CSIT": - # table.extend(tbl_new) - table.extend(tbl_see) - table.extend(tbl_delta) - - return table - - def _tpc_generate_html_table(header, data, out_file_name, legend=u"", footnote=u"", sort_data=True, title=u"", generate_rst=True): @@ -585,8 +536,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", buttons = list() menu_items = [f"<b>{itm}</b> (ascending)" for itm in header] - menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header] - menu_items.extend(menu_items_rev) + menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header]) for idx, hdr in enumerate(menu_items): visible = [False, ] * len(menu_items) visible[idx] = True @@ -667,908 +617,6 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", rst_file.write(footnote.replace(u"\n", u" |br| ")[1:]) -def table_perf_comparison(table, input_data): - """Generate the table(s) with algorithm: table_perf_comparison - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table, continue_on_error=True) - - # Prepare the header of the tables - try: - header = [u"Test Case", ] - legend = u"\nLegend:\n" - - rca_data = None - rca = table.get(u"rca", None) - if rca: - try: - with open(rca.get(u"data-file", u""), u"r") as rca_file: - rca_data = load(rca_file, Loader=FullLoader) - header.insert(0, rca.get(u"title", u"RCA")) - legend += ( - u"RCA: Reference to the Root Cause Analysis, see below.\n" - ) - except (YAMLError, IOError) as err: - logging.warning(repr(err)) - - history = table.get(u"history", list()) - for item in history: - header.extend( - [ - f"{item[u'title']} Avg({table[u'include-tests']})", - f"{item[u'title']} Stdev({table[u'include-tests']})" - ] - ) - legend += ( - f"{item[u'title']} Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from " - f"a series of runs of the listed tests executed against " - f"{item[u'title']}.\n" - f"{item[u'title']} Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {item[u'title']}.\n" - ) - header.extend( - [ - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']})", - u"Stdev(Diff)" - ] - ) - header_str = u";".join(header) + u"\n" - legend += ( - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'reference'][u'title']}.\n" - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'reference'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'compare'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'compare'][u'title']}.\n" - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']}): " - f"Percentage change calculated for mean values.\n" - u"Stdev(Diff): " - u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u"NT: Not Tested\n" - ) - except (AttributeError, KeyError) as err: - logging.error(f"The model is invalid, missing parameter: {repr(err)}") - return - - # Prepare data to the table: - tbl_dict = dict() - for job, builds in table[u"reference"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": True, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"]) - - replacement = table[u"reference"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-ref"]: - tbl_dict[tst_name_mod][u"replace-ref"] = False - tbl_dict[tst_name_mod][u"ref-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for job, builds in table[u"compare"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - replacement = table[u"compare"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": False, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-cmp"]: - tbl_dict[tst_name_mod][u"replace-cmp"] = False - tbl_dict[tst_name_mod][u"cmp-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for item in history: - for job, builds in item[u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - continue - if tbl_dict[tst_name_mod].get(u"history", None) is None: - tbl_dict[tst_name_mod][u"history"] = OrderedDict() - if tbl_dict[tst_name_mod][u"history"].\ - get(item[u"title"], None) is None: - tbl_dict[tst_name_mod][u"history"][item[ - u"title"]] = list() - try: - if table[u"include-tests"] == u"MRR": - res = (tst_data[u"result"][u"receive-rate"], - tst_data[u"result"][u"receive-stdev"]) - elif table[u"include-tests"] == u"PDR": - res = tst_data[u"throughput"][u"PDR"][u"LOWER"] - elif table[u"include-tests"] == u"NDR": - res = tst_data[u"throughput"][u"NDR"][u"LOWER"] - else: - continue - tbl_dict[tst_name_mod][u"history"][item[u"title"]].\ - append(res) - except (TypeError, KeyError): - pass - - tbl_lst = list() - for tst_name in tbl_dict: - item = [tbl_dict[tst_name][u"name"], ] - if history: - if tbl_dict[tst_name].get(u"history", None) is not None: - for hist_data in tbl_dict[tst_name][u"history"].values(): - if hist_data: - if table[u"include-tests"] == u"MRR": - item.append(round(hist_data[0][0] / 1e6, 1)) - item.append(round(hist_data[0][1] / 1e6, 1)) - else: - item.append(round(mean(hist_data) / 1e6, 1)) - item.append(round(stdev(hist_data) / 1e6, 1)) - else: - item.extend([u"NT", u"NT"]) - else: - item.extend([u"NT", u"NT"]) - data_r = tbl_dict[tst_name][u"ref-data"] - if data_r: - if table[u"include-tests"] == u"MRR": - data_r_mean = data_r[0][0] - data_r_stdev = data_r[0][1] - else: - data_r_mean = mean(data_r) - data_r_stdev = stdev(data_r) - item.append(round(data_r_mean / 1e6, 1)) - item.append(round(data_r_stdev / 1e6, 1)) - else: - data_r_mean = None - data_r_stdev = None - item.extend([u"NT", u"NT"]) - data_c = tbl_dict[tst_name][u"cmp-data"] - if data_c: - if table[u"include-tests"] == u"MRR": - data_c_mean = data_c[0][0] - data_c_stdev = data_c[0][1] - else: - data_c_mean = mean(data_c) - data_c_stdev = stdev(data_c) - item.append(round(data_c_mean / 1e6, 1)) - item.append(round(data_c_stdev / 1e6, 1)) - else: - data_c_mean = None - data_c_stdev = None - item.extend([u"NT", u"NT"]) - if item[-2] == u"NT": - pass - elif item[-4] == u"NT": - item.append(u"New in CSIT-2001") - item.append(u"New in CSIT-2001") - elif data_r_mean is not None and data_c_mean is not None: - delta, d_stdev = relative_change_stdev( - data_r_mean, data_c_mean, data_r_stdev, data_c_stdev - ) - try: - item.append(round(delta)) - except ValueError: - item.append(delta) - try: - item.append(round(d_stdev)) - except ValueError: - item.append(d_stdev) - if rca_data: - rca_nr = rca_data.get(item[0], u"-") - item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") - if (len(item) == len(header)) and (item[-4] != u"NT"): - tbl_lst.append(item) - - tbl_lst = _tpc_sort_table(tbl_lst) - - # Generate csv tables: - csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"wt") as file_handler: - file_handler.write(header_str) - for test in tbl_lst: - file_handler.write(u";".join([str(item) for item in test]) + u"\n") - - txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") - - footnote = u"" - with open(txt_file_name, u'a') as txt_file: - txt_file.write(legend) - if rca_data: - footnote = rca_data.get(u"footnote", u"") - if footnote: - txt_file.write(f"\n{footnote}") - txt_file.write(u"\n:END") - - # Generate html table: - _tpc_generate_html_table( - header, - tbl_lst, - table[u'output-file'], - legend=legend, - footnote=footnote, - title=table.get(u"title", u"") - ) - - -def table_perf_comparison_nic(table, input_data): - """Generate the table(s) with algorithm: table_perf_comparison - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table, continue_on_error=True) - - # Prepare the header of the tables - try: - header = [u"Test Case", ] - legend = u"\nLegend:\n" - - rca_data = None - rca = table.get(u"rca", None) - if rca: - try: - with open(rca.get(u"data-file", ""), u"r") as rca_file: - rca_data = load(rca_file, Loader=FullLoader) - header.insert(0, rca.get(u"title", "RCA")) - legend += ( - u"RCA: Reference to the Root Cause Analysis, see below.\n" - ) - except (YAMLError, IOError) as err: - logging.warning(repr(err)) - - history = table.get(u"history", list()) - for item in history: - header.extend( - [ - f"{item[u'title']} Avg({table[u'include-tests']})", - f"{item[u'title']} Stdev({table[u'include-tests']})" - ] - ) - legend += ( - f"{item[u'title']} Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from " - f"a series of runs of the listed tests executed against " - f"{item[u'title']}.\n" - f"{item[u'title']} Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {item[u'title']}.\n" - ) - header.extend( - [ - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']})", - u"Stdev(Diff)" - ] - ) - header_str = u";".join(header) + u"\n" - legend += ( - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'reference'][u'title']}.\n" - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'reference'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'compare'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'compare'][u'title']}.\n" - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']}): " - f"Percentage change calculated for mean values.\n" - u"Stdev(Diff): " - u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u"NT: Not Tested\n" - ) - except (AttributeError, KeyError) as err: - logging.error(f"The model is invalid, missing parameter: {repr(err)}") - return - - # Prepare data to the table: - tbl_dict = dict() - for job, builds in table[u"reference"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - if table[u"reference"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": True, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - replacement = table[u"reference"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - if table[u"reference"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = \ - _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-ref"]: - tbl_dict[tst_name_mod][u"replace-ref"] = False - tbl_dict[tst_name_mod][u"ref-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for job, builds in table[u"compare"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - if table[u"compare"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - replacement = table[u"compare"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - if table[u"compare"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = \ - _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": False, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-cmp"]: - tbl_dict[tst_name_mod][u"replace-cmp"] = False - tbl_dict[tst_name_mod][u"cmp-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for item in history: - for job, builds in item[u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - if item[u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = \ - _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - continue - if tbl_dict[tst_name_mod].get(u"history", None) is None: - tbl_dict[tst_name_mod][u"history"] = OrderedDict() - if tbl_dict[tst_name_mod][u"history"].\ - get(item[u"title"], None) is None: - tbl_dict[tst_name_mod][u"history"][item[ - u"title"]] = list() - try: - if table[u"include-tests"] == u"MRR": - res = (tst_data[u"result"][u"receive-rate"], - tst_data[u"result"][u"receive-stdev"]) - elif table[u"include-tests"] == u"PDR": - res = tst_data[u"throughput"][u"PDR"][u"LOWER"] - elif table[u"include-tests"] == u"NDR": - res = tst_data[u"throughput"][u"NDR"][u"LOWER"] - else: - continue - tbl_dict[tst_name_mod][u"history"][item[u"title"]].\ - append(res) - except (TypeError, KeyError): - pass - - tbl_lst = list() - for tst_name in tbl_dict: - item = [tbl_dict[tst_name][u"name"], ] - if history: - if tbl_dict[tst_name].get(u"history", None) is not None: - for hist_data in tbl_dict[tst_name][u"history"].values(): - if hist_data: - if table[u"include-tests"] == u"MRR": - item.append(round(hist_data[0][0] / 1e6, 1)) - item.append(round(hist_data[0][1] / 1e6, 1)) - else: - item.append(round(mean(hist_data) / 1e6, 1)) - item.append(round(stdev(hist_data) / 1e6, 1)) - else: - item.extend([u"NT", u"NT"]) - else: - item.extend([u"NT", u"NT"]) - data_r = tbl_dict[tst_name][u"ref-data"] - if data_r: - if table[u"include-tests"] == u"MRR": - data_r_mean = data_r[0][0] - data_r_stdev = data_r[0][1] - else: - data_r_mean = mean(data_r) - data_r_stdev = stdev(data_r) - item.append(round(data_r_mean / 1e6, 1)) - item.append(round(data_r_stdev / 1e6, 1)) - else: - data_r_mean = None - data_r_stdev = None - item.extend([u"NT", u"NT"]) - data_c = tbl_dict[tst_name][u"cmp-data"] - if data_c: - if table[u"include-tests"] == u"MRR": - data_c_mean = data_c[0][0] - data_c_stdev = data_c[0][1] - else: - data_c_mean = mean(data_c) - data_c_stdev = stdev(data_c) - item.append(round(data_c_mean / 1e6, 1)) - item.append(round(data_c_stdev / 1e6, 1)) - else: - data_c_mean = None - data_c_stdev = None - item.extend([u"NT", u"NT"]) - if item[-2] == u"NT": - pass - elif item[-4] == u"NT": - item.append(u"New in CSIT-2001") - item.append(u"New in CSIT-2001") - elif data_r_mean is not None and data_c_mean is not None: - delta, d_stdev = relative_change_stdev( - data_r_mean, data_c_mean, data_r_stdev, data_c_stdev - ) - try: - item.append(round(delta)) - except ValueError: - item.append(delta) - try: - item.append(round(d_stdev)) - except ValueError: - item.append(d_stdev) - if rca_data: - rca_nr = rca_data.get(item[0], u"-") - item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") - if (len(item) == len(header)) and (item[-4] != u"NT"): - tbl_lst.append(item) - - tbl_lst = _tpc_sort_table(tbl_lst) - - # Generate csv tables: - csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"wt") as file_handler: - file_handler.write(header_str) - for test in tbl_lst: - file_handler.write(u";".join([str(item) for item in test]) + u"\n") - - txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") - - footnote = u"" - with open(txt_file_name, u'a') as txt_file: - txt_file.write(legend) - if rca_data: - footnote = rca_data.get(u"footnote", u"") - if footnote: - txt_file.write(f"\n{footnote}") - txt_file.write(u"\n:END") - - # Generate html table: - _tpc_generate_html_table( - header, - tbl_lst, - table[u'output-file'], - legend=legend, - footnote=footnote, - title=table.get(u"title", u"") - ) - - -def table_nics_comparison(table, input_data): - """Generate the table(s) with algorithm: table_nics_comparison - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table, continue_on_error=True) - - # Prepare the header of the tables - try: - header = [ - u"Test Case", - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']})", - u"Stdev(Diff)" - ] - legend = ( - u"\nLegend:\n" - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed using " - f"{table[u'reference'][u'title']} NIC.\n" - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"using {table[u'reference'][u'title']} NIC.\n" - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed using " - f"{table[u'compare'][u'title']} NIC.\n" - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"using {table[u'compare'][u'title']} NIC.\n" - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']}): " - f"Percentage change calculated for mean values.\n" - u"Stdev(Diff): " - u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u":END" - ) - - except (AttributeError, KeyError) as err: - logging.error(f"The model is invalid, missing parameter: {repr(err)}") - return - - # Prepare data to the table: - tbl_dict = dict() - for job, builds in table[u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - tbl_dict[tst_name_mod] = { - u"name": name, - u"ref-data": list(), - u"cmp-data": list() - } - try: - if table[u"include-tests"] == u"MRR": - result = (tst_data[u"result"][u"receive-rate"], - tst_data[u"result"][u"receive-stdev"]) - elif table[u"include-tests"] == u"PDR": - result = tst_data[u"throughput"][u"PDR"][u"LOWER"] - elif table[u"include-tests"] == u"NDR": - result = tst_data[u"throughput"][u"NDR"][u"LOWER"] - else: - continue - - if result and \ - table[u"reference"][u"nic"] in tst_data[u"tags"]: - tbl_dict[tst_name_mod][u"ref-data"].append(result) - elif result and \ - table[u"compare"][u"nic"] in tst_data[u"tags"]: - tbl_dict[tst_name_mod][u"cmp-data"].append(result) - except (TypeError, KeyError) as err: - logging.debug(f"No data for {tst_name}\n{repr(err)}") - # No data in output.xml for this test - - tbl_lst = list() - for tst_name in tbl_dict: - item = [tbl_dict[tst_name][u"name"], ] - data_r = tbl_dict[tst_name][u"ref-data"] - if data_r: - if table[u"include-tests"] == u"MRR": - data_r_mean = data_r[0][0] - data_r_stdev = data_r[0][1] - else: - data_r_mean = mean(data_r) - data_r_stdev = stdev(data_r) - item.append(round(data_r_mean / 1e6, 1)) - item.append(round(data_r_stdev / 1e6, 1)) - else: - data_r_mean = None - data_r_stdev = None - item.extend([None, None]) - data_c = tbl_dict[tst_name][u"cmp-data"] - if data_c: - if table[u"include-tests"] == u"MRR": - data_c_mean = data_c[0][0] - data_c_stdev = data_c[0][1] - else: - data_c_mean = mean(data_c) - data_c_stdev = stdev(data_c) - item.append(round(data_c_mean / 1e6, 1)) - item.append(round(data_c_stdev / 1e6, 1)) - else: - data_c_mean = None - data_c_stdev = None - item.extend([None, None]) - if data_r_mean is not None and data_c_mean is not None: - delta, d_stdev = relative_change_stdev( - data_r_mean, data_c_mean, data_r_stdev, data_c_stdev - ) - try: - item.append(round(delta)) - except ValueError: - item.append(delta) - try: - item.append(round(d_stdev)) - except ValueError: - item.append(d_stdev) - tbl_lst.append(item) - - # Sort the table according to the relative change - tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) - - # Generate csv tables: - with open(f"{table[u'output-file']}.csv", u"wt") as file_handler: - file_handler.write(u";".join(header) + u"\n") - for test in tbl_lst: - file_handler.write(u";".join([str(item) for item in test]) + u"\n") - - convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv", - f"{table[u'output-file']}.txt", - delimiter=u";") - - with open(table[u'output-file'], u'a') as txt_file: - txt_file.write(legend) - - # Generate html table: - _tpc_generate_html_table( - header, - tbl_lst, - table[u'output-file'], - legend=legend, - title=table.get(u"title", u"") - ) - - def table_soak_vs_ndr(table, input_data): """Generate the table(s) with algorithm: table_soak_vs_ndr specified in the specification file. @@ -2404,6 +1452,7 @@ def table_comparison(table, input_data): if col.get(u"data-set", None) is None: logging.warning(f"No data for column {col.get(u'title', u'')}") continue + tag = col.get(u"tag", None) data = input_data.filter_data( table, params=[u"throughput", u"result", u"name", u"parent", u"tags"], @@ -2417,8 +1466,11 @@ def table_comparison(table, input_data): for builds in data.values: for build in builds: for tst_name, tst_data in build.items(): + if tag and tag not in tst_data[u"tags"]: + continue tst_name_mod = \ - _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"") + _tpc_modify_test_name(tst_name, ignore_nic=True).\ + replace(u"2n1l-", u"") if col_data[u"data"].get(tst_name_mod, None) is None: name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ @@ -2432,7 +1484,7 @@ def table_comparison(table, input_data): u"stdev": None } _tpc_insert_data( - target=col_data[u"data"][tst_name_mod][u"data"], + target=col_data[u"data"][tst_name_mod], src=tst_data, include_tests=table[u"include-tests"] ) @@ -2448,8 +1500,10 @@ def table_comparison(table, input_data): for builds in rpl_data.values: for build in builds: for tst_name, tst_data in build.items(): + if tag and tag not in tst_data[u"tags"]: + continue tst_name_mod = \ - _tpc_modify_test_name(tst_name).\ + _tpc_modify_test_name(tst_name, ignore_nic=True).\ replace(u"2n1l-", u"") if col_data[u"data"].get(tst_name_mod, None) is None: name = tst_data[u'name'].rsplit(u'-', 1)[0] @@ -2468,7 +1522,7 @@ def table_comparison(table, input_data): col_data[u"data"][tst_name_mod][u"replace"] = False col_data[u"data"][tst_name_mod][u"data"] = list() _tpc_insert_data( - target=col_data[u"data"][tst_name_mod][u"data"], + target=col_data[u"data"][tst_name_mod], src=tst_data, include_tests=table[u"include-tests"] ) @@ -2478,11 +1532,6 @@ def table_comparison(table, input_data): if tst_data[u"data"]: tst_data[u"mean"] = mean(tst_data[u"data"]) tst_data[u"stdev"] = stdev(tst_data[u"data"]) - elif table[u"include-tests"] in (u"MRR", ): - for tst_name, tst_data in col_data[u"data"].items(): - if tst_data[u"data"]: - tst_data[u"mean"] = tst_data[u"data"][0] - tst_data[u"stdev"] = tst_data[u"data"][0] cols.append(col_data) |