diff options
Diffstat (limited to 'resources/tools/presentation')
| -rw-r--r-- | resources/tools/presentation/generator_tables.py | 188 | ||||
| -rw-r--r-- | resources/tools/presentation/pal_utils.py | 2 | ||||
| -rw-r--r-- | resources/tools/presentation/specification_CPTA.yaml | 194 | ||||
| -rw-r--r-- | resources/tools/presentation/specification_parser.py | 5 |
4 files changed, 364 insertions, 25 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 14130e5059..167d58d0fd 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -33,7 +33,7 @@ from numpy import nan, isnan from yaml import load, FullLoader, YAMLError from pal_utils import mean, stdev, classify_anomalies, \ - convert_csv_to_pretty_txt, relative_change_stdev + convert_csv_to_pretty_txt, relative_change_stdev, relative_change REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)') @@ -60,12 +60,15 @@ def generate_tables(spec, data): u"table_failed_tests": table_failed_tests, u"table_failed_tests_html": table_failed_tests_html, u"table_oper_data_html": table_oper_data_html, - u"table_comparison": table_comparison + u"table_comparison": table_comparison, + u"table_weekly_comparison": table_weekly_comparison } logging.info(u"Generating the tables ...") for table in spec.tables: try: + if table[u"algorithm"] == u"table_weekly_comparison": + table[u"testbeds"] = spec.environment.get(u"testbeds", None) generator[table[u"algorithm"]](table, data) except NameError as err: logging.error( @@ -490,7 +493,8 @@ def _tpc_sort_table(table): def _tpc_generate_html_table(header, data, out_file_name, legend=u"", - footnote=u"", sort_data=True, title=u""): + footnote=u"", sort_data=True, title=u"", + generate_rst=True): """Generate html table from input data with simple sorting possibility. :param header: Table header. @@ -504,6 +508,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", :param footnote: The footnote to display below the table (and legend). :param sort_data: If True the data sorting is enabled. :param title: The table (and file) title. + :param generate_rst: If True, wrapping rst file is generated. :type header: list :type data: list of lists :type out_file_name: str @@ -511,6 +516,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", :type footnote: str :type sort_data: bool :type title: str + :type generate_rst: bool """ try: @@ -528,7 +534,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", [u"left", u"left", u"right"], [u"left", u"left", u"left", u"right"] ), - u"width": ([28, 9], [4, 24, 10], [4, 4, 32, 10]) + u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10]) } df_data = pd.DataFrame(data, columns=header) @@ -599,8 +605,8 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", direction=u"down", x=0.0, xanchor=u"left", - y=1.045, - yanchor=u"top", + y=1.002, + yanchor=u"bottom", active=len(menu_items) - 1, buttons=list(buttons) ) @@ -630,6 +636,9 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"", filename=f"{out_file_name}_in.html" ) + if not generate_rst: + return + file_name = out_file_name.split(u"/")[-1] if u"vpp" in out_file_name: path = u"_tmp/src/vpp_performance_tests/comparisons/" @@ -2698,3 +2707,170 @@ def table_comparison(table, input_data): sort_data=False, title=table.get(u"title", u"") ) + + +def table_weekly_comparison(table, in_data): + """Generate the table(s) with algorithm: table_weekly_comparison + specified in the specification file. + + :param table: Table to generate. + :param in_data: Data to process. + :type table: pandas.Series + :type in_data: InputData + """ + logging.info(f" Generating the table {table.get(u'title', u'')} ...") + + # Transform the data + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) + + incl_tests = table.get(u"include-tests", None) + if incl_tests not in (u"NDR", u"PDR"): + logging.error(f"Wrong tests to include specified ({incl_tests}).") + return + + nr_cols = table.get(u"nr-of-data-columns", None) + if not nr_cols or nr_cols < 2: + logging.error( + f"No columns specified for {table.get(u'title', u'')}. Skipping." + ) + return + + data = in_data.filter_data( + table, + params=[u"throughput", u"result", u"name", u"parent", u"tags"], + continue_on_error=True + ) + + header = [ + [u"Version"], + [u"Date", ], + [u"Build", ], + [u"Testbed", ] + ] + tbl_dict = dict() + idx = 0 + tb_tbl = table.get(u"testbeds", None) + for job_name, job_data in data.items(): + for build_nr, build in job_data.items(): + if idx >= nr_cols: + break + if build.empty: + continue + + tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"") + if tb_ip and tb_tbl: + testbed = tb_tbl.get(tb_ip, u"") + else: + testbed = u"" + header[2].insert(1, build_nr) + header[3].insert(1, testbed) + header[1].insert( + 1, in_data.metadata(job_name, build_nr).get(u"generated", u"") + ) + header[0].insert( + 1, in_data.metadata(job_name, build_nr).get(u"version", u"") + ) + + for tst_name, tst_data in build.items(): + tst_name_mod = \ + _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"") + if not tbl_dict.get(tst_name_mod, None): + tbl_dict[tst_name_mod] = dict( + name=tst_data[u'name'].rsplit(u'-', 1)[0], + ) + try: + tbl_dict[tst_name_mod][-idx - 1] = \ + tst_data[u"throughput"][incl_tests][u"LOWER"] + except (TypeError, IndexError, KeyError, ValueError): + pass + idx += 1 + + if idx < nr_cols: + logging.error(u"Not enough data to build the table! Skipping") + return + + cmp_dict = dict() + for idx, cmp in enumerate(table.get(u"comparisons", list())): + idx_ref = cmp.get(u"reference", None) + idx_cmp = cmp.get(u"compare", None) + if idx_ref is None or idx_cmp is None: + continue + header[0].append(f"Diff{idx + 1}") + header[1].append(header[0][idx_ref - idx - 1]) + header[2].append(u"vs") + header[3].append(header[0][idx_cmp - idx - 1]) + for tst_name, tst_data in tbl_dict.items(): + if not cmp_dict.get(tst_name, None): + cmp_dict[tst_name] = list() + ref_data = tst_data.get(idx_ref, None) + cmp_data = tst_data.get(idx_cmp, None) + if ref_data is None or cmp_data is None: + cmp_dict[tst_name].append(float('nan')) + else: + cmp_dict[tst_name].append( + relative_change(ref_data, cmp_data) + ) + + tbl_lst = list() + for tst_name, tst_data in tbl_dict.items(): + itm_lst = [tst_data[u"name"], ] + for idx in range(nr_cols): + item = tst_data.get(-idx - 1, None) + if item is None: + itm_lst.insert(1, None) + else: + itm_lst.insert(1, round(item / 1e6, 1)) + itm_lst.extend( + [ + None if itm is None else round(itm, 1) + for itm in cmp_dict[tst_name] + ] + ) + tbl_lst.append(itm_lst) + + tbl_lst.sort(key=lambda rel: rel[0], reverse=False) + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + + # Generate csv table: + csv_file = f"{table[u'output-file']}.csv" + with open(csv_file, u"wt", encoding='utf-8') as file_handler: + for hdr in header: + file_handler.write(u",".join(hdr) + u"\n") + for test in tbl_lst: + file_handler.write(u",".join( + [ + str(item).replace(u"None", u"-").replace(u"nan", u"-"). + replace(u"null", u"-") for item in test + ] + ) + u"\n") + + txt_file = f"{table[u'output-file']}.txt" + convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",") + + # Reorganize header in txt table + txt_table = list() + with open(txt_file, u"rt", encoding='utf-8') as file_handler: + for line in file_handler: + txt_table.append(line) + try: + txt_table.insert(5, txt_table.pop(2)) + with open(txt_file, u"wt", encoding='utf-8') as file_handler: + file_handler.writelines(txt_table) + except IndexError: + pass + + # Generate html table: + hdr_html = [ + u"<br>".join(row) for row in zip(*header) + ] + _tpc_generate_html_table( + hdr_html, + tbl_lst, + table[u'output-file'], + sort_data=True, + title=table.get(u"title", u""), + generate_rst=False + ) diff --git a/resources/tools/presentation/pal_utils.py b/resources/tools/presentation/pal_utils.py index 20bf9a65c4..1b9131a30e 100644 --- a/resources/tools/presentation/pal_utils.py +++ b/resources/tools/presentation/pal_utils.py @@ -320,6 +320,8 @@ def convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u","): ) txt_table.align = u"r" txt_table.align[u"Test Case"] = u"l" + txt_table.align[u"Build"] = u"l" + txt_table.align[u"Version"] = u"l" if not txt_table: return diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 1e199511bd..22b31e2556 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -198,112 +198,124 @@ csit-vpp-perf-mrr-daily-master: start: 1086 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use csit-dpdk-perf-mrr-weekly-master: start: 107 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use plot-performance-trending-vpp-3n-hsw: csit-vpp-perf-mrr-daily-master: start: 1086 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-dpdk-3n-hsw: csit-dpdk-perf-mrr-weekly-master: start: 107 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: start: 834 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use csit-dpdk-perf-mrr-weekly-master-3n-skx: start: 72 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: start: 834 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-dpdk-3n-skx: csit-dpdk-perf-mrr-weekly-master-3n-skx: start: 72 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: start: 858 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use csit-dpdk-perf-mrr-weekly-master-2n-skx: start: 78 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: start: 858 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-dpdk-2n-skx: csit-dpdk-perf-mrr-weekly-master-2n-skx: start: 78 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use plot-performance-trending-vpp-nfv-2n-skx: csit-vpp-perf-mrr-weekly-master-2n-skx: start: 50 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use plot-performance-trending-vpp-2n-clx: csit-vpp-perf-mrr-daily-master-2n-clx: start: 236 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-dpdk-2n-clx: csit-dpdk-perf-mrr-weekly-master-2n-clx: start: 23 end: "lastCompletedBuild" - max-builds: 15 # Max nr of builds to use # 3n-tsh plot-performance-trending-all-3n-tsh: csit-vpp-perf-mrr-daily-master-3n-tsh: start: 144 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-vpp-3n-tsh: csit-vpp-perf-mrr-daily-master-3n-tsh: start: 144 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-vpp-3n-dnv: csit-vpp-perf-mrr-daily-master-3n-dnv: start: 329 end: "lastCompletedBuild" - max-builds: 180 # Max nr of builds to use plot-performance-trending-vpp-2n-dnv: csit-vpp-perf-mrr-daily-master-2n-dnv: start: 335 end: "lastCompletedBuild" + table-ndrpdr-comparison-2n-skx: + csit-vpp-perf-ndrpdr-weekly-master-2n-skx: + start: 84 + end: "lastCompletedBuild" + max-builds: 4 + reverse: True # Must be reversed + + table-ndrpdr-comparison-3n-skx: + csit-vpp-perf-ndrpdr-weekly-master-3n-skx: + start: 84 + end: "lastCompletedBuild" + max-builds: 4 + reverse: True # Must be reversed + skip: + - 87 + + table-ndrpdr-comparison-2n-clx: + csit-vpp-perf-ndrpdr-weekly-master-2n-clx: + start: 30 + end: "lastCompletedBuild" + max-builds: 4 + reverse: True # Must be reversed + + table-ndrpdr-comparison-3n-hsw: + csit-vpp-perf-ndrpdr-weekly-master-3n-hsw: + start: 84 + end: "lastCompletedBuild" + max-builds: 4 + reverse: True # Must be reversed + plot-layouts: plot-cpta: @@ -398,6 +410,32 @@ builds: + # 2n-skx ndrpdr + csit-vpp-perf-ndrpdr-weekly-master-2n-skx: + start: 84 + end: "lastCompletedBuild" + max-builds: 4 + + # 3n-skx ndrpdr + csit-vpp-perf-ndrpdr-weekly-master-3n-skx: + start: 84 + end: "lastCompletedBuild" + max-builds: 4 + skip: + - 87 + + # 2n-clx ndrpdr + csit-vpp-perf-ndrpdr-weekly-master-2n-clx: + start: 30 + end: "lastCompletedBuild" + max-builds: 4 + + # 3n-hsw ndrpdr + csit-vpp-perf-ndrpdr-weekly-master-3n-hsw: + start: 84 + end: "lastCompletedBuild" + max-builds: 4 + # 3n-hsw csit-vpp-perf-mrr-daily-master: start: 1086 @@ -548,6 +586,126 @@ ################################################################################ +# VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c ndr +- type: "table" + title: "VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c ndr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-2n-skx" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c pdr +- type: "table" + title: "VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c pdr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-skx-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-2n-skx" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c ndr +- type: "table" + title: "VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c ndr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-3n-skx" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c pdr +- type: "table" + title: "VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c pdr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-skx-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-3n-skx" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c ndr +- type: "table" + title: "VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c ndr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-clx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-2n-clx" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c pdr +- type: "table" + title: "VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c pdr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-clx-2t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-2n-clx" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 3n-hsw-xl710 2t1c ndr +- type: "table" + title: "VPP NDRPDR weekly comparison 3n-hsw-xl710 1t1c ndr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-hsw-1t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-3n-hsw" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + +# VPP NDRPDR weekly comparison 3n-hsw-xl710 2t1c pdr +- type: "table" + title: "VPP NDRPDR weekly comparison 3n-hsw-xl710 1t1c pdr" + algorithm: "table_weekly_comparison" + output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-hsw-1t1c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + data: "table-ndrpdr-comparison-3n-hsw" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + nr-of-data-columns: 3 + comparisons: + - reference: -3 + compare: -1 + - reference: -2 + compare: -1 + # Compressed failed tests (last build) - type: "table" diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index 548bbff532..302ce037ab 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -529,6 +529,7 @@ class Specification: if isinstance(builds, dict): build_end = builds.get(u"end", None) max_builds = builds.get(u"max-builds", None) + reverse = builds.get(u"reverse", False) try: build_end = int(build_end) except ValueError: @@ -536,7 +537,9 @@ class Specification: build_end = self._get_build_number(job, build_end) builds = [x for x in range(builds[u"start"], build_end + 1)] if max_builds and max_builds < len(builds): - builds = builds[:max_builds] + builds = builds[-max_builds:] + if reverse: + builds.reverse() self.configuration[u"data-sets"][set_name][job] = builds elif isinstance(builds, list): for idx, item in enumerate(builds): |