diff options
Diffstat (limited to 'resources')
| -rw-r--r-- | resources/tools/presentation/generator_cpta.py | 28 | ||||
| -rw-r--r-- | resources/tools/presentation/generator_tables.py | 32 | ||||
| -rw-r--r-- | resources/tools/presentation/pal_utils.py | 9 | ||||
| -rw-r--r-- | resources/tools/presentation/specification_CPTA.yaml | 285 |
4 files changed, 266 insertions, 88 deletions
diff --git a/resources/tools/presentation/generator_cpta.py b/resources/tools/presentation/generator_cpta.py index ac0a5c60e1..511800a988 100644 --- a/resources/tools/presentation/generator_cpta.py +++ b/resources/tools/presentation/generator_cpta.py @@ -169,8 +169,13 @@ def _generate_trending_traces(in_data, job_name, build_info, """ data_x = list(in_data.keys()) - data_y_pps = list(in_data.values()) - data_y_mpps = [float(item) / 1e6 for item in data_y_pps] + data_y_pps = list() + data_y_mpps = list() + data_y_stdev = list() + for item in in_data.values(): + data_y_pps.append(float(item[u"receive-rate"])) + data_y_stdev.append(float(item[u"receive-stdev"]) / 1e6) + data_y_mpps.append(float(item[u"receive-rate"]) / 1e6) hover_text = list() xaxis = list() @@ -179,6 +184,7 @@ def _generate_trending_traces(in_data, job_name, build_info, date = build_info[job_name][str_key][0] hover_str = (u"date: {date}<br>" u"value [Mpps]: {value:.3f}<br>" + u"stdev [Mpps]: {stdev:.3f}<br>" u"{sut}-ref: {build}<br>" u"csit-ref: mrr-{period}-build-{build_nr}<br>" u"testbed: {testbed}") @@ -186,6 +192,7 @@ def _generate_trending_traces(in_data, job_name, build_info, hover_text.append(hover_str.format( date=date, value=data_y_mpps[index], + stdev=data_y_stdev[index], sut=u"dpdk", build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0], period=u"weekly", @@ -195,6 +202,7 @@ def _generate_trending_traces(in_data, job_name, build_info, hover_text.append(hover_str.format( date=date, value=data_y_mpps[index], + stdev=data_y_stdev[index], sut=u"vpp", build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0], period=u"daily", @@ -221,8 +229,7 @@ def _generate_trending_traces(in_data, job_name, build_info, } if anomaly_classification: for index, (key, value) in enumerate(data_pd.items()): - if anomaly_classification[index] in \ - (u"outlier", u"regression", u"progression"): + if anomaly_classification[index] in (u"regression", u"progression"): anomalies[key] = value / 1e6 anomalies_colors.append( anomaly_color[anomaly_classification[index]]) @@ -377,8 +384,10 @@ def _generate_all_charts(spec, input_data): if chart_data.get(test_name, None) is None: chart_data[test_name] = OrderedDict() try: - chart_data[test_name][int(index)] = \ - test[u"result"][u"receive-rate"] + chart_data[test_name][int(index)] = { + u"receive-rate": test[u"result"][u"receive-rate"], + u"receive-stdev": test[u"result"][u"receive-stdev"] + } chart_tags[test_name] = test.get(u"tags", None) except (KeyError, TypeError): pass @@ -387,9 +396,12 @@ def _generate_all_charts(spec, input_data): for tst_name, tst_data in chart_data.items(): tst_lst = list() for bld in builds_dict[job_name]: - itm = tst_data.get(int(bld), u'') + itm = tst_data.get(int(bld), dict()) # CSIT-1180: Itm will be list, compute stats. - tst_lst.append(str(itm)) + try: + tst_lst.append(str(itm.get(u"receive-rate", u""))) + except AttributeError: + tst_lst.append(u"") csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n') # Generate traces: diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 977ba8d669..08c9d55305 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -829,6 +829,8 @@ def table_perf_trending_dash(table, input_data): ] header_str = u",".join(header) + u"\n" + incl_tests = table.get(u"include-tests", u"MRR") + # Prepare data to the table: tbl_dict = dict() for job, builds in table[u"data"].items(): @@ -846,8 +848,15 @@ def table_perf_trending_dash(table, input_data): u"data": OrderedDict() } try: - tbl_dict[tst_name][u"data"][str(build)] = \ - tst_data[u"result"][u"receive-rate"] + if incl_tests == u"MRR": + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"result"][u"receive-rate"] + elif incl_tests == u"NDR": + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"throughput"][u"NDR"][u"LOWER"] + elif incl_tests == u"PDR": + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"throughput"][u"PDR"][u"LOWER"] except (TypeError, KeyError): pass # No data in output.xml for this test @@ -1798,10 +1807,10 @@ def table_weekly_comparison(table, in_data): ) header = [ - [u"Version"], - [u"Date", ], - [u"Build", ], - [u"Testbed", ] + [u"VPP Version", ], + [u"Start Timestamp", ], + [u"CSIT Build", ], + [u"CSIT Testbed", ] ] tbl_dict = dict() idx = 0 @@ -1851,10 +1860,13 @@ def table_weekly_comparison(table, in_data): idx_cmp = cmp.get(u"compare", None) if idx_ref is None or idx_cmp is None: continue - header[0].append(f"Diff{idx + 1}") - header[1].append(header[0][idx_ref - idx - 1]) - header[2].append(u"vs") - header[3].append(header[0][idx_cmp - idx - 1]) + header[0].append( + f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs " + f"{header[0][idx_cmp - idx].split(u'~')[-1]})" + ) + header[1].append(u"") + header[2].append(u"") + header[3].append(u"") for tst_name, tst_data in tbl_dict.items(): if not cmp_dict.get(tst_name, None): cmp_dict[tst_name] = list() diff --git a/resources/tools/presentation/pal_utils.py b/resources/tools/presentation/pal_utils.py index 1b9131a30e..98d5837989 100644 --- a/resources/tools/presentation/pal_utils.py +++ b/resources/tools/presentation/pal_utils.py @@ -318,14 +318,13 @@ def convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u","): txt_table.add_row( [str(itm.replace(u"\u00B1", u"+-")) for itm in row] ) - txt_table.align = u"r" - txt_table.align[u"Test Case"] = u"l" - txt_table.align[u"Build"] = u"l" - txt_table.align[u"Version"] = u"l" - if not txt_table: return + txt_table.align = u"r" + for itm in (u"Test Case", u"Build", u"Version", u"VPP Version"): + txt_table.align[itm] = u"l" + if txt_file_name.endswith(u".txt"): with open(txt_file_name, u"wt", encoding='utf-8') as txt_file: txt_file.write(str(txt_table)) diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 22b31e2556..6f8dee1bef 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -288,32 +288,32 @@ table-ndrpdr-comparison-2n-skx: csit-vpp-perf-ndrpdr-weekly-master-2n-skx: - start: 84 + start: 76 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 reverse: True # Must be reversed table-ndrpdr-comparison-3n-skx: csit-vpp-perf-ndrpdr-weekly-master-3n-skx: - start: 84 + start: 76 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 reverse: True # Must be reversed skip: - 87 table-ndrpdr-comparison-2n-clx: csit-vpp-perf-ndrpdr-weekly-master-2n-clx: - start: 30 + start: 21 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 reverse: True # Must be reversed table-ndrpdr-comparison-3n-hsw: csit-vpp-perf-ndrpdr-weekly-master-3n-hsw: - start: 84 + start: 76 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 reverse: True # Must be reversed plot-layouts: @@ -412,29 +412,29 @@ # 2n-skx ndrpdr csit-vpp-perf-ndrpdr-weekly-master-2n-skx: - start: 84 + start: 76 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 # 3n-skx ndrpdr csit-vpp-perf-ndrpdr-weekly-master-3n-skx: - start: 84 + start: 76 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 skip: - 87 # 2n-clx ndrpdr csit-vpp-perf-ndrpdr-weekly-master-2n-clx: - start: 30 + start: 21 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 # 3n-hsw ndrpdr csit-vpp-perf-ndrpdr-weekly-master-3n-hsw: - start: 84 + start: 76 end: "lastCompletedBuild" - max-builds: 4 + max-builds: 15 # 3n-hsw csit-vpp-perf-mrr-daily-master: @@ -443,6 +443,7 @@ max-builds: 180 # Max nr of builds to download skip: - 1102 + - 1159 csit-dpdk-perf-mrr-weekly-master: start: 113 end: "lastCompletedBuild" @@ -462,6 +463,7 @@ - 864 - 865 - 866 + - 867 csit-dpdk-perf-mrr-weekly-master-3n-skx: start: 78 end: "lastCompletedBuild" @@ -478,6 +480,7 @@ - 868 - 870 - 874 + - 889 csit-vpp-perf-mrr-weekly-master-2n-skx: start: 50 end: "lastCompletedBuild" @@ -503,6 +506,7 @@ - 251 - 252 - 253 + - 308 csit-dpdk-perf-mrr-weekly-master-2n-clx: start: 23 @@ -525,6 +529,7 @@ - 155 - 156 - 157 + - 192 # 3n-dnv csit-vpp-perf-mrr-daily-master-3n-dnv: @@ -549,6 +554,7 @@ - 390 - 391 - 392 + - 396 # 2n-dnv csit-vpp-perf-mrr-daily-master-2n-dnv: @@ -561,7 +567,7 @@ - 350 - 351 - 354 - - 360 + - 36 - 364 - 365 - 366 @@ -572,6 +578,7 @@ - 395 - 396 - 397 + - 402 - type: "output" @@ -811,10 +818,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-1t1c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -830,10 +833,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-2t2c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -849,10 +848,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-4t4c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -880,6 +875,202 @@ output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-4t4c-3n-hsw.rst" testbed: "3n-hsw" +# 3n-hsw +- + type: "table" + title: "Performance trending dashboard 1t1c-3n-hsw ndr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-1t1c-3n-hsw-ndr" + data: "table-ndrpdr-comparison-3n-hsw" + filter: "'NDRPDR' and '1T1C'" + include-tests: "NDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 4 + long-trend-window: 12 + +- + type: "table" + title: "Performance trending dashboard 1t1c-3n-hsw pdr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-1t1c-3n-hsw-pdr" + data: "table-ndrpdr-comparison-3n-hsw" + filter: "'NDRPDR' and '1T1C'" + include-tests: "PDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 4 + long-trend-window: 12 + +- + type: "table" + title: "HTML performance trending dashboard 1t1c-3n-hsw ndr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-1t1c-3n-hsw-ndr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-1t1c-3n-hsw-ndr.rst" + testbed: "3n-hsw" + +- + type: "table" + title: "HTML performance trending dashboard 1t1c-3n-hsw pdr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-1t1c-3n-hsw-pdr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-1t1c-3n-hsw-pdr.rst" + testbed: "3n-hsw" + +# 3n-skx +- + type: "table" + title: "Performance trending dashboard 2t1c-3n-skx ndr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-skx-ndr" + data: "table-ndrpdr-comparison-3n-skx" + filter: "'NDRPDR' and '2T1C'" + include-tests: "NDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 4 + long-trend-window: 12 + +- + type: "table" + title: "Performance trending dashboard 2t1c-3n-skx pdr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-skx-pdr" + data: "table-ndrpdr-comparison-3n-skx" + filter: "'NDRPDR' and '2T1C'" + include-tests: "PDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 4 + long-trend-window: 12 + +- + type: "table" + title: "HTML performance trending dashboard 2t1c-3n-skx ndr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-skx-ndr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-skx-ndr.rst" + testbed: "3n-skx" + +- + type: "table" + title: "HTML performance trending dashboard 2t1c-3n-skx pdr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-skx-pdr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-3n-skx-pdr.rst" + testbed: "3n-skx" + +# 2n-skx +- + type: "table" + title: "Performance trending dashboard 2t1c-2n-skx ndr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-skx-ndr" + data: "table-ndrpdr-comparison-2n-skx" + filter: "'NDRPDR' and '2T1C'" + include-tests: "NDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 4 + long-trend-window: 12 + +- + type: "table" + title: "Performance trending dashboard 2t1c-2n-skx pdr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-skx-pdr" + data: "table-ndrpdr-comparison-2n-skx" + filter: "'NDRPDR' and '2T1C'" + include-tests: "PDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 4 + long-trend-window: 12 + +- + type: "table" + title: "HTML performance trending dashboard 2t1c-2n-skx ndr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-skx-ndr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-skx-ndr.rst" + testbed: "2n-skx" + +- + type: "table" + title: "HTML performance trending dashboard 2t1c-2n-skx pdr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-skx-pdr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-skx-pdr.rst" + testbed: "2n-skx" + +# 2n-clx +- + type: "table" + title: "Performance trending dashboard 2t1c-2n-clx ndr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-clx-ndr" + data: "table-ndrpdr-comparison-2n-clx" + filter: "'NDRPDR' and '2T1C'" + include-tests: "NDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 14 + long-trend-window: 180 + +- + type: "table" + title: "Performance trending dashboard 2t1c-2n-clx pdr" + algorithm: "table_perf_trending_dash" + output-file-ext: ".csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-clx-pdr" + data: "table-ndrpdr-comparison-2n-clx" + filter: "'NDRPDR' and '2T1C'" + include-tests: "PDR" + parameters: + - "name" + - "parent" + - "throughput" + window: 14 + long-trend-window: 180 + +- + type: "table" + title: "HTML performance trending dashboard 2t1c-2n-clx ndr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-clx-ndr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-clx-ndr.rst" + testbed: "2n-clx" + +- + type: "table" + title: "HTML performance trending dashboard 2t1c-2n-clx pdr" + algorithm: "table_perf_trending_dash_html" + input-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-clx-pdr.csv" + output-file: "{DIR[STATIC,VPP]}/performance-trending-dashboard-2t1c-2n-clx-pdr.rst" + testbed: "2n-clx" + - type: "table" title: "Failed MRR tests 3n-hsw" @@ -892,10 +1083,6 @@ - "name" - "parent" - "status" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-1t1c-eth-l2bdscale1mmaclrn-ndrdisc" window: 7 # days - @@ -920,10 +1107,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -939,10 +1122,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -958,10 +1137,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -1001,10 +1176,6 @@ - "name" - "parent" - "status" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc" window: 7 # days - @@ -1029,10 +1200,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -1048,10 +1215,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -1067,10 +1230,6 @@ - "name" - "parent" - "result" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc" window: 14 long-trend-window: 180 @@ -1110,10 +1269,6 @@ - "name" - "parent" - "status" - ignore-list: - # This test is "ndrdisc" test and was improperly tagged. It was fixed - # but it remains in the old output.xml files. - - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc" window: 7 # days - |