diff options
Diffstat (limited to 'resources')
4 files changed, 571 insertions, 3 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 8b88a52a79..1e1307b5bc 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -29,6 +29,7 @@ from copy import deepcopy import plotly.graph_objects as go import plotly.offline as ploff import pandas as pd +import prettytable from numpy import nan, isnan from yaml import load, FullLoader, YAMLError @@ -59,7 +60,8 @@ def generate_tables(spec, data): u"table_failed_tests_html": table_failed_tests_html, u"table_oper_data_html": table_oper_data_html, u"table_comparison": table_comparison, - u"table_weekly_comparison": table_weekly_comparison + u"table_weekly_comparison": table_weekly_comparison, + u"table_job_spec_duration": table_job_spec_duration } logging.info(u"Generating the tables ...") @@ -76,6 +78,96 @@ def generate_tables(spec, data): logging.info(u"Done.") +def table_job_spec_duration(table, input_data): + """Generate the table(s) with algorithm: table_job_spec_duration + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + _ = input_data + + logging.info(f" Generating the table {table.get(u'title', u'')} ...") + + jb_type = table.get(u"jb-type", None) + + tbl_lst = list() + if jb_type == u"iterative": + for line in table.get(u"lines", tuple()): + tbl_itm = { + u"name": line.get(u"job-spec", u""), + u"data": list() + } + for job, builds in line.get(u"data-set", dict()).items(): + for build_nr in builds: + try: + minutes = input_data.metadata( + job, str(build_nr) + )[u"elapsedtime"] // 60000 + except (KeyError, IndexError, ValueError, AttributeError): + continue + tbl_itm[u"data"].append(minutes) + tbl_itm[u"mean"] = mean(tbl_itm[u"data"]) + tbl_itm[u"stdev"] = stdev(tbl_itm[u"data"]) + tbl_lst.append(tbl_itm) + elif jb_type == u"coverage": + job = table.get(u"data", None) + if not job: + return + for line in table.get(u"lines", tuple()): + try: + tbl_itm = { + u"name": line.get(u"job-spec", u""), + u"mean": input_data.metadata( + list(job.keys())[0], str(line[u"build"]) + )[u"elapsedtime"] // 60000, + u"stdev": float(u"nan") + } + tbl_itm[u"data"] = [tbl_itm[u"mean"], ] + except (KeyError, IndexError, ValueError, AttributeError): + continue + tbl_lst.append(tbl_itm) + else: + logging.warning(f"Wrong type of job-spec: {jb_type}. Skipping.") + return + + for line in tbl_lst: + line[u"mean"] = \ + f"{int(line[u'mean'] // 60):02d}:{int(line[u'mean'] % 60):02d}" + if math.isnan(line[u"stdev"]): + line[u"stdev"] = u"" + else: + line[u"stdev"] = \ + f"{int(line[u'stdev'] //60):02d}:{int(line[u'stdev'] % 60):02d}" + + if not tbl_lst: + return + + rows = list() + for itm in tbl_lst: + rows.append([ + itm[u"name"], + f"{len(itm[u'data'])}", + f"{itm[u'mean']} +- {itm[u'stdev']}" + if itm[u"stdev"] != u"" else f"{itm[u'mean']}" + ]) + + txt_table = prettytable.PrettyTable( + [u"Job Specification", u"Nr of Runs", u"Duration [HH:MM]"] + ) + for row in rows: + txt_table.add_row(row) + txt_table.align = u"r" + txt_table.align[u"Job Specification"] = u"l" + + file_name = f"{table.get(u'output-file', u'')}.txt" + with open(file_name, u"wt", encoding='utf-8') as txt_file: + txt_file.write(str(txt_table)) + + def table_oper_data_html(table, input_data): """Generate the table(s) with algorithm: html_table_oper_data specified in the specification file. diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index a94d09f3fa..00614cf62f 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -666,9 +666,16 @@ class Specification: table[u"columns"][i][u"data-replacement"] = \ self.data_sets[data_set] + if table.get(u"lines", None): + for i in range(len(table[u"lines"])): + data_set = table[u"lines"][i].get(u"data-set", None) + if isinstance(data_set, str): + table[u"lines"][i][u"data-set"] = \ + self.data_sets[data_set] + except KeyError: raise PresentationError( - f"Wrong data set used in {table.get(u'title', u'')}." + f"Wrong set '{data_set}' used in {table.get(u'title', u'')}." ) self._specification[u"tables"].append(table) diff --git a/resources/tools/presentation/specifications/report/data_sets.yaml b/resources/tools/presentation/specifications/report/data_sets.yaml index e0e62c2b0a..31344dbc02 100644 --- a/resources/tools/presentation/specifications/report/data_sets.yaml +++ b/resources/tools/presentation/specifications/report/data_sets.yaml @@ -283,6 +283,11 @@ - 52 # rls2106.rel MRR iter env 7 - 88 # rls2106.rel MRR AF-XDP iter env 7 + # TODO: Remove + vpp-2n-skx-curr-iter-mrr-single: + csit-vpp-perf-report-iterative-2106-2n-skx: + - 52 # rls2106.rel MRR iter env 7 + vpp-2n-skx-curr-soak: csit-vpp-perf-report-iterative-2106-2n-skx: - 58 # rls2106.rel soak env 7 @@ -511,6 +516,11 @@ - 36 # rls2106.rel MRR iter env 7 - 48 # rls2106.rel MRR AF-XDP iter env 7 + # TODO: Remove + vpp-2n-zn2-curr-iter-mrr-single: + csit-vpp-perf-report-iterative-2106-2n-zn2: + - 36 # rls2106.rel MRR iter env 7 + vpp-2n-zn2-curr-cov: csit-vpp-perf-report-coverage-2106-2n-zn2: - 1 # rls2106.rel NDRPDR COV env 7 @@ -694,11 +704,16 @@ vpp-2n-clx-prev-iter-mrr-new-env: csit-vpp-perf-report-iterative-2106-2n-clx: - 8 # rls2101.rel MRR iter env 7 - - 123 # rls2101.rel MRR AF-XDP iter env 7 vpp-2n-clx-curr-iter-mrr: csit-vpp-perf-report-iterative-2106-2n-clx: - 66 # rls2106.rel MRR iter env 7 + - 123 # rls2101.rel MRR AF-XDP iter env 7 + + # TODO: Remove + vpp-2n-clx-curr-iter-mrr-single: + csit-vpp-perf-report-iterative-2106-2n-clx: + - 66 # rls2106.rel MRR iter env 7 vpp-2n-clx-curr-iter-gso: csit-vpp-perf-report-iterative-2106-2n-clx: @@ -1057,6 +1072,11 @@ - 18 # rls2106.rel MRR iter env 7 - 29 # rls2101.rel MRR AF-XDP iter env 7 + # TODO: Remove + vpp-2n-tx2-curr-iter-mrr-single: + csit-vpp-perf-report-iterative-2106-2n-tx2: + - 18 # rls2106.rel MRR iter env 7 + vpp-2n-tx2-curr-cov: csit-vpp-perf-report-coverage-2106-2n-tx2: - 1 # rls2106.rel NDRPDR COV env 7 diff --git a/resources/tools/presentation/specifications/report/elements.yaml b/resources/tools/presentation/specifications/report/elements.yaml index 2c3f33a99d..2396851443 100644 --- a/resources/tools/presentation/specifications/report/elements.yaml +++ b/resources/tools/presentation/specifications/report/elements.yaml @@ -2,6 +2,455 @@ ### T A B L E S ### ################################################################################ +# Statistics: job-specs' durations +- type: "table" + title: "Job Specification Duration: 2n-clx-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-clx-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-2n-clx-curr-iter-mrr-single" + - job-spec: "vpp-mlr-00" + data-set: "vpp-2n-clx-curr-iter-best" + - job-spec: "vpp-gso-mrr-00" + data-set: "vpp-2n-clx-curr-iter-gso" + - job-spec: "soak-00" + data-set: "vpp-2n-clx-curr-soak" + - job-spec: "vsap-00" + data-set: "vpp-2n-clx-curr-iter-vsap" + - job-spec: "nfv-density-mrr-00" + data-set: "vpp-2n-clx-curr-nfv-mrr" + - job-spec: "nfv-density-mlr-00" + data-set: "vpp-2n-clx-curr-nfv-ndrpdr" + - job-spec: "reconf-00" + data-set: "vpp-2n-clx-curr-nfv-reconf" + - job-spec: "dpdk-mlr-00" + data-set: "dpdk-2n-clx-curr-iter" + +- type: "table" + title: "Job Specification Duration: 2n-clx-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-clx-cov" + jb-type: "coverage" + data: "vpp-2n-clx-curr-cov" + lines: + - job-spec: "gso-00" + build: "1" + - job-spec: "ip4-00" + build: "2" + - job-spec: "ip4-01" + build: "3" + - job-spec: "ip4-02" + build: "4" + - job-spec: "ip4-03" + build: "5" + - job-spec: "ip4-04" + build: "6" + - job-spec: "ip4-05" + build: "7" + - job-spec: "ip4-06" + build: "8" + - job-spec: "ip4-07" + build: "9" + - job-spec: "ip4tun-00" + build: "10" + - job-spec: "ip6-00" + build: "11" + - job-spec: "ip6-01" + build: "12" + - job-spec: "l2-00" + build: "13" + - job-spec: "l2-01" + build: "14" + - job-spec: "lb-00" + build: "15" + - job-spec: "memif-00" + build: "16" + - job-spec: "vhost-00" + build: "17" + +- type: "table" + title: "Job Specification Duration: 2n-dnv-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-dnv-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-2n-dnv-curr-iter-mrr" + - job-spec: "vpp-mlr-00" + data-set: "vpp-2n-dnv-curr-iter" + +- type: "table" + title: "Job Specification Duration: 2n-dnv-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-dnv-cov" + jb-type: "coverage" + data: "vpp-2n-dnv-curr-cov" + lines: + - job-spec: "ip4-00" + build: "7" + - job-spec: "ip4-01" + build: "8" + - job-spec: "ip6-00" + build: "9" + - job-spec: "ip6-01" + build: "10" + - job-spec: "l2-00" + build: "11" + - job-spec: "l2-01" + build: "13" + +- type: "table" + title: "Job Specification Duration: 2n-skx-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-skx-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-2n-skx-curr-iter-mrr-single" + - job-spec: "vpp-mlr-00" + data-set: "vpp-2n-skx-curr-iter-best" + - job-spec: "vpp-gso-mrr-00" + data-set: "vpp-2n-skx-curr-gso" + - job-spec: "soak-00" + data-set: "vpp-2n-skx-curr-soak" + - job-spec: "nfv-density-mrr-00" + data-set: "vpp-2n-skx-curr-nfv-mrr" + - job-spec: "nfv-density-mlr-00" + data-set: "vpp-2n-skx-curr-nfv-ndrpdr" + - job-spec: "reconf-00" + data-set: "vpp-2n-skx-curr-nfv-reconf" + - job-spec: "dpdk-mlr-00" + data-set: "dpdk-2n-skx-curr-iter" + +- type: "table" + title: "Job Specification Duration: 2n-skx-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-skx-cov" + jb-type: "coverage" + data: "vpp-2n-skx-curr-cov" + lines: + - job-spec: "gso-00" + build: "1" + - job-spec: "ip4-00" + build: "2" + - job-spec: "ip4-01" + build: "3" + - job-spec: "ip4-02" + build: "4" + - job-spec: "ip4-03" + build: "5" + - job-spec: "ip4-04" + build: "6" + - job-spec: "ip4-05" + build: "7" + - job-spec: "ip4-06" + build: "8" + - job-spec: "ip4-07" + build: "9" + - job-spec: "ip4tun-00" + build: "10" + - job-spec: "ip6-00" + build: "11" + - job-spec: "ip6-01" + build: "12" + - job-spec: "l2-00" + build: "13" + - job-spec: "l2-01" + build: "14" + - job-spec: "lb-00" + build: "15" + - job-spec: "memif-00" + build: "16" + - job-spec: "vhost-00" + build: "17" + +- type: "table" + title: "Job Specification Duration: 2n-tx2-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-tx2-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-2n-tx2-curr-iter-mrr-single" + - job-spec: "vpp-mlr-00" + data-set: "vpp-2n-tx2-curr-iter" + +- type: "table" + title: "Job Specification Duration: 2n-tx2-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-tx2-cov" + jb-type: "coverage" + data: "vpp-2n-tx2-curr-cov" + lines: + - job-spec: "ip4-00" + build: "1" + - job-spec: "ip4-01" + build: "2" + - job-spec: "ip4-02" + build: "3" + - job-spec: "ip4-03" + build: "4" + - job-spec: "ip4-04" + build: "5" + - job-spec: "ip4-05" + build: "6" + - job-spec: "ip4-06" + build: "7" + - job-spec: "ip6-00" + build: "8" + - job-spec: "ip6-01" + build: "9" + - job-spec: "l2-00" + build: "10" + - job-spec: "l2-01" + build: "11" + - job-spec: "l2-02" + build: "12" + - job-spec: "l2-03" + build: "13" + - job-spec: "l2-04" + build: "14" + +- type: "table" + title: "Job Specification Duration: 2n-zn2-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-zn2-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-2n-zn2-curr-iter-mrr-single" + - job-spec: "vpp-mlr-00" + data-set: "vpp-2n-zn2-curr-iter-best" + - job-spec: "vpp-gso-mrr-00" + data-set: "vpp-2n-zn2-curr-iter-gso" + - job-spec: "dpdk-mlr-00" + data-set: "dpdk-2n-zn2-curr-iter" + +- type: "table" + title: "Job Specification Duration: 2n-zn2-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-2n-zn2-cov" + jb-type: "coverage" + data: "vpp-2n-zn2-curr-cov" + lines: + - job-spec: "gso-00" + build: "1" + - job-spec: "ip4-00" + build: "2" + - job-spec: "ip4-01" + build: "3" + - job-spec: "ip4-02" + build: "4" + - job-spec: "ip4-03" + build: "18" + - job-spec: "ip4-04" + build: "6" + - job-spec: "ip4-05" + build: "7" + - job-spec: "ip4tun-00" + build: "8" + - job-spec: "ip6-00" + build: "9" + - job-spec: "ip6-01" + build: "19" + - job-spec: "l2-00" + build: "11" + - job-spec: "l2-01" + build: "12" + - job-spec: "lb-00" + build: "13" + - job-spec: "memif-00" + build: "14" + - job-spec: "vhost-00" + build: "15" + +- type: "table" + title: "Job Specification Duration: 3n-dnv-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-3n-dnv-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-3n-dnv-curr-iter-mrr" + - job-spec: "vpp-mlr-00" + data-set: "vpp-3n-dnv-curr-iter" + +- type: "table" + title: "Job Specification Duration: 3n-dnv-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-3n-dnv-cov" + jb-type: "coverage" + data: "vpp-3n-dnv-curr-cov" + lines: + - job-spec: "crypto-00" + build: "11" + - job-spec: "crypto-01" + build: "12" + - job-spec: "crypto-02" + build: "13" + - job-spec: "ip4-00" + build: "14" + - job-spec: "ip4-01" + build: "15" + - job-spec: "ip4tun-00" + build: "16" + - job-spec: "ip6-00" + build: "17" + - job-spec: "ip6-01" + build: "10" + - job-spec: "l2-00" + build: "20" + - job-spec: "l2-01" + build: "21" + +- type: "table" + title: "Job Specification Duration: 3n-skx-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-3n-skx-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-3n-skx-curr-iter-mrr" + - job-spec: "vpp-mlr-00" + data-set: "vpp-3n-skx-curr-iter" + - job-spec: "hoststack-00" + data-set: "vpp-3n-skx-curr-iter-hoststack" + - job-spec: "dpdk-mlr-00" + data-set: "dpdk-3n-skx-curr-iter" + +- type: "table" + title: "Job Specification Duration: 3n-skx-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-3n-skx-cov" + jb-type: "coverage" + data: "vpp-3n-skx-curr-cov" + lines: + - job-spec: "crypto-00" + build: "1" + - job-spec: "crypto-01" + build: "3" + - job-spec: "crypto-02" + build: "4" + - job-spec: "crypto-03" + build: "36" + - job-spec: "crypto-04" + build: "6" + - job-spec: "ip4-00" + build: "7" + - job-spec: "ip4-01" + build: "8" + - job-spec: "ip4-02" + build: "9" + - job-spec: "ip4-03" + build: "10" + - job-spec: "ip4-04" + build: "11" + - job-spec: "ip4-05" + build: "12" + - job-spec: "ip4-06" + build: "13" + - job-spec: "ip4-07" + build: "14" + - job-spec: "ip4tun-00" + build: "15" + - job-spec: "ip4tun-02" + build: "16" + - job-spec: "ip6-00" + build: "17" + - job-spec: "ip6-01" + build: "18" + - job-spec: "ip6tun-00" + build: "19" + - job-spec: "l2-00" + build: "20" + - job-spec: "l2-01" + build: "21" + - job-spec: "l2-02" + build: "22" + - job-spec: "l2-03" + build: "23" + - job-spec: "l2-04" + build: "24" + - job-spec: "l2-05" + build: "25" + - job-spec: "l2-06" + build: "26" + - job-spec: "l2-07" + build: "27" + - job-spec: "l2-08" + build: "28" + - job-spec: "l2-09" + build: "29" + - job-spec: "l2-10" + build: "30" + - job-spec: "memif-00" + build: "31" + - job-spec: "srv6-00" + build: "32" + - job-spec: "srv6-01" + build: "33" + - job-spec: "vhost-00" + build: "34" + - job-spec: "vhost-01" + build: "35" + +- type: "table" + title: "Job Specification Duration: 3n-tsh-iterative" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-3n-tsh-iter" + jb-type: "iterative" + lines: + - job-spec: "vpp-mrr-00" + data-set: "vpp-3n-tsh-curr-iter-mrr" + - job-spec: "vpp-mlr-00" + data-set: "vpp-3n-tsh-curr-iter" + - job-spec: "dpdk-mlr-00" + data-set: "dpdk-3n-tsh-curr-iter" + +- type: "table" + title: "Job Specification Duration: 3n-tsh-coverage" + algorithm: "table_job_spec_duration" + output-file: "{DIR[STATIC,VPP]}/job-spec-duration-3n-tsh-cov" + jb-type: "coverage" + data: "vpp-3n-tsh-curr-cov" + lines: + - job-spec: "ip4-00" + build: "7" + - job-spec: "ip4-01" + build: "8" + - job-spec: "ip4-02" + build: "9" + - job-spec: "ip4-03" + build: "10" + - job-spec: "ip4tun-00" + build: "15" + - job-spec: "ip6-00" + build: "17" + - job-spec: "ip6-01" + build: "18" + - job-spec: "l2-00" + build: "20" + - job-spec: "l2-01" + build: "21" + - job-spec: "l2-02" + build: "22" + - job-spec: "l2-03" + build: "23" + - job-spec: "l2-04" + build: "24" + - job-spec: "memif-00" + build: "31" + - job-spec: "srv6-00" + build: "32" + - job-spec: "srv6-01" + build: "33" + - job-spec: "vhost-00" + build: "34" + - job-spec: "vhost-01" + build: "35" + # VPP Latency Changes 3n-skx 2t1c - type: "table" title: "VPP Latency Changes 3n-skx 2t1c PDR50-direction1-avg" |