diff options
| -rw-r--r-- | resources/tools/presentation/generator_tables.py | 235 | ||||
| -rw-r--r-- | resources/tools/presentation/specification.yaml | 106 |
2 files changed, 304 insertions, 37 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 5c4452d5a5..f04495c4a1 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -415,6 +415,241 @@ def table_performance_comparison(table, input_data): convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) +def table_performance_comparison_nic(table, input_data): + """Generate the table(s) with algorithm: table_performance_comparison + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + # Prepare the header of the tables + try: + header = ["Test case", ] + + if table["include-tests"] == "MRR": + hdr_param = "Receive Rate" + else: + hdr_param = "Throughput" + + history = table.get("history", None) + if history: + for item in history: + header.extend( + ["{0} {1} [Mpps]".format(item["title"], hdr_param), + "{0} Stdev [Mpps]".format(item["title"])]) + header.extend( + ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Delta [%]"]) + header_str = ",".join(header) + "\n" + except (AttributeError, KeyError) as err: + logging.error("The model is invalid, missing parameter: {0}". + format(err)) + return + + # Prepare data to the table: + tbl_dict = dict() + for job, builds in table["reference"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if table["reference"]["nic"] not in tst_data["tags"]: + continue + tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ + replace("-ndrpdr", "").replace("-pdrdisc", "").\ + replace("-ndrdisc", "").replace("-pdr", "").\ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + name = "{0}".format("-".join(tst_data["name"]. + split("-")[:-1])) + if "across testbeds" in table["title"].lower() or \ + "across topologies" in table["title"].lower(): + name = name.\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tbl_dict[tst_name_mod] = {"name": name, + "ref-data": list(), + "cmp-data": list()} + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["result"]["receive-rate"].avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["ref-data"].append( + tst_data["throughput"]["PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["ref-data"].append( + tst_data["throughput"]["NDR"]["LOWER"]) + else: + continue + except TypeError: + pass # No data in output.xml for this test + + for job, builds in table["compare"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if table["compare"]["nic"] not in tst_data["tags"]: + continue + tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ + replace("-ndrpdr", "").replace("-pdrdisc", ""). \ + replace("-ndrdisc", "").replace("-pdr", ""). \ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["result"]["receive-rate"].avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["NDR"]["LOWER"]) + else: + continue + except KeyError: + pass + except TypeError: + tbl_dict.pop(tst_name_mod, None) + if history: + for item in history: + for job, builds in item["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if item["nic"] not in tst_data["tags"]: + continue + tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ + replace("-ndrpdr", "").replace("-pdrdisc", ""). \ + replace("-ndrdisc", "").replace("-pdr", ""). \ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + continue + if tbl_dict[tst_name_mod].get("history", None) is None: + tbl_dict[tst_name_mod]["history"] = OrderedDict() + if tbl_dict[tst_name_mod]["history"].get(item["title"], + None) is None: + tbl_dict[tst_name_mod]["history"][item["title"]] = \ + list() + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["history"][item["title" + ]].append(tst_data["result"]["receive-rate"]. + avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["history"][ + item["title"]].\ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["history"][item[ + "title"]].append(tst_data["throughput"][ + "PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["history"][ + item["title"]].\ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["history"][item[ + "title"]].append(tst_data["throughput"][ + "NDR"]["LOWER"]) + else: + continue + except (TypeError, KeyError): + pass + + tbl_lst = list() + for tst_name in tbl_dict.keys(): + item = [tbl_dict[tst_name]["name"], ] + if history: + if tbl_dict[tst_name].get("history", None) is not None: + for hist_data in tbl_dict[tst_name]["history"].values(): + if hist_data: + item.append(round(mean(hist_data) / 1000000, 2)) + item.append(round(stdev(hist_data) / 1000000, 2)) + else: + item.extend([None, None]) + else: + item.extend([None, None]) + data_t = tbl_dict[tst_name]["ref-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + data_t = tbl_dict[tst_name]["cmp-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + if item[-4] is not None and item[-2] is not None and item[-4] != 0: + item.append(int(relative_change(float(item[-4]), float(item[-2])))) + if len(item) == len(header): + tbl_lst.append(item) + + # Sort the table according to the relative change + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + + # Generate csv tables: + csv_file = "{0}.csv".format(table["output-file"]) + with open(csv_file, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + file_handler.write(",".join([str(item) for item in test]) + "\n") + + convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) + + def table_nics_comparison(table, input_data): """Generate the table(s) with algorithm: table_nics_comparison specified in the specification file. diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 4fc722eb66..1e3d60e77f 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -1520,8 +1520,8 @@ format: html: - full - pdf: - - minimal +# pdf: +# - minimal ################################################################################ ### T A B L E S ### @@ -2076,17 +2076,20 @@ # VPP Performance Changes 3n-skx 2t1c pdr - type: "table" title: "VPP Performance Changes 3n-skx 2t1c pdr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-3n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-3n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-3n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-3n-skx" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2094,21 +2097,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 3n-skx 4t2c pdr - type: "table" title: "VPP Performance Changes 3n-skx 4t2c pdr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-3n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-3n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-3n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-3n-skx" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2116,21 +2123,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 3n-skx 2t1c ndr - type: "table" title: "VPP Performance Changes 3n-skx 2t1c ndr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-3n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-3n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-3n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-3n-skx" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2138,21 +2149,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 3n-skx 4t2c ndr - type: "table" title: "VPP Performance Changes 3n-skx 4t2c ndr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-3n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-3n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-3n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-3n-skx" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2160,21 +2175,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 2n-skx 2t1c pdr - type: "table" title: "VPP Performance Changes 2n-skx 2t1c pdr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr" history: - - title: "rls1901" - data: "vpp-performance-changes-2n-skx-h1" + - title: "rls1901 Intel-X710" + data: "vpp-performance-changes-3n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" - data: "vpp-performance-changes-2n-skx-ref" + title: "rls1904 Intel-X710" + data: "vpp-performance-changes-3n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" - data: "vpp-performance-changes-2n-skx-cmp" + title: "rls1908 Intel-XXV710" + data: "vpp-performance-changes-3n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-2n-skx" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2182,21 +2201,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 2n-skx 4t2c pdr - type: "table" title: "VPP Performance Changes 2n-skx 4t2c pdr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-2n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-2n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-2n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-2n-skx" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2204,21 +2227,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 2n-skx 2t1c ndr - type: "table" title: "VPP Performance Changes 2n-skx 2t1c ndr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-2n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-2n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-2n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-2n-skx" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2226,21 +2253,25 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Performance Changes 2n-skx 4t2c ndr - type: "table" title: "VPP Performance Changes 2n-skx 4t2c ndr" - algorithm: "table_performance_comparison" + algorithm: "table_performance_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr" history: - - title: "rls1901" + - title: "rls1901 Intel-X710" data: "vpp-performance-changes-2n-skx-h1" + nic: "NIC_Intel-X710" reference: - title: "rls1904" + title: "rls1904 Intel-X710" data: "vpp-performance-changes-2n-skx-ref" + nic: "NIC_Intel-X710" compare: - title: "rls1908" + title: "rls1908 Intel-XXV710" data: "vpp-performance-changes-2n-skx-cmp" + nic: "NIC_Intel-XXV710" data: "vpp-performance-changes-2n-skx" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')" @@ -2248,6 +2279,7 @@ - "name" - "parent" - "throughput" + - "tags" # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr - type: "table" |