aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2019-09-03 14:08:37 +0200
committerTibor Frank <tifrank@cisco.com>2019-09-03 12:13:47 +0000
commit8efda89f80a75bc1ba19c8a214c2edba51540206 (patch)
treea40eae6c0e09eabe2b6f96e14280815c932e13cf
parent83e5550cfe67ba193c3ae9052258c111dbe342e4 (diff)
Report: Comparison tables
Change-Id: Ib4132c4d25f6a7c936793f403877a5819a6dd55a Signed-off-by: Tibor Frank <tifrank@cisco.com> (cherry picked from commit ac688f80adb649bd04ecbe1daa79748a439a9e19)
-rw-r--r--resources/tools/presentation/generator_tables.py84
-rw-r--r--resources/tools/presentation/specification.yaml76
2 files changed, 98 insertions, 62 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 5efe66777b..cf8ac4598c 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -210,9 +210,9 @@ def table_performance_comparison(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
history = table.get("history", None)
if history:
@@ -235,6 +235,7 @@ def table_performance_comparison(table, input_data):
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
+ topo = "2n-skx" if "2n-skx" in job else ""
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
@@ -385,6 +386,7 @@ def table_performance_comparison(table, input_data):
pass
tbl_lst = list()
+ footnote = False
for tst_name in tbl_dict.keys():
item = [tbl_dict[tst_name]["name"], ]
if history:
@@ -394,27 +396,28 @@ def table_performance_comparison(table, input_data):
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
- if "dot1q" in tbl_dict[tst_name]["name"]:
- item.append("Changed methodology")
- elif item[-4] is not None and item[-2] is not None and item[-4] != 0:
+ item.extend(["Not tested", "Not tested"])
+ if item[-4] is not None and item[-2] is not None and item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- else:
- item.append("n/a")
+ elif item[-4] is None or item[-2] is None or item[-4] == 0:
+ item.append("New in CSIT-1908")
+ elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
+ item.append("See footnote [1]")
+ footnote = True
if (len(item) == len(header)) and (item[-3] is not None):
tbl_lst.append(item)
@@ -428,7 +431,22 @@ def table_performance_comparison(table, input_data):
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ txt_file_name = "{0}.txt".format(table["output-file"])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, 'a') as txt_file:
+ txt_file.writelines([
+ "Footnotes:",
+ "[1] CSIT-1908 changed test methodology of dot1q tests in "
+ "2n-skx testbeds, dot1q encapsulation is now used on both "
+ "links of SUT.",
+ " Previously dot1q was used only on a single link with the "
+ "other link carrying untagged Ethernet frames. This change "
+ "results",
+ " in slightly lower throughput in CSIT-1908 for these "
+ "tests. See release notes."
+ ])
def table_performance_comparison_nic(table, input_data):
@@ -454,9 +472,9 @@ def table_performance_comparison_nic(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
history = table.get("history", None)
if history:
@@ -479,6 +497,7 @@ def table_performance_comparison_nic(table, input_data):
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
+ topo = "2n-skx" if "2n-skx" in job else ""
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if table["reference"]["nic"] not in tst_data["tags"]:
@@ -635,6 +654,7 @@ def table_performance_comparison_nic(table, input_data):
pass
tbl_lst = list()
+ footnote = False
for tst_name in tbl_dict.keys():
item = [tbl_dict[tst_name]["name"], ]
if history:
@@ -644,27 +664,28 @@ def table_performance_comparison_nic(table, input_data):
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
- if "dot1q" in tbl_dict[tst_name]["name"]:
- item.append("Changed methodology")
- elif item[-4] is not None and item[-2] is not None and item[-4] != 0:
+ item.extend(["Not tested", "Not tested"])
+ if item[-4] is not None and item[-2] is not None and item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- else:
- item.append("n/a")
+ elif item[-4] is None or item[-2] is None or item[-4] == 0:
+ item.append("New in CSIT-1908")
+ elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
+ item.append("See footnote [1]")
+ footnote = True
if (len(item) == len(header)) and (item[-3] is not None):
tbl_lst.append(item)
@@ -678,7 +699,22 @@ def table_performance_comparison_nic(table, input_data):
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ txt_file_name = "{0}.txt".format(table["output-file"])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, 'a') as txt_file:
+ txt_file.writelines([
+ "Footnotes:",
+ "[1] CSIT-1908 changed test methodology of dot1q tests in "
+ "2n-skx testbeds, dot1q encapsulation is now used on both "
+ "links of SUT.",
+ " Previously dot1q was used only on a single link with the "
+ "other link carrying untagged Ethernet frames. This change "
+ "results",
+ " in slightly lower throughput in CSIT-1908 for these "
+ "tests. See release notes."
+ ])
def table_nics_comparison(table, input_data):
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 00d9d4893e..fdd8cb2d98 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -2329,15 +2329,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-3n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-3n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-3n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-3n-skx"
@@ -2355,15 +2355,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-3n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-3n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-3n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-3n-skx"
@@ -2381,15 +2381,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-3n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-3n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-3n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-3n-skx"
@@ -2407,15 +2407,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-3n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-3n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-3n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-3n-skx"
@@ -2433,15 +2433,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-2n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-2n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-2n-skx"
@@ -2459,15 +2459,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-2n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-2n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-2n-skx"
@@ -2485,15 +2485,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-2n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-2n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-2n-skx"
@@ -2511,15 +2511,15 @@
algorithm: "table_performance_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr"
history:
- - title: "rls1901 Intel-X710"
+ - title: "rls1901 x710"
data: "vpp-performance-changes-2n-skx-h1"
nic: "NIC_Intel-X710"
reference:
- title: "rls1904 Intel-X710"
+ title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-ref"
nic: "NIC_Intel-X710"
compare:
- title: "rls1908 Intel-XXV710"
+ title: "rls1908 xxv710"
data: "vpp-performance-changes-2n-skx-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-performance-changes-2n-skx"
@@ -2537,11 +2537,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-1c-ndr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds"
@@ -2558,11 +2558,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-2c-ndr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds"
@@ -2579,11 +2579,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-1c-pdr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds"
@@ -2600,11 +2600,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-2c-pdr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds"
@@ -2895,11 +2895,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-1c-mrr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-mrr-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-mrr-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds-mrr"
@@ -2916,11 +2916,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-2c-mrr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-mrr-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-mrr-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds-mrr"
@@ -2937,11 +2937,11 @@
algorithm: "table_performance_comparison"
output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-3n-skx-4c-mrr"
reference:
- title: "3-Node Hsw Intel-xl710"
+ title: "3n-hsw xl710"
data: "vpp-compare-testbeds-mrr-ref"
nic: "NIC_Intel-XL710"
compare:
- title: "3-Node Skx Intel-xxv710"
+ title: "3n-skx xxv710"
data: "vpp-compare-testbeds-mrr-cmp"
nic: "NIC_Intel-XXV710"
data: "vpp-compare-testbeds-mrr"