aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--resources/tools/presentation/generator_tables.py147
-rw-r--r--resources/tools/presentation/input_data_parser.py1
-rw-r--r--resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml92
-rw-r--r--resources/tools/presentation/specification.yaml12
4 files changed, 155 insertions, 97 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 1366ea0824..4cbc7c0746 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -427,7 +427,12 @@ def _tpc_insert_data(target, src, include_tests):
"""
try:
if include_tests == u"MRR":
- target.append(src[u"result"][u"receive-rate"])
+ target.append(
+ (
+ src[u"result"][u"receive-rate"],
+ src[u"result"][u"receive-stdev"]
+ )
+ )
elif include_tests == u"PDR":
target.append(src[u"throughput"][u"PDR"][u"LOWER"])
elif include_tests == u"NDR":
@@ -783,7 +788,8 @@ def table_perf_comparison(table, input_data):
u"title"]] = list()
try:
if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
+ res = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
elif table[u"include-tests"] == u"NDR":
@@ -802,28 +808,40 @@ def table_perf_comparison(table, input_data):
if tbl_dict[tst_name].get(u"history", None) is not None:
for hist_data in tbl_dict[tst_name][u"history"].values():
if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ item.append(round(hist_data[0][0] / 1e6, 2))
+ item.append(round(hist_data[0][1] / 1e6, 2))
+ else:
+ item.append(round(mean(hist_data) / 1e6, 2))
+ item.append(round(stdev(hist_data) / 1e6, 2))
else:
item.extend([u"Not tested", u"Not tested"])
else:
item.extend([u"Not tested", u"Not tested"])
data_r = tbl_dict[tst_name][u"ref-data"]
if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 2))
+ item.append(round(data_r_stdev / 1e6, 2))
else:
data_r_mean = None
data_r_stdev = None
item.extend([u"Not tested", u"Not tested"])
data_c = tbl_dict[tst_name][u"cmp-data"]
if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 2))
+ item.append(round(data_c_stdev / 1e6, 2))
else:
data_c_mean = None
data_c_stdev = None
@@ -833,7 +851,7 @@ def table_perf_comparison(table, input_data):
elif item[-4] == u"Not tested":
item.append(u"New in CSIT-2001")
item.append(u"New in CSIT-2001")
- elif data_r_mean and data_c_mean:
+ elif data_r_mean is not None and data_c_mean is not None:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
)
@@ -846,7 +864,8 @@ def table_perf_comparison(table, input_data):
except ValueError:
item.append(d_stdev)
if rca_data:
- item.insert(0, rca_data.get(item[0], u" "))
+ rca_nr = rca_data.get(item[0], u"-")
+ item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
if (len(item) == len(header)) and (item[-4] != u"Not tested"):
tbl_lst.append(item)
@@ -860,7 +879,7 @@ def table_perf_comparison(table, input_data):
file_handler.write(u";".join([str(item) for item in test]) + u"\n")
txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
+ convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
if rca_data:
footnote = rca_data.get(u"footnote", "")
@@ -1080,7 +1099,8 @@ def table_perf_comparison_nic(table, input_data):
u"title"]] = list()
try:
if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
+ res = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
elif table[u"include-tests"] == u"NDR":
@@ -1099,28 +1119,40 @@ def table_perf_comparison_nic(table, input_data):
if tbl_dict[tst_name].get(u"history", None) is not None:
for hist_data in tbl_dict[tst_name][u"history"].values():
if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ item.append(round(hist_data[0][0] / 1e6, 2))
+ item.append(round(hist_data[0][1] / 1e6, 2))
+ else:
+ item.append(round(mean(hist_data) / 1e6, 2))
+ item.append(round(stdev(hist_data) / 1e6, 2))
else:
item.extend([u"Not tested", u"Not tested"])
else:
item.extend([u"Not tested", u"Not tested"])
data_r = tbl_dict[tst_name][u"ref-data"]
if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 2))
+ item.append(round(data_r_stdev / 1e6, 2))
else:
data_r_mean = None
data_r_stdev = None
item.extend([u"Not tested", u"Not tested"])
data_c = tbl_dict[tst_name][u"cmp-data"]
if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 2))
+ item.append(round(data_c_stdev / 1e6, 2))
else:
data_c_mean = None
data_c_stdev = None
@@ -1130,7 +1162,7 @@ def table_perf_comparison_nic(table, input_data):
elif item[-4] == u"Not tested":
item.append(u"New in CSIT-2001")
item.append(u"New in CSIT-2001")
- elif data_r_mean and data_c_mean:
+ elif data_r_mean is not None and data_c_mean is not None:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
)
@@ -1143,7 +1175,8 @@ def table_perf_comparison_nic(table, input_data):
except ValueError:
item.append(d_stdev)
if rca_data:
- item.insert(0, rca_data.get(item[0], u" "))
+ rca_nr = rca_data.get(item[0], u"-")
+ item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
if (len(item) == len(header)) and (item[-4] != u"Not tested"):
tbl_lst.append(item)
@@ -1227,7 +1260,8 @@ def table_nics_comparison(table, input_data):
}
try:
if table[u"include-tests"] == u"MRR":
- result = tst_data[u"result"][u"receive-rate"]
+ result = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
elif table[u"include-tests"] == u"NDR":
@@ -1250,25 +1284,33 @@ def table_nics_comparison(table, input_data):
item = [tbl_dict[tst_name][u"name"], ]
data_r = tbl_dict[tst_name][u"ref-data"]
if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 2))
+ item.append(round(data_r_stdev / 1e6, 2))
else:
data_r_mean = None
data_r_stdev = None
item.extend([None, None])
data_c = tbl_dict[tst_name][u"cmp-data"]
if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 2))
+ item.append(round(data_c_stdev / 1e6, 2))
else:
data_c_mean = None
data_c_stdev = None
item.extend([None, None])
- if data_r_mean and data_c_mean:
+ if data_r_mean is not None and data_c_mean is not None:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
)
@@ -1371,7 +1413,8 @@ def table_soak_vs_ndr(table, input_data):
if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
continue
if table[u"include-tests"] == u"MRR":
- result = tst_data[u"result"][u"receive-rate"]
+ result = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
result = \
tst_data[u"throughput"][u"PDR"][u"LOWER"]
@@ -1391,25 +1434,33 @@ def table_soak_vs_ndr(table, input_data):
item = [tbl_dict[tst_name][u"name"], ]
data_r = tbl_dict[tst_name][u"ref-data"]
if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 2))
+ item.append(round(data_r_stdev / 1e6, 2))
else:
data_r_mean = None
data_r_stdev = None
item.extend([None, None])
data_c = tbl_dict[tst_name][u"cmp-data"]
if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 2))
+ item.append(round(data_c_stdev / 1e6, 2))
else:
data_c_mean = None
data_c_stdev = None
item.extend([None, None])
- if data_r_mean and data_c_mean:
+ if data_r_mean is not None and data_c_mean is not None:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
try:
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 9e460f934e..987b9964bc 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -1100,6 +1100,7 @@ class ExecutionChecker(ResultVisitor):
# Use whole list in CSIT-1180.
stats = jumpavg.AvgStdevStats.for_runs(items_float)
test_result[u"result"][u"receive-rate"] = stats.avg
+ test_result[u"result"][u"receive-stdev"] = stats.stdev
else:
groups = re.search(self.REGEX_MRR, test.message)
test_result[u"result"][u"receive-rate"] = \
diff --git a/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
index 564e15a219..267e4c90e3 100644
--- a/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
+++ b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
@@ -1,46 +1,46 @@
-64b-2t1c-eth-l2patch: rc1
-64b-2t1c-eth-l2xcbase-eth-2memif-1dcr: rc4
-64b-2t1c-avf-eth-l2patch: rc1
-64b-2t1c-avf-eth-l2xcbase: rc1
-64b-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr: rc1, rc5
-64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc1
-64b-2t1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr: rc1
-64b-2t1c-dot1q-l2xcbase: rc1
-64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4: rc1
-64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc: rc3
-64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc3
-64b-2t1c-ethip4-ip4base-eth-2memif-1dcr: rc1
-64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc1
-64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm: rc2
-64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm: rc1
-64b-2t1c-eth-l2bdscale10kmaclrn: rc1
-64b-2t1c-eth-l2bdbasemaclrn: rc1
-64b-2t1c-eth-l2bdscale100kmaclrn: rc1
-64b-2t1c-eth-l2bdscale1mmaclrn: rc1
-78b-2t1c-ethip6-ip6base: rc1
-64b-2t1c-eth-l2xcbase: rc1
-78b-2t1c-dot1q-ip6base: rc1
-64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc2
-78b-2t1c-ethip6-ip6scale200k: rc1
-78b-2t1c-ethip6-ip6scale20k: rc1
-78b-2t1c-ethip6-ip6scale2m: rc1
-64b-2t1c-dot1q-l2bdbasemaclrn: rc1
-64b-2t1c-ethip4udp-ip4base-oacl50sf-10kflows: rc1
-64b-2t1c-ethip4udp-ip4base-oacl50sl-10kflows: rc1
-64b-2t1c-avf-eth-l2bdbasemaclrn: rc1
-64b-2t1c-ethip4-ip4scale20k: rc1
-64b-2t1c-avf-dot1q-l2bdbasemaclrn: rc1, rc3
-64b-2t1c-dot1q-ip4base: rc1
-64b-2t1c-ethip4-ip4base: rc1
-64b-2t1c-ethip4udp-ip4base-iacl50sf-10kflows: rc1
-64b-2t1c-ethip4udp-ip4base-iacl50sl-10kflows: rc1
-64b-2t1c-avf-dot1q-l2bdbasemaclrn-gbp: rc1
-64b-2t1c-ethip4-ip4scale200k: rc1
-64b-2t1c-ethip4-ip4scale2m: rc1
-64b-2t1c-avf-dot1q-ip4base: rc1
-64b-2t1c-avf-ethip4-ip4base: rc1
-64b-2t1c-avf-ethip4-ip4scale200k: rc1
-64b-2t1c-avf-ethip4-ip4scale20k: rc1
-64b-2t1c-avf-ethip4-ip4scale2m: rc1
-64b-2t1c-ethip4udp-ip4base-nat44: rc1
-footnote: "\nRoot causes:\nrc1: Description 1\nrc2: Description 2\nrc3: Description 3"
+64b-2t1c-eth-l2patch: 0
+64b-2t1c-eth-l2xcbase-eth-2memif-1dcr: 9
+64b-2t1c-avf-eth-l2patch: 8
+64b-2t1c-avf-eth-l2xcbase: 7
+64b-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr: 6
+64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: 5
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr: "-"
+64b-2t1c-dot1q-l2xcbase: 1
+64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4: 2
+64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc: 3
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: 4
+64b-2t1c-ethip4-ip4base-eth-2memif-1dcr: 0
+64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm: 0
+64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm: 8
+64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm: "-"
+64b-2t1c-eth-l2bdscale10kmaclrn: 2
+64b-2t1c-eth-l2bdbasemaclrn: 2
+64b-2t1c-eth-l2bdscale100kmaclrn: 56
+64b-2t1c-eth-l2bdscale1mmaclrn: 5
+78b-2t1c-ethip6-ip6base: 4
+64b-2t1c-eth-l2xcbase: 34
+78b-2t1c-dot1q-ip6base: 0
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm: 2
+78b-2t1c-ethip6-ip6scale200k: 6
+78b-2t1c-ethip6-ip6scale20k: 5
+78b-2t1c-ethip6-ip6scale2m: 3
+64b-2t1c-dot1q-l2bdbasemaclrn: 2
+64b-2t1c-ethip4udp-ip4base-oacl50sf-10kflows: "-"
+64b-2t1c-ethip4udp-ip4base-oacl50sl-10kflows: 4
+64b-2t1c-avf-eth-l2bdbasemaclrn: 3
+64b-2t1c-ethip4-ip4scale20k: 6
+64b-2t1c-avf-dot1q-l2bdbasemaclrn: 8
+64b-2t1c-dot1q-ip4base: 4
+64b-2t1c-ethip4-ip4base: 6
+64b-2t1c-ethip4udp-ip4base-iacl50sf-10kflows: 0
+64b-2t1c-ethip4udp-ip4base-iacl50sl-10kflows: 9
+64b-2t1c-avf-dot1q-l2bdbasemaclrn-gbp: 8
+64b-2t1c-ethip4-ip4scale200k: 9
+64b-2t1c-ethip4-ip4scale2m: 22
+64b-2t1c-avf-dot1q-ip4base: 11
+64b-2t1c-avf-ethip4-ip4base: 4
+64b-2t1c-avf-ethip4-ip4scale200k: 5
+64b-2t1c-avf-ethip4-ip4scale20k: 6
+64b-2t1c-avf-ethip4-ip4scale2m: 1
+64b-2t1c-ethip4udp-ip4base-nat44: 4
+footnote: "\nRoot causes:\n[0]: Description 0\n[1]: Description \n[2]: Description 2"
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 45bef9efab..e5810a215f 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -286,6 +286,7 @@
- 60 # NDRPDR sel addition
- 61 # NDRPDR sel addition
- 62 # NDRPDR sel addition
+ - 63 # NDRPDR sel addition
plot-vpp-hdrh-lat-3n-skx:
csit-vpp-perf-verify-2001-3n-skx:
@@ -624,6 +625,7 @@
- 60 # NDRPDR sel addition
- 61 # NDRPDR sel addition
- 62 # NDRPDR sel addition
+ - 63 # NDRPDR sel addition
vpp-nic-comparison-2n-skx:
csit-vpp-perf-verify-2001-2n-skx:
@@ -909,6 +911,7 @@
- 60 # NDRPDR sel addition
- 61 # NDRPDR sel addition
- 62 # NDRPDR sel addition
+ - 63 # NDRPDR sel addition
vpp-performance-changes-3n-skx:
- "vpp-performance-changes-3n-skx-h1"
- "vpp-performance-changes-3n-skx-ref"
@@ -1418,6 +1421,7 @@
- 60 # NDRPDR sel addition
- 61 # NDRPDR sel addition
- 62 # NDRPDR sel addition
+ - 63 # NDRPDR sel addition
vpp-compare-testbeds:
- "vpp-compare-testbeds-ref"
- "vpp-compare-testbeds-cmp"
@@ -1637,6 +1641,7 @@
- 60 # NDRPDR sel addition
- 61 # NDRPDR sel addition
- 62 # NDRPDR sel addition
+ - 63 # NDRPDR sel addition
vpp-compare-topologies-cmp:
# TODO: Add the data
csit-vpp-perf-verify-2001-2n-skx:
@@ -2519,6 +2524,7 @@
- 60 # NDRPDR sel addition
- 61 # NDRPDR sel addition
- 62 # NDRPDR sel addition
+ - 63 # NDRPDR sel addition
- 50 # MRR sel
- 53 # MRR sel addition
@@ -3462,9 +3468,9 @@
title: "VPP Performance Changes 2n-skx 2t1c pdr"
algorithm: "table_perf_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr"
-# rca:
-# title: "RCA"
-# data-file: "rca/rca-2n-skx-2t1c.yaml"
+ rca:
+ title: "RCA"
+ data-file: "rca/rca-2n-skx-2t1c.yaml"
history:
- title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-h1"