aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2020-03-23 20:25:17 +0100
committerTibor Frank <tifrank@cisco.com>2020-03-24 13:07:34 +0000
commit063c97d6607c1cc61a8cd129a37cca0dee6d4df8 (patch)
tree0244b87c4eb30139e44380c279daa2ebcd1a1c77
parent9959ab2ff0122d0c6c7523ab9d4241b271098845 (diff)
Report: Add RCA to comp tables
For now, only these comaprison tables: - table_perf_comparison_nic - table_perf_comparison Change-Id: I52c5a960eafbe2bbf11f6c371d4a72419373dc76 Signed-off-by: Tibor Frank <tifrank@cisco.com> (cherry picked from commit 6b353c8ae146ed5ce1c30addff6744954ed4d305)
-rw-r--r--resources/tools/presentation/generator_tables.py109
-rw-r--r--resources/tools/presentation/pal_utils.py7
-rw-r--r--resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml46
-rw-r--r--resources/tools/presentation/specification.yaml44
4 files changed, 137 insertions, 69 deletions
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 3bc8a0866f..4dca5d84d5 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -29,6 +29,7 @@ import plotly.offline as ploff
import pandas as pd
from numpy import nan, isnan
+from yaml import load, FullLoader, YAMLError
from pal_utils import mean, stdev, classify_anomalies, \
convert_csv_to_pretty_txt, relative_change_stdev
@@ -492,14 +493,24 @@ def _tpc_generate_html_table(header, data, output_file_name):
:type output_file_name: str
"""
+ try:
+ idx = header.index(u"Test case")
+ except ValueError:
+ idx = 0
+ params = {
+ u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]),
+ u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]),
+ u"width": ([28, 9], [4, 24, 10])
+ }
+
df_data = pd.DataFrame(data, columns=header)
df_sorted = [df_data.sort_values(
- by=[key, header[0]], ascending=[True, True]
- if key != header[0] else [False, True]) for key in header]
+ by=[key, header[idx]], ascending=[True, True]
+ if key != header[idx] else [False, True]) for key in header]
df_sorted_rev = [df_data.sort_values(
- by=[key, header[0]], ascending=[False, True]
- if key != header[0] else [True, True]) for key in header]
+ by=[key, header[idx]], ascending=[False, True]
+ if key != header[idx] else [True, True]) for key in header]
df_sorted.extend(df_sorted_rev)
fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
@@ -507,7 +518,7 @@ def _tpc_generate_html_table(header, data, output_file_name):
table_header = dict(
values=[f"<b>{item}</b>" for item in header],
fill_color=u"#7eade7",
- align=[u"left", u"center"]
+ align=params[u"align-hdr"][idx]
)
fig = go.Figure()
@@ -516,12 +527,12 @@ def _tpc_generate_html_table(header, data, output_file_name):
columns = [table.get(col) for col in header]
fig.add_trace(
go.Table(
- columnwidth=[30, 10],
+ columnwidth=params[u"width"][idx],
header=table_header,
cells=dict(
values=columns,
fill_color=fill_color,
- align=[u"left", u"right"]
+ align=params[u"align-itm"][idx]
)
)
)
@@ -550,7 +561,7 @@ def _tpc_generate_html_table(header, data, output_file_name):
xanchor=u"left",
y=1.045,
yanchor=u"top",
- active=len(menu_items) - 2,
+ active=len(menu_items) - 1,
buttons=list(buttons)
)
],
@@ -593,6 +604,16 @@ def table_perf_comparison(table, input_data):
try:
header = [u"Test case", ]
+ rca_data = None
+ rca = table.get(u"rca", None)
+ if rca:
+ try:
+ with open(rca.get(u"data-file", ""), u"r") as rca_file:
+ rca_data = load(rca_file, Loader=FullLoader)
+ header.insert(0, rca.get(u"title", "RCA"))
+ except (YAMLError, IOError) as err:
+ logging.warning(repr(err))
+
if table[u"include-tests"] == u"MRR":
hdr_param = u"Rec Rate"
else:
@@ -616,16 +637,14 @@ def table_perf_comparison(table, input_data):
u"Stdev of delta [%]"
]
)
- header_str = u",".join(header) + u"\n"
+ header_str = u";".join(header) + u"\n"
except (AttributeError, KeyError) as err:
logging.error(f"The model is invalid, missing parameter: {repr(err)}")
return
# Prepare data to the table:
tbl_dict = dict()
- # topo = ""
for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
tst_name_mod = _tpc_modify_test_name(tst_name)
@@ -777,7 +796,6 @@ def table_perf_comparison(table, input_data):
pass
tbl_lst = list()
- footnote = False
for tst_name in tbl_dict:
item = [tbl_dict[tst_name][u"name"], ]
if history:
@@ -815,9 +833,6 @@ def table_perf_comparison(table, input_data):
elif item[-4] == u"Not tested":
item.append(u"New in CSIT-2001")
item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
elif data_r_mean and data_c_mean:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
@@ -830,6 +845,8 @@ def table_perf_comparison(table, input_data):
item.append(round(d_stdev))
except ValueError:
item.append(d_stdev)
+ if rca_data:
+ item.insert(0, rca_data.get(item[0], u" "))
if (len(item) == len(header)) and (item[-4] != u"Not tested"):
tbl_lst.append(item)
@@ -840,24 +857,16 @@ def table_perf_comparison(table, input_data):
with open(csv_file, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
txt_file_name = f"{table[u'output-file']}.txt"
convert_csv_to_pretty_txt(csv_file, txt_file_name)
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
+ if rca_data:
+ footnote = rca_data.get(u"footnote", "")
+ if footnote:
+ with open(txt_file_name, u'a') as txt_file:
+ txt_file.writelines(footnote)
# Generate html table:
_tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
@@ -886,6 +895,16 @@ def table_perf_comparison_nic(table, input_data):
try:
header = [u"Test case", ]
+ rca_data = None
+ rca = table.get(u"rca", None)
+ if rca:
+ try:
+ with open(rca.get(u"data-file", ""), u"r") as rca_file:
+ rca_data = load(rca_file, Loader=FullLoader)
+ header.insert(0, rca.get(u"title", "RCA"))
+ except (YAMLError, IOError) as err:
+ logging.warning(repr(err))
+
if table[u"include-tests"] == u"MRR":
hdr_param = u"Rec Rate"
else:
@@ -909,16 +928,14 @@ def table_perf_comparison_nic(table, input_data):
u"Stdev of delta [%]"
]
)
- header_str = u",".join(header) + u"\n"
+ header_str = u";".join(header) + u"\n"
except (AttributeError, KeyError) as err:
logging.error(f"The model is invalid, missing parameter: {repr(err)}")
return
# Prepare data to the table:
tbl_dict = dict()
- # topo = u""
for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
@@ -1076,7 +1093,6 @@ def table_perf_comparison_nic(table, input_data):
pass
tbl_lst = list()
- footnote = False
for tst_name in tbl_dict:
item = [tbl_dict[tst_name][u"name"], ]
if history:
@@ -1114,9 +1130,6 @@ def table_perf_comparison_nic(table, input_data):
elif item[-4] == u"Not tested":
item.append(u"New in CSIT-2001")
item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
elif data_r_mean and data_c_mean:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
@@ -1129,6 +1142,8 @@ def table_perf_comparison_nic(table, input_data):
item.append(round(d_stdev))
except ValueError:
item.append(d_stdev)
+ if rca_data:
+ item.insert(0, rca_data.get(item[0], u" "))
if (len(item) == len(header)) and (item[-4] != u"Not tested"):
tbl_lst.append(item)
@@ -1139,24 +1154,16 @@ def table_perf_comparison_nic(table, input_data):
with open(csv_file, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
+ convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
+ if rca_data:
+ footnote = rca_data.get(u"footnote", "")
+ if footnote:
+ with open(txt_file_name, u'a') as txt_file:
+ txt_file.writelines(footnote)
# Generate html table:
_tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
diff --git a/resources/tools/presentation/pal_utils.py b/resources/tools/presentation/pal_utils.py
index 45fd277048..1da3350fe1 100644
--- a/resources/tools/presentation/pal_utils.py
+++ b/resources/tools/presentation/pal_utils.py
@@ -297,24 +297,27 @@ def classify_anomalies(data):
return classification, avgs
-def convert_csv_to_pretty_txt(csv_file_name, txt_file_name):
+def convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u","):
"""Convert the given csv table to pretty text table.
:param csv_file_name: The path to the input csv file.
:param txt_file_name: The path to the output pretty text file.
+ :param delimiter: Delimiter for csv file.
:type csv_file_name: str
:type txt_file_name: str
+ :type delimiter: str
"""
txt_table = None
with open(csv_file_name, u"rt") as csv_file:
- csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
+ csv_content = csv.reader(csv_file, delimiter=delimiter, quotechar=u'"')
for row in csv_content:
if txt_table is None:
txt_table = prettytable.PrettyTable(row)
else:
txt_table.add_row(row)
txt_table.align[u"Test case"] = u"l"
+ txt_table.align[u"RCA"] = u"l"
if txt_table:
with open(txt_file_name, u"wt") as txt_file:
txt_file.write(str(txt_table))
diff --git a/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
new file mode 100644
index 0000000000..564e15a219
--- /dev/null
+++ b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml
@@ -0,0 +1,46 @@
+64b-2t1c-eth-l2patch: rc1
+64b-2t1c-eth-l2xcbase-eth-2memif-1dcr: rc4
+64b-2t1c-avf-eth-l2patch: rc1
+64b-2t1c-avf-eth-l2xcbase: rc1
+64b-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr: rc1, rc5
+64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc1
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr: rc1
+64b-2t1c-dot1q-l2xcbase: rc1
+64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4: rc1
+64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc: rc3
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc3
+64b-2t1c-ethip4-ip4base-eth-2memif-1dcr: rc1
+64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc1
+64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm: rc2
+64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm: rc1
+64b-2t1c-eth-l2bdscale10kmaclrn: rc1
+64b-2t1c-eth-l2bdbasemaclrn: rc1
+64b-2t1c-eth-l2bdscale100kmaclrn: rc1
+64b-2t1c-eth-l2bdscale1mmaclrn: rc1
+78b-2t1c-ethip6-ip6base: rc1
+64b-2t1c-eth-l2xcbase: rc1
+78b-2t1c-dot1q-ip6base: rc1
+64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc2
+78b-2t1c-ethip6-ip6scale200k: rc1
+78b-2t1c-ethip6-ip6scale20k: rc1
+78b-2t1c-ethip6-ip6scale2m: rc1
+64b-2t1c-dot1q-l2bdbasemaclrn: rc1
+64b-2t1c-ethip4udp-ip4base-oacl50sf-10kflows: rc1
+64b-2t1c-ethip4udp-ip4base-oacl50sl-10kflows: rc1
+64b-2t1c-avf-eth-l2bdbasemaclrn: rc1
+64b-2t1c-ethip4-ip4scale20k: rc1
+64b-2t1c-avf-dot1q-l2bdbasemaclrn: rc1, rc3
+64b-2t1c-dot1q-ip4base: rc1
+64b-2t1c-ethip4-ip4base: rc1
+64b-2t1c-ethip4udp-ip4base-iacl50sf-10kflows: rc1
+64b-2t1c-ethip4udp-ip4base-iacl50sl-10kflows: rc1
+64b-2t1c-avf-dot1q-l2bdbasemaclrn-gbp: rc1
+64b-2t1c-ethip4-ip4scale200k: rc1
+64b-2t1c-ethip4-ip4scale2m: rc1
+64b-2t1c-avf-dot1q-ip4base: rc1
+64b-2t1c-avf-ethip4-ip4base: rc1
+64b-2t1c-avf-ethip4-ip4scale200k: rc1
+64b-2t1c-avf-ethip4-ip4scale20k: rc1
+64b-2t1c-avf-ethip4-ip4scale2m: rc1
+64b-2t1c-ethip4udp-ip4base-nat44: rc1
+footnote: "\nRoot causes:\nrc1: Description 1\nrc2: Description 2\nrc3: Description 3"
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index ddd9706c09..45bef9efab 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -3462,6 +3462,9 @@
title: "VPP Performance Changes 2n-skx 2t1c pdr"
algorithm: "table_perf_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr"
+# rca:
+# title: "RCA"
+# data-file: "rca/rca-2n-skx-2t1c.yaml"
history:
- title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-h1"
@@ -3479,16 +3482,19 @@
include-tests: "PDR" # "PDR" | "NDR" | "MRR"
filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
parameters:
- - "name"
- - "parent"
- - "throughput"
- - "tags"
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
# VPP Performance Changes 2n-skx 4t2c pdr
- type: "table"
title: "VPP Performance Changes 2n-skx 4t2c pdr"
algorithm: "table_perf_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr"
+# rca:
+# title: "RCA"
+# data-file: "rca/rca-2n-skx-4t2c.yaml"
history:
- title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-h1"
@@ -3506,16 +3512,19 @@
include-tests: "PDR" # "PDR" | "NDR" | "MRR"
filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
parameters:
- - "name"
- - "parent"
- - "throughput"
- - "tags"
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
# VPP Performance Changes 2n-skx 2t1c ndr
- type: "table"
title: "VPP Performance Changes 2n-skx 2t1c ndr"
algorithm: "table_perf_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr"
+# rca:
+# title: "RCA"
+# data-file: "rca/rca-2n-skx-2t1c.yaml"
history:
- title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-h1"
@@ -3533,16 +3542,19 @@
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
parameters:
- - "name"
- - "parent"
- - "throughput"
- - "tags"
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
# VPP Performance Changes 2n-skx 4t2c ndr
- type: "table"
title: "VPP Performance Changes 2n-skx 4t2c ndr"
algorithm: "table_perf_comparison_nic"
output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr"
+# rca:
+# title: "RCA"
+# data-file: "rca/rca-2n-skx-4t2c.yaml"
history:
- title: "rls1904 x710"
data: "vpp-performance-changes-2n-skx-h1"
@@ -3560,10 +3572,10 @@
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
parameters:
- - "name"
- - "parent"
- - "throughput"
- - "tags"
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
## VPP Performance Changes NFV Service Density 2n-skx 2t1c pdr
#- type: "table"