aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--csit.infra.dash/app/cdash/comparisons/layout.py5
-rw-r--r--csit.infra.dash/app/cdash/comparisons/tables.py6
-rw-r--r--csit.infra.dash/app/cdash/coverage/layout.py4
-rw-r--r--csit.infra.dash/app/cdash/coverage/tables.py8
-rw-r--r--csit.infra.dash/app/cdash/news/layout.py4
-rw-r--r--csit.infra.dash/app/cdash/report/graphs.py55
-rw-r--r--csit.infra.dash/app/cdash/report/layout.py4
-rw-r--r--csit.infra.dash/app/cdash/search/layout.py9
-rw-r--r--csit.infra.dash/app/cdash/search/tables.py3
-rw-r--r--csit.infra.dash/app/cdash/stats/layout.py5
-rw-r--r--csit.infra.dash/app/cdash/trending/graphs.py14
-rw-r--r--csit.infra.dash/app/cdash/trending/layout.py4
-rw-r--r--csit.infra.dash/app/cdash/utils/constants.py2
-rw-r--r--csit.infra.dash/app/cdash/utils/telemetry_data.py7
-rw-r--r--csit.infra.dash/app/cdash/utils/utils.py22
15 files changed, 80 insertions, 72 deletions
diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
index 57a1c2a1b4..735f199158 100644
--- a/csit.infra.dash/app/cdash/comparisons/layout.py
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -33,7 +33,8 @@ from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.url_processing import url_decode
from ..utils.utils import generate_options, gen_new_url, navbar_report, \
- filter_table_data, sort_table_data, show_iterative_graph_data, show_tooltip
+ filter_table_data, sort_table_data, show_iterative_graph_data, \
+ show_tooltip, get_topo_arch
from .tables import comparison_table
from ..report.graphs import graph_iterative
@@ -124,7 +125,7 @@ class Layout:
lst_job = row["job"].split("-")
dut = lst_job[1]
dver = f"{row['release']}-{row['dut_version']}"
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test_id = row["test_id"].split(".")
suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
index a951ab85e9..fa92f3da7f 100644
--- a/csit.infra.dash/app/cdash/comparisons/tables.py
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -232,7 +232,7 @@ def comparison_table(
:returns: A list of simple items.
:rtype: list
"""
- l_infra = sel["infra"].split("-")
+ l_infra = sel["infra"].rsplit("-", maxsplit=2)
selection = list()
for core in sel["core"]:
for fsize in sel["frmsize"]:
@@ -240,8 +240,8 @@ def comparison_table(
selection.append({
"dut": sel["dut"],
"dutver": sel["dutver"],
- "tbed": f"{l_infra[0]}-{l_infra[1]}",
- "nic": l_infra[2],
+ "tbed": l_infra[0],
+ "nic": l_infra[1],
"driver": l_infra[-1].replace("_", "-"),
"core": core,
"frmsize": fsize,
diff --git a/csit.infra.dash/app/cdash/coverage/layout.py b/csit.infra.dash/app/cdash/coverage/layout.py
index b8fa0236a5..7442891198 100644
--- a/csit.infra.dash/app/cdash/coverage/layout.py
+++ b/csit.infra.dash/app/cdash/coverage/layout.py
@@ -32,7 +32,7 @@ from ..utils.constants import Constants as C
from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.utils import label, gen_new_url, generate_options, navbar_report, \
- show_tooltip
+ show_tooltip, get_topo_arch
from ..utils.url_processing import url_decode
from .tables import coverage_tables, select_coverage_data
@@ -96,7 +96,7 @@ class Layout:
lst_job = row["job"].split("-")
dut = lst_job[1]
d_ver = row["dut_version"]
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test_id = row["test_id"].split(".")
if dut == "dpdk":
area = "dpdk"
diff --git a/csit.infra.dash/app/cdash/coverage/tables.py b/csit.infra.dash/app/cdash/coverage/tables.py
index 84adb091a1..d38e51e243 100644
--- a/csit.infra.dash/app/cdash/coverage/tables.py
+++ b/csit.infra.dash/app/cdash/coverage/tables.py
@@ -52,9 +52,9 @@ def select_coverage_data(
l_data = list()
# Filter data selected by the user.
- phy = selected["phy"].split("-")
- if len(phy) == 4:
- topo, arch, nic, drv = phy
+ phy = selected["phy"].rsplit("-", maxsplit=2)
+ if len(phy) == 3:
+ topo_arch, nic, drv = phy
drv_str = "" if drv == "dpdk" else drv.replace("_", "-")
else:
return l_data, None
@@ -66,7 +66,7 @@ def select_coverage_data(
(data["release"] == selected["rls"])
)])
df = df[
- (df.job.str.endswith(f"{topo}-{arch}")) &
+ (df.job.str.endswith(topo_arch)) &
(df.test_id.str.contains(
f"^.*\.{selected['area']}\..*{nic}.*{drv_str}.*$",
regex=True
diff --git a/csit.infra.dash/app/cdash/news/layout.py b/csit.infra.dash/app/cdash/news/layout.py
index ba4fc85163..8fe3d35e6a 100644
--- a/csit.infra.dash/app/cdash/news/layout.py
+++ b/csit.infra.dash/app/cdash/news/layout.py
@@ -24,7 +24,7 @@ from dash import callback_context
from dash import Input, Output, State
from ..utils.constants import Constants as C
-from ..utils.utils import gen_new_url, navbar_trending
+from ..utils.utils import gen_new_url, navbar_trending, get_topo_arch
from ..utils.anomalies import classify_anomalies
from ..utils.url_processing import url_decode
from .tables import table_summary
@@ -78,7 +78,7 @@ class Layout:
d_job_info["dut"].append(lst_job[1])
d_job_info["ttype"].append(lst_job[3])
d_job_info["cadence"].append(lst_job[4])
- d_job_info["tbed"].append("-".join(lst_job[-2:]))
+ d_job_info["tbed"].append(get_topo_arch(lst_job))
self.job_info = pd.DataFrame.from_dict(d_job_info)
# Pre-process the data:
diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py
index e13ec54a89..5e57713f11 100644
--- a/csit.infra.dash/app/cdash/report/graphs.py
+++ b/csit.infra.dash/app/cdash/report/graphs.py
@@ -24,7 +24,7 @@ from ..utils.constants import Constants as C
from ..utils.utils import get_color, get_hdrh_latencies
-def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
+def select_iterative_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
"""Select the data for graphs and tables from the provided data frame.
:param data: Data frame with data for graphs and tables.
@@ -36,9 +36,9 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
:rtype: pandas.DataFrame
"""
- phy = itm["phy"].split("-")
- if len(phy) == 4:
- topo, arch, nic, drv = phy
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ if len(phy) == 3:
+ topo_arch, nic, drv = phy
if drv == "dpdk":
drv = ""
else:
@@ -61,12 +61,12 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
(data["passed"] == True)
)]
- core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+ core = str() if itm["dut"] == "trex" else itm["core"]
ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
regex_test = \
f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$"
df = df[
- (df.job.str.endswith(f"{topo}-{arch}")) &
+ (df.job.str.endswith(topo_arch)) &
(df.dut_version.str.contains(itm["dutver"].replace(".r", "-r").\
replace("rls", "release"))) &
(df.test_id.str.contains(regex_test, regex=True))
@@ -96,16 +96,9 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
:rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
"""
- def get_y_values(data, y_data_max, param, norm_factor, release=str(),
+ def get_y_values(data, y_data_max, param, norm_factor,
remove_outliers=False):
- if param == "result_receive_rate_rate_values":
- if release in ("rls2402", "rls2406", "rls2410"):
- y_vals_raw = data["result_receive_rate_rate_avg"].to_list()
- else:
- y_vals_raw = data[param].to_list()[0]
- else:
- y_vals_raw = data[param].to_list()
- y_data = [(y * norm_factor) for y in y_vals_raw]
+ y_data = [(y * norm_factor) for y in data[param].to_list()]
if remove_outliers:
try:
@@ -146,9 +139,9 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
if itm_data.empty:
continue
- phy = itm["phy"].split("-")
- topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
- norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ topo_arch = phy[0] if len(phy) == 3 else str()
+ norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
if normalize else 1.0
if itm["area"] == "hoststack":
@@ -163,7 +156,6 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
y_tput_max,
C.VALUE_ITER[ttype],
norm_factor,
- itm["rls"],
remove_outliers
)
@@ -181,28 +173,19 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
)
}
- if itm["testtype"] == "mrr" and itm["rls"] == "rls2310":
- trial_run = "trial"
- metadata["csit-ref"] = (
- f"{itm_data['job'].to_list()[0]}/",
- f"{itm_data['build'].to_list()[0]}"
- )
- customdata = [{"metadata": metadata}, ] * nr_of_samples
- else:
- trial_run = "run"
- for _, row in itm_data.iterrows():
- metadata["csit-ref"] = f"{row['job']}/{row['build']}"
- try:
- metadata["hosts"] = ", ".join(row["hosts"])
- except (KeyError, TypeError):
- pass
- customdata.append({"metadata": deepcopy(metadata)})
+ for _, row in itm_data.iterrows():
+ metadata["csit-ref"] = f"{row['job']}/{row['build']}"
+ try:
+ metadata["hosts"] = ", ".join(row["hosts"])
+ except (KeyError, TypeError):
+ pass
+ customdata.append({"metadata": deepcopy(metadata)})
tput_kwargs = dict(
y=y_data,
name=(
f"{idx + 1}. "
f"({nr_of_samples:02d} "
- f"{trial_run}{'s' if nr_of_samples > 1 else ''}) "
+ f"run{'s' if nr_of_samples > 1 else ''}) "
f"{itm['id']}"
),
hoverinfo=u"y+name",
diff --git a/csit.infra.dash/app/cdash/report/layout.py b/csit.infra.dash/app/cdash/report/layout.py
index 2d325b0447..cd16bfbca0 100644
--- a/csit.infra.dash/app/cdash/report/layout.py
+++ b/csit.infra.dash/app/cdash/report/layout.py
@@ -32,7 +32,7 @@ from ..utils.constants import Constants as C
from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.utils import show_tooltip, label, sync_checklists, gen_new_url, \
- generate_options, get_list_group_items, navbar_report, \
+ generate_options, get_list_group_items, navbar_report, get_topo_arch, \
show_iterative_graph_data
from ..utils.url_processing import url_decode
from .graphs import graph_iterative, select_iterative_data
@@ -122,7 +122,7 @@ class Layout:
lst_job = row["job"].split("-")
dut = lst_job[1]
d_ver = row["dut_version"]
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test_id = row["test_id"].split(".")
if dut == "dpdk":
area = "dpdk"
diff --git a/csit.infra.dash/app/cdash/search/layout.py b/csit.infra.dash/app/cdash/search/layout.py
index aa4dd53d5b..8dd244a3c1 100644
--- a/csit.infra.dash/app/cdash/search/layout.py
+++ b/csit.infra.dash/app/cdash/search/layout.py
@@ -33,7 +33,7 @@ from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.utils import gen_new_url, generate_options, navbar_trending, \
filter_table_data, sort_table_data, show_trending_graph_data, \
- show_iterative_graph_data, show_tooltip
+ show_iterative_graph_data, show_tooltip, get_topo_arch
from ..utils.url_processing import url_decode
from .tables import search_table
from ..coverage.tables import coverage_tables
@@ -110,7 +110,7 @@ class Layout:
l_id = row["test_id"].split(".")
suite = l_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
replace("2n-", "")
- tb = "-".join(row["job"].split("-")[-2:])
+ tb = get_topo_arch(row["job"].split("-"))
nic = suite.split("-")[0]
for driver in C.DRIVERS:
if driver in suite:
@@ -644,7 +644,7 @@ class Layout:
"release-dis": C.STYLE_DONT_DISPLAY,
"release-val": str(),
"help-dis": disabled,
- "help-val": "<testbed> <nic> <driver> " + \
+ "help-val": "<topo> <arch> <nic> <driver> " + \
"<framesize> <cores> <test>",
"search-dis": disabled,
"search-val": str()
@@ -664,7 +664,7 @@ class Layout:
ctrl_panel.set({
"release-val": trigger.value,
"help-dis": C.STYLE_DISPLAY,
- "help-val": "<DUT version> <testbed> <nic> " + \
+ "help-val": "<DUT version> <topo> <arch> <nic> " + \
"<driver> <framesize> <core> <test>",
"search-dis": C.STYLE_DISPLAY,
"search-val": str()
@@ -758,7 +758,6 @@ class Layout:
(data["dut_version"] == dutver) &
(data["release"] == rls)
)])
-
df = df[df.full_id.str.contains(
f".*{tb}.*{nic}.*{test_name}",
regex=True
diff --git a/csit.infra.dash/app/cdash/search/tables.py b/csit.infra.dash/app/cdash/search/tables.py
index a5ffd76d3d..3492fba16f 100644
--- a/csit.infra.dash/app/cdash/search/tables.py
+++ b/csit.infra.dash/app/cdash/search/tables.py
@@ -18,6 +18,7 @@
import pandas as pd
from ..utils.constants import Constants as C
+from ..utils.utils import get_topo_arch
def select_search_data(data: pd.DataFrame, selection: list) -> pd.DataFrame:
@@ -80,7 +81,7 @@ def search_table(data: pd.DataFrame, selection: list) -> pd.DataFrame:
l_id = row["test_id"].split(".")
suite = l_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
replace("2n-", "")
- l_tb.append("-".join(row["job"].split("-")[-2:]))
+ l_tb.append(get_topo_arch(row["job"].split("-")))
l_nic.append(suite.split("-")[0])
if selection["datatype"] != "trending":
l_dutver.append(row["dut_version"])
diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py
index 823e32659a..4aae11a06b 100644
--- a/csit.infra.dash/app/cdash/stats/layout.py
+++ b/csit.infra.dash/app/cdash/stats/layout.py
@@ -30,7 +30,7 @@ from ..utils.constants import Constants as C
from ..utils.control_panel import ControlPanel
from ..utils.utils import show_tooltip, gen_new_url, get_ttypes, get_cadences, \
get_test_beds, get_job, generate_options, set_job_params, navbar_trending, \
- get_url_job, get_url_logs
+ get_url_job, get_url_logs, get_topo_arch
from ..utils.url_processing import url_decode
from .graphs import graph_statistics, select_data
@@ -93,13 +93,12 @@ class Layout:
"tbed": list()
}
for job in jobs:
- idx = -3 if "-x-" in job else -2
lst_job = job.split("-")
d_job_info["job"].append(job)
d_job_info["dut"].append(lst_job[1])
d_job_info["ttype"].append(lst_job[3])
d_job_info["cadence"].append(lst_job[4])
- d_job_info["tbed"].append("-".join(lst_job[idx:]))
+ d_job_info["tbed"].append(get_topo_arch(lst_job))
self._job_info = pd.DataFrame.from_dict(d_job_info)
self._default = set_job_params(self._job_info, d_job_info["job"][0])
diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index 10eb392648..7417fc1733 100644
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -37,9 +37,9 @@ def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
:rtype: pandas.DataFrame
"""
- phy = itm["phy"].split("-")
- if len(phy) == 4:
- topo, arch, nic, drv = phy
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ if len(phy) == 3:
+ topo_arch, nic, drv = phy
if drv == "dpdk":
drv = ""
else:
@@ -60,8 +60,8 @@ def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
(data["test_type"] == test_type) &
(data["passed"] == True)
)]
- df = df[df.job.str.endswith(f"{topo}-{arch}")]
- core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+ df = df[df.job.str.endswith(topo_arch)]
+ core = str() if itm["dut"] == "trex" else itm["core"]
ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
df = df[df.test_id.str.contains(
f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
@@ -453,8 +453,8 @@ def graph_trending(
continue
if normalize:
- phy = itm["phy"].split("-")
- topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
+ phy = itm["phy"].rsplit("-", maxsplit=2)
+ topo_arch = phy[0] if len(phy) == 3 else str()
norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
if topo_arch else 1.0
else:
diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index c844f73b65..1e2d4cdbd2 100644
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -34,7 +34,7 @@ from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.telemetry_data import TelemetryData
from ..utils.utils import show_tooltip, label, sync_checklists, gen_new_url, \
- generate_options, get_list_group_items, navbar_trending, \
+ generate_options, get_list_group_items, navbar_trending, get_topo_arch, \
show_trending_graph_data
from ..utils.url_processing import url_decode
from .graphs import graph_trending, select_trending_data, graph_tm_trending
@@ -116,7 +116,7 @@ class Layout:
for _, row in self._data[cols].drop_duplicates().iterrows():
lst_job = row["job"].split("-")
dut = lst_job[1]
- tbed = "-".join(lst_job[-2:])
+ tbed = get_topo_arch(lst_job)
lst_test = row["test_id"].split(".")
if dut == "dpdk":
area = "dpdk"
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 3b6e125d8e..444ccd3981 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -331,7 +331,7 @@ class Constants:
}
VALUE_ITER = {
- "mrr": "result_receive_rate_rate_values",
+ "mrr": "result_receive_rate_rate_avg",
"ndr": "result_ndr_lower_rate_value",
"pdr": "result_pdr_lower_rate_value",
"mrr-bandwidth": "result_receive_rate_bandwidth_avg",
diff --git a/csit.infra.dash/app/cdash/utils/telemetry_data.py b/csit.infra.dash/app/cdash/utils/telemetry_data.py
index c63ee0057a..d3a114c596 100644
--- a/csit.infra.dash/app/cdash/utils/telemetry_data.py
+++ b/csit.infra.dash/app/cdash/utils/telemetry_data.py
@@ -283,8 +283,11 @@ class TelemetryData:
for _, row in self._unique_metrics_labels.iterrows():
if _is_selected(row["labels"], selection):
lst_items.append(row.to_frame().T)
- self._selected_metrics_labels = \
- pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
+ if len(lst_items) == 1:
+ self._selected_metrics_labels = lst_items[0]
+ elif len(lst_items) > 1:
+ self._selected_metrics_labels = \
+ pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
return self._selected_metrics_labels
def select_tm_trending_data(
diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py
index 62d4770937..8171c905c3 100644
--- a/csit.infra.dash/app/cdash/utils/utils.py
+++ b/csit.infra.dash/app/cdash/utils/utils.py
@@ -946,3 +946,25 @@ def get_url_logs(job_build: str) -> str:
return str()
else:
return str()
+
+
+def get_topo_arch(lst_job: list) -> str:
+ """Get the topology and architecture string from the job name.
+
+ :param lst_job: The job name split into substrings.
+ :type lst_job: list
+ :returns: String with information about topology and architecture.
+ :rtype: str
+ """
+
+ if len(lst_job) < 2:
+ return str()
+
+ if "x" in lst_job[1:-1]:
+ # External testbeds.
+ # The structure of the name:
+ # <to be removed>-x-<topo>-<arch>-<other parts separated by dashes>
+ return "-".join(lst_job[(lst_job.index("x") + 1):])
+ else:
+ # Topology and architecture are the last two substrings.
+ return "-".join(lst_job[-2:])