C-Dash: Make extraction of topo-arch more flexible

Change-Id: I5f59a881d6c23c3e40054a6d829abdcbcafb966b
Signed-off-by: Tibor Frank <tifrank@cisco.com>

15 files changed, 80 insertions, 72 deletions

diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
index 57a1c2a1b4..735f199158 100644
--- a/csit.infra.dash/app/cdash/comparisons/layout.py
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -33,7 +33,8 @@ from ..utils.control_panel import ControlPanel
 from ..utils.trigger import Trigger
 from ..utils.url_processing import url_decode
 from ..utils.utils import generate_options, gen_new_url, navbar_report, \
-    filter_table_data, sort_table_data, show_iterative_graph_data, show_tooltip
+    filter_table_data, sort_table_data, show_iterative_graph_data, \
+    show_tooltip, get_topo_arch
 from .tables import comparison_table
 from ..report.graphs import graph_iterative
 
@@ -124,7 +125,7 @@ class Layout:
             lst_job = row["job"].split("-")
             dut = lst_job[1]
             dver = f"{row['release']}-{row['dut_version']}"
-            tbed = "-".join(lst_job[-2:])
+            tbed = get_topo_arch(lst_job)
             lst_test_id = row["test_id"].split(".")
 
             suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
index a951ab85e9..fa92f3da7f 100644
--- a/csit.infra.dash/app/cdash/comparisons/tables.py
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -232,7 +232,7 @@ def comparison_table(
         :returns: A list of simple items.
         :rtype: list
         """
-        l_infra = sel["infra"].split("-")
+        l_infra = sel["infra"].rsplit("-", maxsplit=2)
         selection = list()
         for core in sel["core"]:
             for fsize in sel["frmsize"]:
@@ -240,8 +240,8 @@ def comparison_table(
                     selection.append({
                         "dut": sel["dut"],
                         "dutver": sel["dutver"],
-                        "tbed": f"{l_infra[0]}-{l_infra[1]}",
-                        "nic": l_infra[2],
+                        "tbed": l_infra[0],
+                        "nic": l_infra[1],
                         "driver": l_infra[-1].replace("_", "-"),
                         "core": core,
                         "frmsize": fsize,
diff --git a/csit.infra.dash/app/cdash/coverage/layout.py b/csit.infra.dash/app/cdash/coverage/layout.py
index b8fa0236a5..7442891198 100644
--- a/csit.infra.dash/app/cdash/coverage/layout.py
+++ b/csit.infra.dash/app/cdash/coverage/layout.py
@@ -32,7 +32,7 @@ from ..utils.constants import Constants as C
 from ..utils.control_panel import ControlPanel
 from ..utils.trigger import Trigger
 from ..utils.utils import label, gen_new_url, generate_options, navbar_report, \
-    show_tooltip
+    show_tooltip, get_topo_arch
 from ..utils.url_processing import url_decode
 from .tables import coverage_tables, select_coverage_data
 
@@ -96,7 +96,7 @@ class Layout:
             lst_job = row["job"].split("-")
             dut = lst_job[1]
             d_ver = row["dut_version"]
-            tbed = "-".join(lst_job[-2:])
+            tbed = get_topo_arch(lst_job)
             lst_test_id = row["test_id"].split(".")
             if dut == "dpdk":
                 area = "dpdk"
diff --git a/csit.infra.dash/app/cdash/coverage/tables.py b/csit.infra.dash/app/cdash/coverage/tables.py
index 84adb091a1..d38e51e243 100644
--- a/csit.infra.dash/app/cdash/coverage/tables.py
+++ b/csit.infra.dash/app/cdash/coverage/tables.py
@@ -52,9 +52,9 @@ def select_coverage_data(
     l_data = list()
 
     # Filter data selected by the user.
-    phy = selected["phy"].split("-")
-    if len(phy) == 4:
-        topo, arch, nic, drv = phy
+    phy = selected["phy"].rsplit("-", maxsplit=2)
+    if len(phy) == 3:
+        topo_arch, nic, drv = phy
         drv_str = "" if drv == "dpdk" else drv.replace("_", "-")
     else:
         return l_data, None
@@ -66,7 +66,7 @@ def select_coverage_data(
         (data["release"] == selected["rls"])
     )])
     df = df[
-        (df.job.str.endswith(f"{topo}-{arch}")) &
+        (df.job.str.endswith(topo_arch)) &
         (df.test_id.str.contains(
             f"^.*\.{selected['area']}\..*{nic}.*{drv_str}.*$",
             regex=True
diff --git a/csit.infra.dash/app/cdash/news/layout.py b/csit.infra.dash/app/cdash/news/layout.py
index ba4fc85163..8fe3d35e6a 100644
--- a/csit.infra.dash/app/cdash/news/layout.py
+++ b/csit.infra.dash/app/cdash/news/layout.py
@@ -24,7 +24,7 @@ from dash import callback_context
 from dash import Input, Output, State
 
 from ..utils.constants import Constants as C
-from ..utils.utils import gen_new_url, navbar_trending
+from ..utils.utils import gen_new_url, navbar_trending, get_topo_arch
 from ..utils.anomalies import classify_anomalies
 from ..utils.url_processing import url_decode
 from .tables import table_summary
@@ -78,7 +78,7 @@ class Layout:
             d_job_info["dut"].append(lst_job[1])
             d_job_info["ttype"].append(lst_job[3])
             d_job_info["cadence"].append(lst_job[4])
-            d_job_info["tbed"].append("-".join(lst_job[-2:]))
+            d_job_info["tbed"].append(get_topo_arch(lst_job))
         self.job_info = pd.DataFrame.from_dict(d_job_info)
 
         # Pre-process the data:
diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py
index e13ec54a89..5e57713f11 100644
--- a/csit.infra.dash/app/cdash/report/graphs.py
+++ b/csit.infra.dash/app/cdash/report/graphs.py
@@ -24,7 +24,7 @@ from ..utils.constants import Constants as C
 from ..utils.utils import get_color, get_hdrh_latencies
 
 
-def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
+def select_iterative_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
     """Select the data for graphs and tables from the provided data frame.
 
     :param data: Data frame with data for graphs and tables.
@@ -36,9 +36,9 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
     :rtype: pandas.DataFrame
     """
 
-    phy = itm["phy"].split("-")
-    if len(phy) == 4:
-        topo, arch, nic, drv = phy
+    phy = itm["phy"].rsplit("-", maxsplit=2)
+    if len(phy) == 3:
+        topo_arch, nic, drv = phy
         if drv == "dpdk":
             drv = ""
         else:
@@ -61,12 +61,12 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
         (data["passed"] == True)
     )]
 
-    core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+    core = str() if itm["dut"] == "trex" else itm["core"]
     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
     regex_test = \
         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$"
     df = df[
-        (df.job.str.endswith(f"{topo}-{arch}")) &
+        (df.job.str.endswith(topo_arch)) &
         (df.dut_version.str.contains(itm["dutver"].replace(".r", "-r").\
             replace("rls", "release"))) &
         (df.test_id.str.contains(regex_test, regex=True))
@@ -96,16 +96,9 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
 
-    def get_y_values(data, y_data_max, param, norm_factor, release=str(),
+    def get_y_values(data, y_data_max, param, norm_factor,
                      remove_outliers=False):
-        if param == "result_receive_rate_rate_values":
-            if release in ("rls2402", "rls2406", "rls2410"):
-                y_vals_raw = data["result_receive_rate_rate_avg"].to_list()
-            else:
-                y_vals_raw = data[param].to_list()[0]
-        else:
-            y_vals_raw = data[param].to_list()
-        y_data = [(y * norm_factor) for y in y_vals_raw]
+        y_data = [(y * norm_factor) for y in data[param].to_list()]
 
         if remove_outliers:
             try:
@@ -146,9 +139,9 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
         if itm_data.empty:
             continue
 
-        phy = itm["phy"].split("-")
-        topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
-        norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
+        phy = itm["phy"].rsplit("-", maxsplit=2)
+        topo_arch = phy[0] if len(phy) == 3 else str()
+        norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
             if normalize else 1.0
 
         if itm["area"] == "hoststack":
@@ -163,7 +156,6 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
             y_tput_max,
             C.VALUE_ITER[ttype],
             norm_factor,
-            itm["rls"],
             remove_outliers
         )
 
@@ -181,28 +173,19 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
             )
         }
 
-        if itm["testtype"] == "mrr" and itm["rls"] == "rls2310":
-            trial_run = "trial"
-            metadata["csit-ref"] = (
-                f"{itm_data['job'].to_list()[0]}/",
-                f"{itm_data['build'].to_list()[0]}"
-            )
-            customdata = [{"metadata": metadata}, ] * nr_of_samples
-        else:
-            trial_run = "run"
-            for _, row in itm_data.iterrows():
-                metadata["csit-ref"] = f"{row['job']}/{row['build']}"
-                try:
-                    metadata["hosts"] = ", ".join(row["hosts"])
-                except (KeyError, TypeError):
-                    pass
-                customdata.append({"metadata": deepcopy(metadata)})
+        for _, row in itm_data.iterrows():
+            metadata["csit-ref"] = f"{row['job']}/{row['build']}"
+            try:
+                metadata["hosts"] = ", ".join(row["hosts"])
+            except (KeyError, TypeError):
+                pass
+            customdata.append({"metadata": deepcopy(metadata)})
         tput_kwargs = dict(
             y=y_data,
             name=(
                 f"{idx + 1}. "
                 f"({nr_of_samples:02d} "
-                f"{trial_run}{'s' if nr_of_samples > 1 else ''}) "
+                f"run{'s' if nr_of_samples > 1 else ''}) "
                 f"{itm['id']}"
             ),
             hoverinfo=u"y+name",
diff --git a/csit.infra.dash/app/cdash/report/layout.py b/csit.infra.dash/app/cdash/report/layout.py
index 2d325b0447..cd16bfbca0 100644
--- a/csit.infra.dash/app/cdash/report/layout.py
+++ b/csit.infra.dash/app/cdash/report/layout.py
@@ -32,7 +32,7 @@ from ..utils.constants import Constants as C
 from ..utils.control_panel import ControlPanel
 from ..utils.trigger import Trigger
 from ..utils.utils import show_tooltip, label, sync_checklists, gen_new_url, \
-    generate_options, get_list_group_items, navbar_report, \
+    generate_options, get_list_group_items, navbar_report, get_topo_arch, \
     show_iterative_graph_data
 from ..utils.url_processing import url_decode
 from .graphs import graph_iterative, select_iterative_data
@@ -122,7 +122,7 @@ class Layout:
             lst_job = row["job"].split("-")
             dut = lst_job[1]
             d_ver = row["dut_version"]
-            tbed = "-".join(lst_job[-2:])
+            tbed = get_topo_arch(lst_job)
             lst_test_id = row["test_id"].split(".")
             if dut == "dpdk":
                 area = "dpdk"
diff --git a/csit.infra.dash/app/cdash/search/layout.py b/csit.infra.dash/app/cdash/search/layout.py
index aa4dd53d5b..8dd244a3c1 100644
--- a/csit.infra.dash/app/cdash/search/layout.py
+++ b/csit.infra.dash/app/cdash/search/layout.py
@@ -33,7 +33,7 @@ from ..utils.control_panel import ControlPanel
 from ..utils.trigger import Trigger
 from ..utils.utils import gen_new_url, generate_options, navbar_trending, \
     filter_table_data, sort_table_data, show_trending_graph_data, \
-    show_iterative_graph_data, show_tooltip
+    show_iterative_graph_data, show_tooltip, get_topo_arch
 from ..utils.url_processing import url_decode
 from .tables import search_table
 from ..coverage.tables import coverage_tables
@@ -110,7 +110,7 @@ class Layout:
                 l_id = row["test_id"].split(".")
                 suite = l_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
                     replace("2n-", "")
-                tb = "-".join(row["job"].split("-")[-2:])
+                tb = get_topo_arch(row["job"].split("-"))
                 nic = suite.split("-")[0]
                 for driver in C.DRIVERS:
                     if driver in suite:
@@ -644,7 +644,7 @@ class Layout:
                             "release-dis": C.STYLE_DONT_DISPLAY,
                             "release-val": str(),
                             "help-dis": disabled,
-                            "help-val": "<testbed> <nic> <driver> " + \
+                            "help-val": "<topo> <arch> <nic> <driver> " + \
                                 "<framesize> <cores> <test>",
                             "search-dis": disabled,
                             "search-val": str()
@@ -664,7 +664,7 @@ class Layout:
                     ctrl_panel.set({
                         "release-val": trigger.value,
                         "help-dis": C.STYLE_DISPLAY,
-                        "help-val": "<DUT version> <testbed> <nic> " + \
+                        "help-val": "<DUT version> <topo> <arch> <nic> " + \
                             "<driver> <framesize> <core> <test>",
                         "search-dis": C.STYLE_DISPLAY,
                         "search-val": str()
@@ -758,7 +758,6 @@ class Layout:
                     (data["dut_version"] == dutver) &
                     (data["release"] == rls)
                 )])
-
             df = df[df.full_id.str.contains(
                 f".*{tb}.*{nic}.*{test_name}",
                 regex=True
diff --git a/csit.infra.dash/app/cdash/search/tables.py b/csit.infra.dash/app/cdash/search/tables.py
index a5ffd76d3d..3492fba16f 100644
--- a/csit.infra.dash/app/cdash/search/tables.py
+++ b/csit.infra.dash/app/cdash/search/tables.py
@@ -18,6 +18,7 @@
 import pandas as pd
 
 from ..utils.constants import Constants as C
+from ..utils.utils import get_topo_arch
 
 
 def select_search_data(data: pd.DataFrame, selection: list) -> pd.DataFrame:
@@ -80,7 +81,7 @@ def search_table(data: pd.DataFrame, selection: list) -> pd.DataFrame:
         l_id = row["test_id"].split(".")
         suite = l_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
             replace("2n-", "")
-        l_tb.append("-".join(row["job"].split("-")[-2:]))
+        l_tb.append(get_topo_arch(row["job"].split("-")))
         l_nic.append(suite.split("-")[0])
         if selection["datatype"] != "trending":
             l_dutver.append(row["dut_version"])
diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py
index 823e32659a..4aae11a06b 100644
--- a/csit.infra.dash/app/cdash/stats/layout.py
+++ b/csit.infra.dash/app/cdash/stats/layout.py
@@ -30,7 +30,7 @@ from ..utils.constants import Constants as C
 from ..utils.control_panel import ControlPanel
 from ..utils.utils import show_tooltip, gen_new_url, get_ttypes, get_cadences, \
     get_test_beds, get_job, generate_options, set_job_params, navbar_trending, \
-    get_url_job, get_url_logs
+    get_url_job, get_url_logs, get_topo_arch
 from ..utils.url_processing import url_decode
 from .graphs import graph_statistics, select_data
 
@@ -93,13 +93,12 @@ class Layout:
             "tbed": list()
         }
         for job in jobs:
-            idx = -3 if "-x-" in job else -2
             lst_job = job.split("-")
             d_job_info["job"].append(job)
             d_job_info["dut"].append(lst_job[1])
             d_job_info["ttype"].append(lst_job[3])
             d_job_info["cadence"].append(lst_job[4])
-            d_job_info["tbed"].append("-".join(lst_job[idx:]))
+            d_job_info["tbed"].append(get_topo_arch(lst_job))
         self._job_info = pd.DataFrame.from_dict(d_job_info)
 
         self._default = set_job_params(self._job_info, d_job_info["job"][0])
diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index 10eb392648..7417fc1733 100644
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -37,9 +37,9 @@ def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
     :rtype: pandas.DataFrame
     """
 
-    phy = itm["phy"].split("-")
-    if len(phy) == 4:
-        topo, arch, nic, drv = phy
+    phy = itm["phy"].rsplit("-", maxsplit=2)
+    if len(phy) == 3:
+        topo_arch, nic, drv = phy
         if drv == "dpdk":
             drv = ""
         else:
@@ -60,8 +60,8 @@ def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
         (data["test_type"] == test_type) &
         (data["passed"] == True)
     )]
-    df = df[df.job.str.endswith(f"{topo}-{arch}")]
-    core = str() if itm["dut"] == "trex" else f"{itm['core']}"
+    df = df[df.job.str.endswith(topo_arch)]
+    core = str() if itm["dut"] == "trex" else itm["core"]
     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
     df = df[df.test_id.str.contains(
         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
@@ -453,8 +453,8 @@ def graph_trending(
             continue
 
         if normalize:
-            phy = itm["phy"].split("-")
-            topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
+            phy = itm["phy"].rsplit("-", maxsplit=2)
+            topo_arch = phy[0] if len(phy) == 3 else str()
             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
                 if topo_arch else 1.0
         else:
diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index c844f73b65..1e2d4cdbd2 100644
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -34,7 +34,7 @@ from ..utils.control_panel import ControlPanel
 from ..utils.trigger import Trigger
 from ..utils.telemetry_data import TelemetryData
 from ..utils.utils import show_tooltip, label, sync_checklists, gen_new_url, \
-    generate_options, get_list_group_items, navbar_trending, \
+    generate_options, get_list_group_items, navbar_trending, get_topo_arch, \
     show_trending_graph_data
 from ..utils.url_processing import url_decode
 from .graphs import graph_trending, select_trending_data, graph_tm_trending
@@ -116,7 +116,7 @@ class Layout:
         for _, row in self._data[cols].drop_duplicates().iterrows():
             lst_job = row["job"].split("-")
             dut = lst_job[1]
-            tbed = "-".join(lst_job[-2:])
+            tbed = get_topo_arch(lst_job)
             lst_test = row["test_id"].split(".")
             if dut == "dpdk":
                 area = "dpdk"
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 3b6e125d8e..444ccd3981 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -331,7 +331,7 @@ class Constants:
     }
 
     VALUE_ITER = {
-        "mrr": "result_receive_rate_rate_values",
+        "mrr": "result_receive_rate_rate_avg",
         "ndr": "result_ndr_lower_rate_value",
         "pdr": "result_pdr_lower_rate_value",
         "mrr-bandwidth": "result_receive_rate_bandwidth_avg",
diff --git a/csit.infra.dash/app/cdash/utils/telemetry_data.py b/csit.infra.dash/app/cdash/utils/telemetry_data.py
index c63ee0057a..d3a114c596 100644
--- a/csit.infra.dash/app/cdash/utils/telemetry_data.py
+++ b/csit.infra.dash/app/cdash/utils/telemetry_data.py
@@ -283,8 +283,11 @@ class TelemetryData:
         for _, row in self._unique_metrics_labels.iterrows():
             if _is_selected(row["labels"], selection):
                 lst_items.append(row.to_frame().T)
-        self._selected_metrics_labels = \
-            pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
+        if len(lst_items) == 1:
+            self._selected_metrics_labels = lst_items[0]
+        elif len(lst_items) > 1:
+            self._selected_metrics_labels = \
+                pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
         return self._selected_metrics_labels
 
     def select_tm_trending_data(
diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py
index 62d4770937..8171c905c3 100644
--- a/csit.infra.dash/app/cdash/utils/utils.py
+++ b/csit.infra.dash/app/cdash/utils/utils.py
@@ -946,3 +946,25 @@ def get_url_logs(job_build: str) -> str:
             return str()
     else:
         return str()
+
+
+def get_topo_arch(lst_job: list) -> str:
+    """Get the topology and architecture string from the job name.
+
+    :param lst_job: The job name split into substrings.
+    :type lst_job: list
+    :returns: String with information about topology and architecture.
+    :rtype: str
+    """
+
+    if len(lst_job) < 2:
+        return str()
+
+    if "x" in lst_job[1:-1]:
+        # External testbeds.
+        # The structure of the name:
+        # <to be removed>-x-<topo>-<arch>-<other parts separated by dashes>
+        return "-".join(lst_job[(lst_job.index("x") + 1):])
+    else:
+        # Topology and architecture are the last two substrings.
+        return "-".join(lst_job[-2:])