aboutsummaryrefslogtreecommitdiffstats
path: root/csit.infra.dash
diff options
context:
space:
mode:
Diffstat (limited to 'csit.infra.dash')
-rw-r--r--csit.infra.dash/app/cdash/__init__.py6
-rw-r--r--csit.infra.dash/app/cdash/comparisons/__init__.py12
-rw-r--r--csit.infra.dash/app/cdash/comparisons/comparisons.py51
-rw-r--r--csit.infra.dash/app/cdash/comparisons/layout.py982
-rw-r--r--csit.infra.dash/app/cdash/comparisons/tables.py283
-rw-r--r--csit.infra.dash/app/cdash/report/graphs.py2
-rw-r--r--csit.infra.dash/app/cdash/routes.py1
-rw-r--r--csit.infra.dash/app/cdash/templates/base_layout.jinja25
-rw-r--r--csit.infra.dash/app/cdash/utils/constants.py13
-rw-r--r--csit.infra.dash/app/cdash/utils/utils.py26
10 files changed, 1380 insertions, 1 deletions
diff --git a/csit.infra.dash/app/cdash/__init__.py b/csit.infra.dash/app/cdash/__init__.py
index 77722c78bd..0bc8bf7a19 100644
--- a/csit.infra.dash/app/cdash/__init__.py
+++ b/csit.infra.dash/app/cdash/__init__.py
@@ -93,6 +93,12 @@ def init_app():
data_iterative=data["iterative"]
)
+ from .comparisons.comparisons import init_comparisons
+ app = init_comparisons(
+ app,
+ data_iterative=data["iterative"]
+ )
+
return app
diff --git a/csit.infra.dash/app/cdash/comparisons/__init__.py b/csit.infra.dash/app/cdash/comparisons/__init__.py
new file mode 100644
index 0000000000..f0d52c25b6
--- /dev/null
+++ b/csit.infra.dash/app/cdash/comparisons/__init__.py
@@ -0,0 +1,12 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/csit.infra.dash/app/cdash/comparisons/comparisons.py b/csit.infra.dash/app/cdash/comparisons/comparisons.py
new file mode 100644
index 0000000000..bc42085dc8
--- /dev/null
+++ b/csit.infra.dash/app/cdash/comparisons/comparisons.py
@@ -0,0 +1,51 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Instantiate the Report Dash application.
+"""
+
+import dash
+import pandas as pd
+
+from ..utils.constants import Constants as C
+from .layout import Layout
+
+
+def init_comparisons(
+ server,
+ data_iterative: pd.DataFrame
+ ) -> dash.Dash:
+ """Create a Plotly Dash dashboard.
+
+ :param server: Flask server.
+ :type server: Flask
+ :returns: Dash app server.
+ :rtype: Dash
+ """
+
+ dash_app = dash.Dash(
+ server=server,
+ routes_pathname_prefix=C.COMP_ROUTES_PATHNAME_PREFIX,
+ external_stylesheets=C.EXTERNAL_STYLESHEETS,
+ title=C.COMP_TITLE
+ )
+
+ layout = Layout(
+ app=dash_app,
+ data_iterative=data_iterative,
+ html_layout_file=C.HTML_LAYOUT_FILE
+ )
+ dash_app.index_string = layout.html_layout
+ dash_app.layout = layout.add_content()
+
+ return dash_app.server
diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
new file mode 100644
index 0000000000..bb4c6dd93c
--- /dev/null
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -0,0 +1,982 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Plotly Dash HTML layout override.
+"""
+
+import pandas as pd
+import dash_bootstrap_components as dbc
+
+from flask import Flask
+from dash import dcc, html, dash_table, callback_context, no_update, ALL
+from dash import Input, Output, State
+from dash.exceptions import PreventUpdate
+from dash.dash_table.Format import Format, Scheme
+from ast import literal_eval
+
+from ..utils.constants import Constants as C
+from ..utils.control_panel import ControlPanel
+from ..utils.trigger import Trigger
+from ..utils.url_processing import url_decode
+from ..utils.utils import generate_options, gen_new_url
+from .tables import comparison_table
+
+
+# Control panel partameters and their default values.
+CP_PARAMS = {
+ "dut-val": str(),
+ "dutver-opt": list(),
+ "dutver-dis": True,
+ "dutver-val": str(),
+ "infra-opt": list(),
+ "infra-dis": True,
+ "infra-val": str(),
+ "core-opt": list(),
+ "core-val": list(),
+ "frmsize-opt": list(),
+ "frmsize-val": list(),
+ "ttype-opt": list(),
+ "ttype-val": list(),
+ "cmp-par-opt": list(),
+ "cmp-par-dis": True,
+ "cmp-par-val": str(),
+ "cmp-val-opt": list(),
+ "cmp-val-dis": True,
+ "cmp-val-val": str(),
+ "normalize-val": list()
+}
+
+# List of comparable parameters.
+CMP_PARAMS = {
+ "dutver": "Release and Version",
+ "infra": "Infrastructure",
+ "frmsize": "Frame Size",
+ "core": "Number of Cores",
+ "ttype": "Test Type"
+}
+
+
+class Layout:
+ """The layout of the dash app and the callbacks.
+ """
+
+ def __init__(
+ self,
+ app: Flask,
+ data_iterative: pd.DataFrame,
+ html_layout_file: str
+ ) -> None:
+ """Initialization:
+ - save the input parameters,
+ - prepare data for the control panel,
+ - read HTML layout file,
+
+ :param app: Flask application running the dash application.
+ :param data_iterative: Iterative data to be used in comparison tables.
+ :param html_layout_file: Path and name of the file specifying the HTML
+ layout of the dash application.
+ :type app: Flask
+ :type data_iterative: pandas.DataFrame
+ :type html_layout_file: str
+ """
+
+ # Inputs
+ self._app = app
+ self._html_layout_file = html_layout_file
+ self._data = data_iterative
+
+ # Get structure of tests:
+ tbs = dict()
+ cols = [
+ "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type",
+ "release", "passed"
+ ]
+ for _, row in self._data[cols].drop_duplicates().iterrows():
+ lst_job = row["job"].split("-")
+ dut = lst_job[1]
+ dver = f"{row['release']}-{row['dut_version']}"
+ tbed = "-".join(lst_job[-2:])
+ lst_test_id = row["test_id"].split(".")
+
+ suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\
+ replace("2n-", "")
+ test = lst_test_id[-1]
+ nic = suite.split("-")[0]
+ for driver in C.DRIVERS:
+ if driver in test:
+ drv = driver.replace("-", "_")
+ test = test.replace(f"{driver}-", "")
+ break
+ else:
+ drv = "dpdk"
+ infra = "-".join((tbed, nic, drv))
+ lst_test = test.split("-")
+ fsize = lst_test[0]
+ core = lst_test[1] if lst_test[1] else "8C"
+
+ if tbs.get(dut, None) is None:
+ tbs[dut] = dict()
+ if tbs[dut].get(dver, None) is None:
+ tbs[dut][dver] = dict()
+ if tbs[dut][dver].get(infra, None) is None:
+ tbs[dut][dver][infra] = dict()
+ tbs[dut][dver][infra]["core"] = list()
+ tbs[dut][dver][infra]["fsize"] = list()
+ tbs[dut][dver][infra]["ttype"] = list()
+ if core.upper() not in tbs[dut][dver][infra]["core"]:
+ tbs[dut][dver][infra]["core"].append(core.upper())
+ if fsize.upper() not in tbs[dut][dver][infra]["fsize"]:
+ tbs[dut][dver][infra]["fsize"].append(fsize.upper())
+ if row["test_type"] == "mrr":
+ if "MRR" not in tbs[dut][dver][infra]["ttype"]:
+ tbs[dut][dver][infra]["ttype"].append("MRR")
+ elif row["test_type"] == "ndrpdr":
+ if "NDR" not in tbs[dut][dver][infra]["ttype"]:
+ tbs[dut][dver][infra]["ttype"].extend(("NDR", "PDR", ))
+ elif row["test_type"] == "hoststack" and \
+ row["tg_type"] in ("iperf", "vpp"):
+ if "BPS" not in tbs[dut][dver][infra]["ttype"]:
+ tbs[dut][dver][infra]["ttype"].append("BPS")
+ elif row["test_type"] == "hoststack" and row["tg_type"] == "ab":
+ if "CPS" not in tbs[dut][dver][infra]["ttype"]:
+ tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", ))
+ self._tbs = tbs
+
+ # Read from files:
+ self._html_layout = str()
+ try:
+ with open(self._html_layout_file, "r") as file_read:
+ self._html_layout = file_read.read()
+ except IOError as err:
+ raise RuntimeError(
+ f"Not possible to open the file {self._html_layout_file}\n{err}"
+ )
+
+ # Callbacks:
+ if self._app is not None and hasattr(self, "callbacks"):
+ self.callbacks(self._app)
+
+ @property
+ def html_layout(self):
+ return self._html_layout
+
+ def add_content(self):
+ """Top level method which generated the web page.
+
+ It generates:
+ - Store for user input data,
+ - Navigation bar,
+ - Main area with control panel and ploting area.
+
+ If no HTML layout is provided, an error message is displayed instead.
+
+ :returns: The HTML div with the whole page.
+ :rtype: html.Div
+ """
+
+ if self.html_layout and self._tbs:
+ return html.Div(
+ id="div-main",
+ className="small",
+ children=[
+ dbc.Row(
+ id="row-navbar",
+ class_name="g-0",
+ children=[
+ self._add_navbar()
+ ]
+ ),
+ dbc.Row(
+ id="row-main",
+ class_name="g-0",
+ children=[
+ dcc.Store(id="store-control-panel"),
+ dcc.Store(id="store-selected"),
+ dcc.Location(id="url", refresh=False),
+ self._add_ctrl_col(),
+ self._add_plotting_col()
+ ]
+ )
+ ]
+ )
+ else:
+ return html.Div(
+ id="div-main-error",
+ children=[
+ dbc.Alert(
+ [
+ "An Error Occured"
+ ],
+ color="danger"
+ )
+ ]
+ )
+
+ def _add_navbar(self):
+ """Add nav element with navigation panel. It is placed on the top.
+
+ :returns: Navigation bar.
+ :rtype: dbc.NavbarSimple
+ """
+ return dbc.NavbarSimple(
+ id="navbarsimple-main",
+ children=[
+ dbc.NavItem(
+ dbc.NavLink(
+ C.COMP_TITLE,
+ disabled=True,
+ external_link=True,
+ href="#"
+ )
+ )
+ ],
+ brand=C.BRAND,
+ brand_href="/",
+ brand_external_link=True,
+ class_name="p-2",
+ fluid=True
+ )
+
+ def _add_ctrl_col(self) -> dbc.Col:
+ """Add column with controls. It is placed on the left side.
+
+ :returns: Column with the control panel.
+ :rtype: dbc.Col
+ """
+ return dbc.Col([
+ html.Div(
+ children=self._add_ctrl_panel(),
+ className="sticky-top"
+ )
+ ])
+
+ def _add_plotting_col(self) -> dbc.Col:
+ """Add column with plots. It is placed on the right side.
+
+ :returns: Column with plots.
+ :rtype: dbc.Col
+ """
+ return dbc.Col(
+ id="col-plotting-area",
+ children=[
+ dbc.Spinner(
+ children=[
+ dbc.Row(
+ id="plotting-area",
+ class_name="g-0 p-0",
+ children=[
+ C.PLACEHOLDER
+ ]
+ )
+ ]
+ )
+ ],
+ width=9
+ )
+
+ def _add_ctrl_panel(self) -> list:
+ """Add control panel.
+
+ :returns: Control panel.
+ :rtype: list
+ """
+
+ reference = [
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("DUT"),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "dut"},
+ placeholder="Select a Device under Test...",
+ options=sorted(
+ [
+ {"label": k, "value": k} \
+ for k in self._tbs.keys()
+ ],
+ key=lambda d: d["label"]
+ )
+ )
+ ],
+ size="sm"
+ )
+ ]
+ ),
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("CSIT and DUT Version"),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "dutver"},
+ placeholder="Select a CSIT and DUT Version...")
+ ],
+ size="sm"
+ )
+ ]
+ ),
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("Infra"),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "infra"},
+ placeholder=\
+ "Select a Physical Test Bed Topology..."
+ )
+ ],
+ size="sm"
+ )
+ ]
+ ),
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("Frame Size"),
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "frmsize"},
+ inline=True,
+ class_name="ms-2"
+ )
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ )
+ ]
+ ),
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("Number of Cores"),
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "core"},
+ inline=True,
+ class_name="ms-2"
+ )
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ )
+ ]
+ ),
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("Test Type"),
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "ttype"},
+ inline=True,
+ class_name="ms-2"
+ )
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ )
+ ]
+ )
+ ]
+
+ compare = [
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("Parameter"),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "cmpprm"},
+ placeholder="Select a Parameter..."
+ )
+ ],
+ size="sm"
+ )
+ ]
+ ),
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText("Value"),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "cmpval"},
+ placeholder="Select a Value..."
+ )
+ ],
+ size="sm"
+ )
+ ]
+ )
+ ]
+
+ normalize = [
+ dbc.Row(
+ class_name="g-0 p-1",
+ children=[
+ dbc.InputGroup(
+ dbc.Checklist(
+ id="normalize",
+ options=[{
+ "value": "normalize",
+ "label": "Normalize to 2GHz CPU frequency"
+ }],
+ value=[],
+ inline=True,
+ class_name="ms-2"
+ ),
+ style={"align-items": "center"},
+ size="sm"
+ )
+ ]
+ )
+ ]
+
+ return [
+ dbc.Row(
+ dbc.Card(
+ [
+ dbc.CardHeader(
+ html.H5("Reference Value")
+ ),
+ dbc.CardBody(
+ children=reference,
+ class_name="g-0 p-0"
+ )
+ ],
+ color="secondary",
+ outline=True
+ ),
+ class_name="g-0 p-1"
+ ),
+ dbc.Row(
+ dbc.Card(
+ [
+ dbc.CardHeader(
+ html.H5("Compared Value")
+ ),
+ dbc.CardBody(
+ children=compare,
+ class_name="g-0 p-0"
+ )
+ ],
+ color="secondary",
+ outline=True
+ ),
+ class_name="g-0 p-1"
+ ),
+ dbc.Row(
+ dbc.Card(
+ [
+ dbc.CardHeader(
+ html.H5("Normalization")
+ ),
+ dbc.CardBody(
+ children=normalize,
+ class_name="g-0 p-0"
+ )
+ ],
+ color="secondary",
+ outline=True
+ ),
+ class_name="g-0 p-1"
+ )
+ ]
+
+ def _get_plotting_area(
+ self,
+ selected: dict,
+ url: str,
+ normalize: bool
+ ) -> list:
+ """Generate the plotting area with all its content.
+
+ :param selected: Selected parameters of tests.
+ :param normalize: If true, the values in tables are normalized.
+ :param url: URL to be displayed in the modal window.
+ :type selected: dict
+ :type normalize: bool
+ :type url: str
+ :returns: List of rows with elements to be displayed in the plotting
+ area.
+ :rtype: list
+ """
+
+ title, df = comparison_table(self._data, selected, normalize)
+
+ if df.empty:
+ return dbc.Row(
+ dbc.Col(
+ children=dbc.Alert(
+ "No data for comparison.",
+ color="danger"
+ ),
+ class_name="g-0 p-1",
+ ),
+ class_name="g-0 p-0"
+ )
+
+ cols = list()
+ for idx, col in enumerate(df.columns):
+ if idx == 0:
+ cols.append({
+ "name": ["", col],
+ "id": col,
+ "deletable": False,
+ "selectable": False,
+ "type": "text"
+ })
+ else:
+ l_col = col.rsplit(" ", 2)
+ cols.append({
+ "name": [l_col[0], " ".join(l_col[-2:])],
+ "id": col,
+ "deletable": False,
+ "selectable": False,
+ "type": "numeric",
+ "format": Format(precision=2, scheme=Scheme.fixed)
+ })
+
+ return [
+ dbc.Row(
+ children=html.H5(title),
+ class_name="g-0 p-1"
+ ),
+ dbc.Row(
+ children=[
+ dbc.Col(
+ children=dash_table.DataTable(
+ columns=cols,
+ data=df.to_dict("records"),
+ merge_duplicate_headers=True,
+ editable=True,
+ filter_action="native",
+ sort_action="native",
+ sort_mode="multi",
+ selected_columns=[],
+ selected_rows=[],
+ page_action="none",
+ style_cell={"textAlign": "right"},
+ style_cell_conditional=[{
+ "if": {"column_id": "Test Name"},
+ "textAlign": "left"
+ }]
+ ),
+ class_name="g-0 p-1"
+ )
+ ],
+ class_name="g-0 p-0"
+ ),
+ dbc.Row(
+ [
+ dbc.Col([html.Div(
+ [
+ dbc.Button(
+ id="plot-btn-url",
+ children="Show URL",
+ class_name="me-1",
+ color="info",
+ style={
+ "text-transform": "none",
+ "padding": "0rem 1rem"
+ }
+ ),
+ dbc.Modal(
+ [
+ dbc.ModalHeader(dbc.ModalTitle("URL")),
+ dbc.ModalBody(url)
+ ],
+ id="plot-mod-url",
+ size="xl",
+ is_open=False,
+ scrollable=True
+ ),
+ dbc.Button(
+ id="plot-btn-download",
+ children="Download Data",
+ class_name="me-1",
+ color="info",
+ style={
+ "text-transform": "none",
+ "padding": "0rem 1rem"
+ }
+ ),
+ dcc.Download(id="download-iterative-data")
+ ],
+ className=\
+ "d-grid gap-0 d-md-flex justify-content-md-end"
+ )])
+ ],
+ class_name="g-0 p-0"
+ ),
+ dbc.Row(
+ children=C.PLACEHOLDER,
+ class_name="g-0 p-1"
+ )
+ ]
+
+ def callbacks(self, app):
+ """Callbacks for the whole application.
+
+ :param app: The application.
+ :type app: Flask
+ """
+
+ @app.callback(
+ [
+ Output("store-control-panel", "data"),
+ Output("store-selected", "data"),
+ Output("plotting-area", "children"),
+ Output({"type": "ctrl-dd", "index": "dut"}, "value"),
+ Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
+ Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
+ Output({"type": "ctrl-dd", "index": "dutver"}, "value"),
+ Output({"type": "ctrl-dd", "index": "infra"}, "options"),
+ Output({"type": "ctrl-dd", "index": "infra"}, "disabled"),
+ Output({"type": "ctrl-dd", "index": "infra"}, "value"),
+ Output({"type": "ctrl-cl", "index": "core"}, "options"),
+ Output({"type": "ctrl-cl", "index": "core"}, "value"),
+ Output({"type": "ctrl-cl", "index": "frmsize"}, "options"),
+ Output({"type": "ctrl-cl", "index": "frmsize"}, "value"),
+ Output({"type": "ctrl-cl", "index": "ttype"}, "options"),
+ Output({"type": "ctrl-cl", "index": "ttype"}, "value"),
+ Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"),
+ Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"),
+ Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"),
+ Output({"type": "ctrl-dd", "index": "cmpval"}, "options"),
+ Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"),
+ Output({"type": "ctrl-dd", "index": "cmpval"}, "value"),
+ Output("normalize", "value")
+ ],
+ [
+ State("store-control-panel", "data"),
+ State("store-selected", "data")
+ ],
+ [
+ Input("url", "href"),
+ Input("normalize", "value"),
+ Input({"type": "ctrl-dd", "index": ALL}, "value"),
+ Input({"type": "ctrl-cl", "index": ALL}, "value"),
+ Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
+ ]
+ )
+ def _update_application(
+ control_panel: dict,
+ selected: dict,
+ href: str,
+ normalize: list,
+ *_
+ ) -> tuple:
+ """Update the application when the event is detected.
+ """
+
+ ctrl_panel = ControlPanel(CP_PARAMS, control_panel)
+
+ if selected is None:
+ selected = {
+ "reference": {
+ "set": False,
+ },
+ "compare": {
+ "set": False,
+ }
+ }
+
+ # Parse the url:
+ parsed_url = url_decode(href)
+ if parsed_url:
+ url_params = parsed_url["params"]
+ else:
+ url_params = None
+
+ on_draw = False
+ plotting_area = no_update
+
+ trigger = Trigger(callback_context.triggered)
+ if trigger.type == "url" and url_params:
+ process_url = False
+ try:
+ selected = literal_eval(url_params["selected"][0])
+ r_sel = selected["reference"]["selection"]
+ c_sel = selected["compare"]
+ normalize = literal_eval(url_params["norm"][0])
+ process_url = bool(
+ (selected["reference"]["set"] == True) and
+ (c_sel["set"] == True)
+ )
+ except (KeyError, IndexError):
+ pass
+ if process_url:
+ ctrl_panel.set({
+ "dut-val": r_sel["dut"],
+ "dutver-opt": generate_options(
+ self._tbs[r_sel["dut"]].keys()
+ ),
+ "dutver-dis": False,
+ "dutver-val": r_sel["dutver"],
+ "infra-opt": generate_options(
+ self._tbs[r_sel["dut"]][r_sel["dutver"]].keys()
+ ),
+ "infra-dis": False,
+ "infra-val": r_sel["infra"],
+ "core-opt": generate_options(
+ self._tbs[r_sel["dut"]][r_sel["dutver"]]\
+ [r_sel["infra"]]["core"]
+ ),
+ "core-val": r_sel["core"],
+ "frmsize-opt": generate_options(
+ self._tbs[r_sel["dut"]][r_sel["dutver"]]\
+ [r_sel["infra"]]["fsize"]
+ ),
+ "frmsize-val": r_sel["frmsize"],
+ "ttype-opt": generate_options(
+ self._tbs[r_sel["dut"]][r_sel["dutver"]]\
+ [r_sel["infra"]]["ttype"]
+ ),
+ "ttype-val": r_sel["ttype"],
+ "normalize-val": normalize
+ })
+ opts = list()
+ for itm, label in CMP_PARAMS.items():
+ if len(ctrl_panel.get(f"{itm}-opt")) > 1:
+ opts.append({"label": label, "value": itm})
+ ctrl_panel.set({
+ "cmp-par-opt": opts,
+ "cmp-par-dis": False,
+ "cmp-par-val": c_sel["parameter"]
+ })
+ opts = list()
+ for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"):
+ set_val = ctrl_panel.get(f"{c_sel['parameter']}-val")
+ if isinstance(set_val, list):
+ if itm["value"] not in set_val:
+ opts.append(itm)
+ else:
+ if itm["value"] != set_val:
+ opts.append(itm)
+ ctrl_panel.set({
+ "cmp-val-opt": opts,
+ "cmp-val-dis": False,
+ "cmp-val-val": c_sel["value"]
+ })
+ on_draw = True
+ elif trigger.type == "normalize":
+ ctrl_panel.set({"normalize-val": normalize})
+ on_draw = True
+ elif trigger.type == "ctrl-dd":
+ if trigger.idx == "dut":
+ try:
+ opts = generate_options(self._tbs[trigger.value].keys())
+ disabled = False
+ except KeyError:
+ opts = list()
+ disabled = True
+ ctrl_panel.set({
+ "dut-val": trigger.value,
+ "dutver-opt": opts,
+ "dutver-dis": disabled,
+ "dutver-val": str(),
+ "infra-opt": list(),
+ "infra-dis": True,
+ "infra-val": str(),
+ "core-opt": list(),
+ "core-val": list(),
+ "frmsize-opt": list(),
+ "frmsize-val": list(),
+ "ttype-opt": list(),
+ "ttype-val": list(),
+ "cmp-par-opt": list(),
+ "cmp-par-dis": True,
+ "cmp-par-val": str(),
+ "cmp-val-opt": list(),
+ "cmp-val-dis": True,
+ "cmp-val-val": str()
+ })
+ elif trigger.idx == "dutver":
+ try:
+ dut = ctrl_panel.get("dut-val")
+ dver = self._tbs[dut][trigger.value]
+ opts = generate_options(dver.keys())
+ disabled = False
+ except KeyError:
+ opts = list()
+ disabled = True
+ ctrl_panel.set({
+ "dutver-val": trigger.value,
+ "infra-opt": opts,
+ "infra-dis": disabled,
+ "infra-val": str(),
+ "core-opt": list(),
+ "core-val": list(),
+ "frmsize-opt": list(),
+ "frmsize-val": list(),
+ "ttype-opt": list(),
+ "ttype-val": list(),
+ "cmp-par-opt": list(),
+ "cmp-par-dis": True,
+ "cmp-par-val": str(),
+ "cmp-val-opt": list(),
+ "cmp-val-dis": True,
+ "cmp-val-val": str()
+ })
+ elif trigger.idx == "infra":
+ dut = ctrl_panel.get("dut-val")
+ dver = ctrl_panel.get("dutver-val")
+ if all((dut, dver, trigger.value, )):
+ driver = self._tbs[dut][dver][trigger.value]
+ ctrl_panel.set({
+ "infra-val": trigger.value,
+ "core-opt": generate_options(driver["core"]),
+ "core-val": list(),
+ "frmsize-opt": generate_options(driver["fsize"]),
+ "frmsize-val": list(),
+ "ttype-opt": generate_options(driver["ttype"]),
+ "ttype-val": list(),
+ "cmp-par-opt": list(),
+ "cmp-par-dis": True,
+ "cmp-par-val": str(),
+ "cmp-val-opt": list(),
+ "cmp-val-dis": True,
+ "cmp-val-val": str()
+ })
+ elif trigger.idx == "cmpprm":
+ value = trigger.value
+ opts = list()
+ for itm in ctrl_panel.get(f"{value}-opt"):
+ set_val = ctrl_panel.get(f"{value}-val")
+ if isinstance(set_val, list):
+ if itm["value"] not in set_val:
+ opts.append(itm)
+ else:
+ if itm["value"] != set_val:
+ opts.append(itm)
+ ctrl_panel.set({
+ "cmp-par-val": value,
+ "cmp-val-opt": opts,
+ "cmp-val-dis": False,
+ "cmp-val-val": str()
+ })
+ elif trigger.idx == "cmpval":
+ ctrl_panel.set({"cmp-val-val": trigger.value})
+ selected["reference"] = {
+ "set": True,
+ "selection": {
+ "dut": ctrl_panel.get("dut-val"),
+ "dutver": ctrl_panel.get("dutver-val"),
+ "infra": ctrl_panel.get("infra-val"),
+ "core": ctrl_panel.get("core-val"),
+ "frmsize": ctrl_panel.get("frmsize-val"),
+ "ttype": ctrl_panel.get("ttype-val")
+ }
+ }
+ selected["compare"] = {
+ "set": True,
+ "parameter": ctrl_panel.get("cmp-par-val"),
+ "value": trigger.value
+ }
+ on_draw = True
+ elif trigger.type == "ctrl-cl":
+ ctrl_panel.set({f"{trigger.idx}-val": trigger.value})
+ if all((ctrl_panel.get("core-val"),
+ ctrl_panel.get("frmsize-val"),
+ ctrl_panel.get("ttype-val"), )):
+ opts = list()
+ for itm, label in CMP_PARAMS.items():
+ if len(ctrl_panel.get(f"{itm}-opt")) > 1:
+ if isinstance(ctrl_panel.get(f"{itm}-val"), list):
+ if len(ctrl_panel.get(f"{itm}-opt")) == \
+ len(ctrl_panel.get(f"{itm}-val")):
+ continue
+ opts.append({"label": label, "value": itm})
+ ctrl_panel.set({
+ "cmp-par-opt": opts,
+ "cmp-par-dis": False,
+ "cmp-par-val": str(),
+ "cmp-val-opt": list(),
+ "cmp-val-dis": True,
+ "cmp-val-val": str()
+ })
+ else:
+ ctrl_panel.set({
+ "cmp-par-opt": list(),
+ "cmp-par-dis": True,
+ "cmp-par-val": str(),
+ "cmp-val-opt": list(),
+ "cmp-val-dis": True,
+ "cmp-val-val": str()
+ })
+
+ if all((on_draw, selected["reference"]["set"],
+ selected["compare"]["set"], )):
+ plotting_area = self._get_plotting_area(
+ selected=selected,
+ normalize=bool(normalize),
+ url=gen_new_url(
+ parsed_url,
+ params={"selected": selected, "norm": normalize}
+ )
+ )
+
+ ret_val = [ctrl_panel.panel, selected, plotting_area]
+ ret_val.extend(ctrl_panel.values)
+ return ret_val
+
+ @app.callback(
+ Output("plot-mod-url", "is_open"),
+ Input("plot-btn-url", "n_clicks"),
+ State("plot-mod-url", "is_open")
+ )
+ def toggle_plot_mod_url(n, is_open):
+ """Toggle the modal window with url.
+ """
+ if n:
+ return not is_open
+ return is_open
+
+ @app.callback(
+ Output("download-iterative-data", "data"),
+ State("store-selected", "data"),
+ State("normalize", "value"),
+ Input("plot-btn-download", "n_clicks"),
+ prevent_initial_call=True
+ )
+ def _download_trending_data(selected: dict, normalize: list, _: int):
+ """Download the data.
+
+ :param selected: List of tests selected by user stored in the
+ browser.
+ :param normalize: If set, the data is normalized to 2GHz CPU
+ frequency.
+ :type selected: list
+ :type normalize: list
+ :returns: dict of data frame content (base64 encoded) and meta data
+ used by the Download component.
+ :rtype: dict
+ """
+
+ if not selected:
+ raise PreventUpdate
+
+ _, table = comparison_table(self._data, selected, normalize, "csv")
+
+ return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
new file mode 100644
index 0000000000..14d5d552af
--- /dev/null
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -0,0 +1,283 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The comparison tables.
+"""
+
+import pandas as pd
+
+from numpy import mean, std
+from copy import deepcopy
+from ..utils.constants import Constants as C
+from ..utils.utils import relative_change_stdev
+
+
+def select_comparison_data(
+ data: pd.DataFrame,
+ selected: dict,
+ normalize: bool=False
+ ) -> pd.DataFrame:
+ """Select data for a comparison table.
+
+ :param data: Data to be filtered for the comparison table.
+ :param selected: A dictionary with parameters and their values selected by
+ the user.
+ :param normalize: If True, the data is normalized to CPU frequency
+ Constants.NORM_FREQUENCY.
+ :type data: pandas.DataFrame
+ :type selected: dict
+ :type normalize: bool
+ :returns: A data frame with selected data.
+ :rtype: pandas.DataFrame
+ """
+
+ def _calculate_statistics(
+ data_in: pd.DataFrame,
+ ttype: str,
+ drv: str,
+ norm_factor: float
+ ) -> pd.DataFrame:
+ """Calculates mean value and standard deviation for provided data.
+
+ :param data_in: Input data for calculations.
+ :param ttype: The test type.
+ :param drv: The driver.
+ :param norm_factor: The data normalization factor.
+ :type data_in: pandas.DataFrame
+ :type ttype: str
+ :type drv: str
+ :type norm_factor: float
+ :returns: A pandas dataframe with: test name, mean value, standard
+ deviation and unit.
+ :rtype: pandas.DataFrame
+ """
+ d_data = {
+ "name": list(),
+ "mean": list(),
+ "stdev": list(),
+ "unit": list()
+ }
+ for itm in data_in["test_id"].unique().tolist():
+ itm_lst = itm.split(".")
+ test = itm_lst[-1].rsplit("-", 1)[0]
+ df = data_in.loc[(data_in["test_id"] == itm)]
+ l_df = df[C.VALUE_ITER[ttype]].to_list()
+ if len(l_df) and isinstance(l_df[0], list):
+ tmp_df = list()
+ for l_itm in l_df:
+ tmp_df.extend(l_itm)
+ l_df = tmp_df
+ d_data["name"].append(f"{test.replace(f'{drv}-', '')}-{ttype}")
+ d_data["mean"].append(int(mean(l_df) * norm_factor))
+ d_data["stdev"].append(int(std(l_df) * norm_factor))
+ d_data["unit"].append(df[C.UNIT[ttype]].to_list()[0])
+ return pd.DataFrame(d_data)
+
+ lst_df = list()
+ for itm in selected:
+ if itm["ttype"] in ("NDR", "PDR"):
+ test_type = "ndrpdr"
+ else:
+ test_type = itm["ttype"].lower()
+
+ dutver = itm["dutver"].split("-", 1) # 0 -> release, 1 -> dut version
+ tmp_df = pd.DataFrame(data.loc[(
+ (data["passed"] == True) &
+ (data["dut_type"] == itm["dut"]) &
+ (data["dut_version"] == dutver[1]) &
+ (data["test_type"] == test_type) &
+ (data["release"] == dutver[0])
+ )])
+
+ drv = "" if itm["driver"] == "dpdk" else itm["driver"].replace("_", "-")
+ core = str() if itm["dut"] == "trex" else itm["core"].lower()
+ reg_id = \
+ f"^.*[.|-]{itm['nic']}.*{itm['frmsize'].lower()}-{core}-{drv}.*$"
+ tmp_df = tmp_df[
+ (tmp_df.job.str.endswith(itm["tbed"])) &
+ (tmp_df.test_id.str.contains(reg_id, regex=True))
+ ]
+ if itm["driver"] == "dpdk":
+ for drv in C.DRIVERS:
+ tmp_df.drop(
+ tmp_df[tmp_df.test_id.str.contains(f"-{drv}-")].index,
+ inplace=True
+ )
+
+ # Change the data type from ndrpdr to one of ("NDR", "PDR")
+ if test_type == "ndrpdr":
+ tmp_df = tmp_df.assign(test_type=itm["ttype"].lower())
+
+ if not tmp_df.empty:
+ tmp_df = _calculate_statistics(
+ tmp_df,
+ itm["ttype"].lower(),
+ itm["driver"],
+ C.NORM_FREQUENCY / C.FREQUENCY[itm["tbed"]] if normalize else 1
+ )
+
+ lst_df.append(tmp_df)
+
+ if len(lst_df) == 1:
+ df = lst_df[0]
+ elif len(lst_df) > 1:
+ df = pd.concat(
+ lst_df,
+ ignore_index=True,
+ copy=False
+ )
+ else:
+ df = pd.DataFrame()
+
+ return df
+
+
+def comparison_table(
+ data: pd.DataFrame,
+ selected: dict,
+ normalize: bool,
+ format: str="html"
+ ) -> tuple:
+ """Generate a comparison table.
+
+ :param data: Iterative data for the comparison table.
+ :param selected: A dictionary with parameters and their values selected by
+ the user.
+ :param normalize: If True, the data is normalized to CPU frequency
+ Constants.NORM_FREQUENCY.
+ :param format: The output format of the table:
+ - html: To be displayed on html page, the values are shown in millions
+ of the unit.
+ - csv: To be downloaded as a CSV file the values are stored in base
+ units.
+ :type data: pandas.DataFrame
+ :type selected: dict
+ :type normalize: bool
+ :type format: str
+ :returns: A tuple with the tabe title and the comparison table.
+ :rtype: tuple[str, pandas.DataFrame]
+ """
+
+ def _create_selection(sel: dict) -> list:
+ """Transform the complex dictionary with user selection to list
+ of simple items.
+
+ :param sel: A complex dictionary with user selection.
+ :type sel: dict
+ :returns: A list of simple items.
+ :rtype: list
+ """
+ l_infra = sel["infra"].split("-")
+ selection = list()
+ for core in sel["core"]:
+ for fsize in sel["frmsize"]:
+ for ttype in sel["ttype"]:
+ selection.append({
+ "dut": sel["dut"],
+ "dutver": sel["dutver"],
+ "tbed": f"{l_infra[0]}-{l_infra[1]}",
+ "nic": l_infra[2],
+ "driver": l_infra[-1].replace("_", "-"),
+ "core": core,
+ "frmsize": fsize,
+ "ttype": ttype
+ })
+ return selection
+
+ unit_factor, s_unit_factor = (1e6, "M") if format == "html" else (1, str())
+
+ r_sel = deepcopy(selected["reference"]["selection"])
+ c_params = selected["compare"]
+ r_selection = _create_selection(r_sel)
+
+ # Create Table title and titles of columns with data
+ params = list(r_sel)
+ params.remove(c_params["parameter"])
+ lst_title = list()
+ for param in params:
+ value = r_sel[param]
+ if isinstance(value, list):
+ lst_title.append("|".join(value))
+ else:
+ lst_title.append(value)
+ title = "Comparison for: " + "-".join(lst_title)
+ r_name = r_sel[c_params["parameter"]]
+ if isinstance(r_name, list):
+ r_name = "|".join(r_name)
+ c_name = c_params["value"]
+
+ # Select reference data
+ r_data = select_comparison_data(data, r_selection, normalize)
+
+ # Select compare data
+ c_sel = deepcopy(selected["reference"]["selection"])
+ if c_params["parameter"] in ("core", "frmsize", "ttype"):
+ c_sel[c_params["parameter"]] = [c_params["value"], ]
+ else:
+ c_sel[c_params["parameter"]] = c_params["value"]
+
+ c_selection = _create_selection(c_sel)
+ c_data = select_comparison_data(data, c_selection, normalize)
+
+ if r_data.empty or c_data.empty:
+ return str(), pd.DataFrame()
+
+ l_name, l_r_mean, l_r_std, l_c_mean, l_c_std, l_rc_mean, l_rc_std, unit = \
+ list(), list(), list(), list(), list(), list(), list(), set()
+ for _, row in r_data.iterrows():
+ if c_params["parameter"] in ("core", "frmsize", "ttype"):
+ l_cmp = row["name"].split("-")
+ if c_params["parameter"] == "core":
+ c_row = c_data[
+ (c_data.name.str.contains(l_cmp[0])) &
+ (c_data.name.str.contains("-".join(l_cmp[2:])))
+ ]
+ elif c_params["parameter"] == "frmsize":
+ c_row = c_data[c_data.name.str.contains("-".join(l_cmp[1:]))]
+ elif c_params["parameter"] == "ttype":
+ regex = r"^" + f"{'-'.join(l_cmp[:-1])}" + r"-.{3}$"
+ c_row = c_data[c_data.name.str.contains(regex, regex=True)]
+ else:
+ c_row = c_data[c_data["name"] == row["name"]]
+ if not c_row.empty:
+ unit.add(f"{s_unit_factor}{row['unit']}")
+ r_mean = row["mean"]
+ r_std = row["stdev"]
+ c_mean = c_row["mean"].values[0]
+ c_std = c_row["stdev"].values[0]
+ l_name.append(row["name"])
+ l_r_mean.append(r_mean / unit_factor)
+ l_r_std.append(r_std / unit_factor)
+ l_c_mean.append(c_mean / unit_factor)
+ l_c_std.append(c_std / unit_factor)
+ delta, d_stdev = relative_change_stdev(r_mean, c_mean, r_std, c_std)
+ l_rc_mean.append(delta)
+ l_rc_std.append(d_stdev)
+
+ s_unit = "|".join(unit)
+ df_cmp = pd.DataFrame.from_dict({
+ "Test Name": l_name,
+ f"{r_name} Mean [{s_unit}]": l_r_mean,
+ f"{r_name} Stdev [{s_unit}]": l_r_std,
+ f"{c_name} Mean [{s_unit}]": l_c_mean,
+ f"{c_name} Stdev [{s_unit}]": l_c_std,
+ "Relative Change Mean [%]": l_rc_mean,
+ "Relative Change Stdev [%]": l_rc_std
+ })
+ df_cmp.sort_values(
+ by="Relative Change Mean [%]",
+ ascending=False,
+ inplace=True
+ )
+
+ return (title, df_cmp)
diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py
index 870f16a533..9d10efc4f0 100644
--- a/csit.infra.dash/app/cdash/report/graphs.py
+++ b/csit.infra.dash/app/cdash/report/graphs.py
@@ -80,7 +80,7 @@ def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
:param data: Data frame with iterative data.
:param sel: Selected tests.
:param layout: Layout of plot.ly graph.
- :param normalize: If True, the data is normalized to CPU frquency
+ :param normalize: If True, the data is normalized to CPU frequency
Constants.NORM_FREQUENCY.
:param data: pandas.DataFrame
:param sel: dict
diff --git a/csit.infra.dash/app/cdash/routes.py b/csit.infra.dash/app/cdash/routes.py
index 1906d00ee5..71c13edd6d 100644
--- a/csit.infra.dash/app/cdash/routes.py
+++ b/csit.infra.dash/app/cdash/routes.py
@@ -30,6 +30,7 @@ def home():
description=C.DESCRIPTION,
trending_title=C.TREND_TITLE,
report_title=C.REPORT_TITLE,
+ comp_title=C.COMP_TITLE,
stats_title=C.STATS_TITLE,
news_title=C.NEWS_TITLE
)
diff --git a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 b/csit.infra.dash/app/cdash/templates/base_layout.jinja2
index ff31c72178..b799bda3a1 100644
--- a/csit.infra.dash/app/cdash/templates/base_layout.jinja2
+++ b/csit.infra.dash/app/cdash/templates/base_layout.jinja2
@@ -31,6 +31,11 @@
</a>
</p>
<p>
+ <a href="/comparisons/" class="btn btn-primary fw-bold w-25">
+ {{ comp_title }}
+ </a>
+ </p>
+ <p>
<a href="/stats/" class="btn btn-primary fw-bold w-25">
{{ stats_title }}
</a>
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 12d7ee5702..1aedb9b96f 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -142,6 +142,7 @@ class Constants:
NORM_FREQUENCY = 2.0 # [GHz]
FREQUENCY = { # [GHz]
+ "1n-aws": 1.000,
"2n-aws": 1.000,
"2n-dnv": 2.000,
"2n-clx": 2.300,
@@ -291,6 +292,18 @@ class Constants:
REPORT_DOWNLOAD_FILE_NAME = "iterative_data.csv"
############################################################################
+ # Comparisons.
+
+ # The title.
+ COMP_TITLE = "Per Release Performance Comparisons"
+
+ # The pathname prefix for the application.
+ COMP_ROUTES_PATHNAME_PREFIX = "/comparisons/"
+
+ # Default name of downloaded file with selected data.
+ COMP_DOWNLOAD_FILE_NAME = "comparison_data.csv"
+
+ ############################################################################
# Statistics.
# The title.
diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py
index 424456088d..68099987d2 100644
--- a/csit.infra.dash/app/cdash/utils/utils.py
+++ b/csit.infra.dash/app/cdash/utils/utils.py
@@ -17,6 +17,7 @@
import pandas as pd
import dash_bootstrap_components as dbc
+from math import sqrt
from numpy import isnan
from dash import dcc
from datetime import datetime
@@ -391,3 +392,28 @@ def get_list_group_items(
)
return children
+
+def relative_change_stdev(mean1, mean2, std1, std2):
+ """Compute relative standard deviation of change of two values.
+
+ The "1" values are the base for comparison.
+ Results are returned as percentage (and percentual points for stdev).
+ Linearized theory is used, so results are wrong for relatively large stdev.
+
+ :param mean1: Mean of the first number.
+ :param mean2: Mean of the second number.
+ :param std1: Standard deviation estimate of the first number.
+ :param std2: Standard deviation estimate of the second number.
+ :type mean1: float
+ :type mean2: float
+ :type std1: float
+ :type std2: float
+ :returns: Relative change and its stdev.
+ :rtype: float
+ """
+ mean1, mean2 = float(mean1), float(mean2)
+ quotient = mean2 / mean1
+ first = std1 / mean1
+ second = std2 / mean2
+ std = quotient * sqrt(first * first + second * second)
+ return (quotient - 1) * 100, std * 100