diff options
Diffstat (limited to 'csit.infra.dash/app/cdash')
| -rw-r--r-- | csit.infra.dash/app/cdash/__init__.py | 6 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/comparisons/__init__.py | 12 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/comparisons/comparisons.py | 51 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/comparisons/layout.py | 982 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/comparisons/tables.py | 283 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/report/graphs.py | 2 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/routes.py | 1 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/templates/base_layout.jinja2 | 5 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/constants.py | 13 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/utils.py | 26 |
10 files changed, 1380 insertions, 1 deletions
diff --git a/csit.infra.dash/app/cdash/__init__.py b/csit.infra.dash/app/cdash/__init__.py index 77722c78bd..0bc8bf7a19 100644 --- a/csit.infra.dash/app/cdash/__init__.py +++ b/csit.infra.dash/app/cdash/__init__.py @@ -93,6 +93,12 @@ def init_app(): data_iterative=data["iterative"] ) + from .comparisons.comparisons import init_comparisons + app = init_comparisons( + app, + data_iterative=data["iterative"] + ) + return app diff --git a/csit.infra.dash/app/cdash/comparisons/__init__.py b/csit.infra.dash/app/cdash/comparisons/__init__.py new file mode 100644 index 0000000000..f0d52c25b6 --- /dev/null +++ b/csit.infra.dash/app/cdash/comparisons/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/csit.infra.dash/app/cdash/comparisons/comparisons.py b/csit.infra.dash/app/cdash/comparisons/comparisons.py new file mode 100644 index 0000000000..bc42085dc8 --- /dev/null +++ b/csit.infra.dash/app/cdash/comparisons/comparisons.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Instantiate the Report Dash application. +""" + +import dash +import pandas as pd + +from ..utils.constants import Constants as C +from .layout import Layout + + +def init_comparisons( + server, + data_iterative: pd.DataFrame + ) -> dash.Dash: + """Create a Plotly Dash dashboard. + + :param server: Flask server. + :type server: Flask + :returns: Dash app server. + :rtype: Dash + """ + + dash_app = dash.Dash( + server=server, + routes_pathname_prefix=C.COMP_ROUTES_PATHNAME_PREFIX, + external_stylesheets=C.EXTERNAL_STYLESHEETS, + title=C.COMP_TITLE + ) + + layout = Layout( + app=dash_app, + data_iterative=data_iterative, + html_layout_file=C.HTML_LAYOUT_FILE + ) + dash_app.index_string = layout.html_layout + dash_app.layout = layout.add_content() + + return dash_app.server diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py new file mode 100644 index 0000000000..bb4c6dd93c --- /dev/null +++ b/csit.infra.dash/app/cdash/comparisons/layout.py @@ -0,0 +1,982 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Plotly Dash HTML layout override. +""" + +import pandas as pd +import dash_bootstrap_components as dbc + +from flask import Flask +from dash import dcc, html, dash_table, callback_context, no_update, ALL +from dash import Input, Output, State +from dash.exceptions import PreventUpdate +from dash.dash_table.Format import Format, Scheme +from ast import literal_eval + +from ..utils.constants import Constants as C +from ..utils.control_panel import ControlPanel +from ..utils.trigger import Trigger +from ..utils.url_processing import url_decode +from ..utils.utils import generate_options, gen_new_url +from .tables import comparison_table + + +# Control panel partameters and their default values. +CP_PARAMS = { + "dut-val": str(), + "dutver-opt": list(), + "dutver-dis": True, + "dutver-val": str(), + "infra-opt": list(), + "infra-dis": True, + "infra-val": str(), + "core-opt": list(), + "core-val": list(), + "frmsize-opt": list(), + "frmsize-val": list(), + "ttype-opt": list(), + "ttype-val": list(), + "cmp-par-opt": list(), + "cmp-par-dis": True, + "cmp-par-val": str(), + "cmp-val-opt": list(), + "cmp-val-dis": True, + "cmp-val-val": str(), + "normalize-val": list() +} + +# List of comparable parameters. +CMP_PARAMS = { + "dutver": "Release and Version", + "infra": "Infrastructure", + "frmsize": "Frame Size", + "core": "Number of Cores", + "ttype": "Test Type" +} + + +class Layout: + """The layout of the dash app and the callbacks. + """ + + def __init__( + self, + app: Flask, + data_iterative: pd.DataFrame, + html_layout_file: str + ) -> None: + """Initialization: + - save the input parameters, + - prepare data for the control panel, + - read HTML layout file, + + :param app: Flask application running the dash application. + :param data_iterative: Iterative data to be used in comparison tables. + :param html_layout_file: Path and name of the file specifying the HTML + layout of the dash application. + :type app: Flask + :type data_iterative: pandas.DataFrame + :type html_layout_file: str + """ + + # Inputs + self._app = app + self._html_layout_file = html_layout_file + self._data = data_iterative + + # Get structure of tests: + tbs = dict() + cols = [ + "job", "test_id", "test_type", "dut_type", "dut_version", "tg_type", + "release", "passed" + ] + for _, row in self._data[cols].drop_duplicates().iterrows(): + lst_job = row["job"].split("-") + dut = lst_job[1] + dver = f"{row['release']}-{row['dut_version']}" + tbed = "-".join(lst_job[-2:]) + lst_test_id = row["test_id"].split(".") + + suite = lst_test_id[-2].replace("2n1l-", "").replace("1n1l-", "").\ + replace("2n-", "") + test = lst_test_id[-1] + nic = suite.split("-")[0] + for driver in C.DRIVERS: + if driver in test: + drv = driver.replace("-", "_") + test = test.replace(f"{driver}-", "") + break + else: + drv = "dpdk" + infra = "-".join((tbed, nic, drv)) + lst_test = test.split("-") + fsize = lst_test[0] + core = lst_test[1] if lst_test[1] else "8C" + + if tbs.get(dut, None) is None: + tbs[dut] = dict() + if tbs[dut].get(dver, None) is None: + tbs[dut][dver] = dict() + if tbs[dut][dver].get(infra, None) is None: + tbs[dut][dver][infra] = dict() + tbs[dut][dver][infra]["core"] = list() + tbs[dut][dver][infra]["fsize"] = list() + tbs[dut][dver][infra]["ttype"] = list() + if core.upper() not in tbs[dut][dver][infra]["core"]: + tbs[dut][dver][infra]["core"].append(core.upper()) + if fsize.upper() not in tbs[dut][dver][infra]["fsize"]: + tbs[dut][dver][infra]["fsize"].append(fsize.upper()) + if row["test_type"] == "mrr": + if "MRR" not in tbs[dut][dver][infra]["ttype"]: + tbs[dut][dver][infra]["ttype"].append("MRR") + elif row["test_type"] == "ndrpdr": + if "NDR" not in tbs[dut][dver][infra]["ttype"]: + tbs[dut][dver][infra]["ttype"].extend(("NDR", "PDR", )) + elif row["test_type"] == "hoststack" and \ + row["tg_type"] in ("iperf", "vpp"): + if "BPS" not in tbs[dut][dver][infra]["ttype"]: + tbs[dut][dver][infra]["ttype"].append("BPS") + elif row["test_type"] == "hoststack" and row["tg_type"] == "ab": + if "CPS" not in tbs[dut][dver][infra]["ttype"]: + tbs[dut][dver][infra]["ttype"].extend(("CPS", "RPS", )) + self._tbs = tbs + + # Read from files: + self._html_layout = str() + try: + with open(self._html_layout_file, "r") as file_read: + self._html_layout = file_read.read() + except IOError as err: + raise RuntimeError( + f"Not possible to open the file {self._html_layout_file}\n{err}" + ) + + # Callbacks: + if self._app is not None and hasattr(self, "callbacks"): + self.callbacks(self._app) + + @property + def html_layout(self): + return self._html_layout + + def add_content(self): + """Top level method which generated the web page. + + It generates: + - Store for user input data, + - Navigation bar, + - Main area with control panel and ploting area. + + If no HTML layout is provided, an error message is displayed instead. + + :returns: The HTML div with the whole page. + :rtype: html.Div + """ + + if self.html_layout and self._tbs: + return html.Div( + id="div-main", + className="small", + children=[ + dbc.Row( + id="row-navbar", + class_name="g-0", + children=[ + self._add_navbar() + ] + ), + dbc.Row( + id="row-main", + class_name="g-0", + children=[ + dcc.Store(id="store-control-panel"), + dcc.Store(id="store-selected"), + dcc.Location(id="url", refresh=False), + self._add_ctrl_col(), + self._add_plotting_col() + ] + ) + ] + ) + else: + return html.Div( + id="div-main-error", + children=[ + dbc.Alert( + [ + "An Error Occured" + ], + color="danger" + ) + ] + ) + + def _add_navbar(self): + """Add nav element with navigation panel. It is placed on the top. + + :returns: Navigation bar. + :rtype: dbc.NavbarSimple + """ + return dbc.NavbarSimple( + id="navbarsimple-main", + children=[ + dbc.NavItem( + dbc.NavLink( + C.COMP_TITLE, + disabled=True, + external_link=True, + href="#" + ) + ) + ], + brand=C.BRAND, + brand_href="/", + brand_external_link=True, + class_name="p-2", + fluid=True + ) + + def _add_ctrl_col(self) -> dbc.Col: + """Add column with controls. It is placed on the left side. + + :returns: Column with the control panel. + :rtype: dbc.Col + """ + return dbc.Col([ + html.Div( + children=self._add_ctrl_panel(), + className="sticky-top" + ) + ]) + + def _add_plotting_col(self) -> dbc.Col: + """Add column with plots. It is placed on the right side. + + :returns: Column with plots. + :rtype: dbc.Col + """ + return dbc.Col( + id="col-plotting-area", + children=[ + dbc.Spinner( + children=[ + dbc.Row( + id="plotting-area", + class_name="g-0 p-0", + children=[ + C.PLACEHOLDER + ] + ) + ] + ) + ], + width=9 + ) + + def _add_ctrl_panel(self) -> list: + """Add control panel. + + :returns: Control panel. + :rtype: list + """ + + reference = [ + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("DUT"), + dbc.Select( + id={"type": "ctrl-dd", "index": "dut"}, + placeholder="Select a Device under Test...", + options=sorted( + [ + {"label": k, "value": k} \ + for k in self._tbs.keys() + ], + key=lambda d: d["label"] + ) + ) + ], + size="sm" + ) + ] + ), + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("CSIT and DUT Version"), + dbc.Select( + id={"type": "ctrl-dd", "index": "dutver"}, + placeholder="Select a CSIT and DUT Version...") + ], + size="sm" + ) + ] + ), + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("Infra"), + dbc.Select( + id={"type": "ctrl-dd", "index": "infra"}, + placeholder=\ + "Select a Physical Test Bed Topology..." + ) + ], + size="sm" + ) + ] + ), + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("Frame Size"), + dbc.Checklist( + id={"type": "ctrl-cl", "index": "frmsize"}, + inline=True, + class_name="ms-2" + ) + ], + style={"align-items": "center"}, + size="sm" + ) + ] + ), + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("Number of Cores"), + dbc.Checklist( + id={"type": "ctrl-cl", "index": "core"}, + inline=True, + class_name="ms-2" + ) + ], + style={"align-items": "center"}, + size="sm" + ) + ] + ), + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("Test Type"), + dbc.Checklist( + id={"type": "ctrl-cl", "index": "ttype"}, + inline=True, + class_name="ms-2" + ) + ], + style={"align-items": "center"}, + size="sm" + ) + ] + ) + ] + + compare = [ + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("Parameter"), + dbc.Select( + id={"type": "ctrl-dd", "index": "cmpprm"}, + placeholder="Select a Parameter..." + ) + ], + size="sm" + ) + ] + ), + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + [ + dbc.InputGroupText("Value"), + dbc.Select( + id={"type": "ctrl-dd", "index": "cmpval"}, + placeholder="Select a Value..." + ) + ], + size="sm" + ) + ] + ) + ] + + normalize = [ + dbc.Row( + class_name="g-0 p-1", + children=[ + dbc.InputGroup( + dbc.Checklist( + id="normalize", + options=[{ + "value": "normalize", + "label": "Normalize to 2GHz CPU frequency" + }], + value=[], + inline=True, + class_name="ms-2" + ), + style={"align-items": "center"}, + size="sm" + ) + ] + ) + ] + + return [ + dbc.Row( + dbc.Card( + [ + dbc.CardHeader( + html.H5("Reference Value") + ), + dbc.CardBody( + children=reference, + class_name="g-0 p-0" + ) + ], + color="secondary", + outline=True + ), + class_name="g-0 p-1" + ), + dbc.Row( + dbc.Card( + [ + dbc.CardHeader( + html.H5("Compared Value") + ), + dbc.CardBody( + children=compare, + class_name="g-0 p-0" + ) + ], + color="secondary", + outline=True + ), + class_name="g-0 p-1" + ), + dbc.Row( + dbc.Card( + [ + dbc.CardHeader( + html.H5("Normalization") + ), + dbc.CardBody( + children=normalize, + class_name="g-0 p-0" + ) + ], + color="secondary", + outline=True + ), + class_name="g-0 p-1" + ) + ] + + def _get_plotting_area( + self, + selected: dict, + url: str, + normalize: bool + ) -> list: + """Generate the plotting area with all its content. + + :param selected: Selected parameters of tests. + :param normalize: If true, the values in tables are normalized. + :param url: URL to be displayed in the modal window. + :type selected: dict + :type normalize: bool + :type url: str + :returns: List of rows with elements to be displayed in the plotting + area. + :rtype: list + """ + + title, df = comparison_table(self._data, selected, normalize) + + if df.empty: + return dbc.Row( + dbc.Col( + children=dbc.Alert( + "No data for comparison.", + color="danger" + ), + class_name="g-0 p-1", + ), + class_name="g-0 p-0" + ) + + cols = list() + for idx, col in enumerate(df.columns): + if idx == 0: + cols.append({ + "name": ["", col], + "id": col, + "deletable": False, + "selectable": False, + "type": "text" + }) + else: + l_col = col.rsplit(" ", 2) + cols.append({ + "name": [l_col[0], " ".join(l_col[-2:])], + "id": col, + "deletable": False, + "selectable": False, + "type": "numeric", + "format": Format(precision=2, scheme=Scheme.fixed) + }) + + return [ + dbc.Row( + children=html.H5(title), + class_name="g-0 p-1" + ), + dbc.Row( + children=[ + dbc.Col( + children=dash_table.DataTable( + columns=cols, + data=df.to_dict("records"), + merge_duplicate_headers=True, + editable=True, + filter_action="native", + sort_action="native", + sort_mode="multi", + selected_columns=[], + selected_rows=[], + page_action="none", + style_cell={"textAlign": "right"}, + style_cell_conditional=[{ + "if": {"column_id": "Test Name"}, + "textAlign": "left" + }] + ), + class_name="g-0 p-1" + ) + ], + class_name="g-0 p-0" + ), + dbc.Row( + [ + dbc.Col([html.Div( + [ + dbc.Button( + id="plot-btn-url", + children="Show URL", + class_name="me-1", + color="info", + style={ + "text-transform": "none", + "padding": "0rem 1rem" + } + ), + dbc.Modal( + [ + dbc.ModalHeader(dbc.ModalTitle("URL")), + dbc.ModalBody(url) + ], + id="plot-mod-url", + size="xl", + is_open=False, + scrollable=True + ), + dbc.Button( + id="plot-btn-download", + children="Download Data", + class_name="me-1", + color="info", + style={ + "text-transform": "none", + "padding": "0rem 1rem" + } + ), + dcc.Download(id="download-iterative-data") + ], + className=\ + "d-grid gap-0 d-md-flex justify-content-md-end" + )]) + ], + class_name="g-0 p-0" + ), + dbc.Row( + children=C.PLACEHOLDER, + class_name="g-0 p-1" + ) + ] + + def callbacks(self, app): + """Callbacks for the whole application. + + :param app: The application. + :type app: Flask + """ + + @app.callback( + [ + Output("store-control-panel", "data"), + Output("store-selected", "data"), + Output("plotting-area", "children"), + Output({"type": "ctrl-dd", "index": "dut"}, "value"), + Output({"type": "ctrl-dd", "index": "dutver"}, "options"), + Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"), + Output({"type": "ctrl-dd", "index": "dutver"}, "value"), + Output({"type": "ctrl-dd", "index": "infra"}, "options"), + Output({"type": "ctrl-dd", "index": "infra"}, "disabled"), + Output({"type": "ctrl-dd", "index": "infra"}, "value"), + Output({"type": "ctrl-cl", "index": "core"}, "options"), + Output({"type": "ctrl-cl", "index": "core"}, "value"), + Output({"type": "ctrl-cl", "index": "frmsize"}, "options"), + Output({"type": "ctrl-cl", "index": "frmsize"}, "value"), + Output({"type": "ctrl-cl", "index": "ttype"}, "options"), + Output({"type": "ctrl-cl", "index": "ttype"}, "value"), + Output({"type": "ctrl-dd", "index": "cmpprm"}, "options"), + Output({"type": "ctrl-dd", "index": "cmpprm"}, "disabled"), + Output({"type": "ctrl-dd", "index": "cmpprm"}, "value"), + Output({"type": "ctrl-dd", "index": "cmpval"}, "options"), + Output({"type": "ctrl-dd", "index": "cmpval"}, "disabled"), + Output({"type": "ctrl-dd", "index": "cmpval"}, "value"), + Output("normalize", "value") + ], + [ + State("store-control-panel", "data"), + State("store-selected", "data") + ], + [ + Input("url", "href"), + Input("normalize", "value"), + Input({"type": "ctrl-dd", "index": ALL}, "value"), + Input({"type": "ctrl-cl", "index": ALL}, "value"), + Input({"type": "ctrl-btn", "index": ALL}, "n_clicks") + ] + ) + def _update_application( + control_panel: dict, + selected: dict, + href: str, + normalize: list, + *_ + ) -> tuple: + """Update the application when the event is detected. + """ + + ctrl_panel = ControlPanel(CP_PARAMS, control_panel) + + if selected is None: + selected = { + "reference": { + "set": False, + }, + "compare": { + "set": False, + } + } + + # Parse the url: + parsed_url = url_decode(href) + if parsed_url: + url_params = parsed_url["params"] + else: + url_params = None + + on_draw = False + plotting_area = no_update + + trigger = Trigger(callback_context.triggered) + if trigger.type == "url" and url_params: + process_url = False + try: + selected = literal_eval(url_params["selected"][0]) + r_sel = selected["reference"]["selection"] + c_sel = selected["compare"] + normalize = literal_eval(url_params["norm"][0]) + process_url = bool( + (selected["reference"]["set"] == True) and + (c_sel["set"] == True) + ) + except (KeyError, IndexError): + pass + if process_url: + ctrl_panel.set({ + "dut-val": r_sel["dut"], + "dutver-opt": generate_options( + self._tbs[r_sel["dut"]].keys() + ), + "dutver-dis": False, + "dutver-val": r_sel["dutver"], + "infra-opt": generate_options( + self._tbs[r_sel["dut"]][r_sel["dutver"]].keys() + ), + "infra-dis": False, + "infra-val": r_sel["infra"], + "core-opt": generate_options( + self._tbs[r_sel["dut"]][r_sel["dutver"]]\ + [r_sel["infra"]]["core"] + ), + "core-val": r_sel["core"], + "frmsize-opt": generate_options( + self._tbs[r_sel["dut"]][r_sel["dutver"]]\ + [r_sel["infra"]]["fsize"] + ), + "frmsize-val": r_sel["frmsize"], + "ttype-opt": generate_options( + self._tbs[r_sel["dut"]][r_sel["dutver"]]\ + [r_sel["infra"]]["ttype"] + ), + "ttype-val": r_sel["ttype"], + "normalize-val": normalize + }) + opts = list() + for itm, label in CMP_PARAMS.items(): + if len(ctrl_panel.get(f"{itm}-opt")) > 1: + opts.append({"label": label, "value": itm}) + ctrl_panel.set({ + "cmp-par-opt": opts, + "cmp-par-dis": False, + "cmp-par-val": c_sel["parameter"] + }) + opts = list() + for itm in ctrl_panel.get(f"{c_sel['parameter']}-opt"): + set_val = ctrl_panel.get(f"{c_sel['parameter']}-val") + if isinstance(set_val, list): + if itm["value"] not in set_val: + opts.append(itm) + else: + if itm["value"] != set_val: + opts.append(itm) + ctrl_panel.set({ + "cmp-val-opt": opts, + "cmp-val-dis": False, + "cmp-val-val": c_sel["value"] + }) + on_draw = True + elif trigger.type == "normalize": + ctrl_panel.set({"normalize-val": normalize}) + on_draw = True + elif trigger.type == "ctrl-dd": + if trigger.idx == "dut": + try: + opts = generate_options(self._tbs[trigger.value].keys()) + disabled = False + except KeyError: + opts = list() + disabled = True + ctrl_panel.set({ + "dut-val": trigger.value, + "dutver-opt": opts, + "dutver-dis": disabled, + "dutver-val": str(), + "infra-opt": list(), + "infra-dis": True, + "infra-val": str(), + "core-opt": list(), + "core-val": list(), + "frmsize-opt": list(), + "frmsize-val": list(), + "ttype-opt": list(), + "ttype-val": list(), + "cmp-par-opt": list(), + "cmp-par-dis": True, + "cmp-par-val": str(), + "cmp-val-opt": list(), + "cmp-val-dis": True, + "cmp-val-val": str() + }) + elif trigger.idx == "dutver": + try: + dut = ctrl_panel.get("dut-val") + dver = self._tbs[dut][trigger.value] + opts = generate_options(dver.keys()) + disabled = False + except KeyError: + opts = list() + disabled = True + ctrl_panel.set({ + "dutver-val": trigger.value, + "infra-opt": opts, + "infra-dis": disabled, + "infra-val": str(), + "core-opt": list(), + "core-val": list(), + "frmsize-opt": list(), + "frmsize-val": list(), + "ttype-opt": list(), + "ttype-val": list(), + "cmp-par-opt": list(), + "cmp-par-dis": True, + "cmp-par-val": str(), + "cmp-val-opt": list(), + "cmp-val-dis": True, + "cmp-val-val": str() + }) + elif trigger.idx == "infra": + dut = ctrl_panel.get("dut-val") + dver = ctrl_panel.get("dutver-val") + if all((dut, dver, trigger.value, )): + driver = self._tbs[dut][dver][trigger.value] + ctrl_panel.set({ + "infra-val": trigger.value, + "core-opt": generate_options(driver["core"]), + "core-val": list(), + "frmsize-opt": generate_options(driver["fsize"]), + "frmsize-val": list(), + "ttype-opt": generate_options(driver["ttype"]), + "ttype-val": list(), + "cmp-par-opt": list(), + "cmp-par-dis": True, + "cmp-par-val": str(), + "cmp-val-opt": list(), + "cmp-val-dis": True, + "cmp-val-val": str() + }) + elif trigger.idx == "cmpprm": + value = trigger.value + opts = list() + for itm in ctrl_panel.get(f"{value}-opt"): + set_val = ctrl_panel.get(f"{value}-val") + if isinstance(set_val, list): + if itm["value"] not in set_val: + opts.append(itm) + else: + if itm["value"] != set_val: + opts.append(itm) + ctrl_panel.set({ + "cmp-par-val": value, + "cmp-val-opt": opts, + "cmp-val-dis": False, + "cmp-val-val": str() + }) + elif trigger.idx == "cmpval": + ctrl_panel.set({"cmp-val-val": trigger.value}) + selected["reference"] = { + "set": True, + "selection": { + "dut": ctrl_panel.get("dut-val"), + "dutver": ctrl_panel.get("dutver-val"), + "infra": ctrl_panel.get("infra-val"), + "core": ctrl_panel.get("core-val"), + "frmsize": ctrl_panel.get("frmsize-val"), + "ttype": ctrl_panel.get("ttype-val") + } + } + selected["compare"] = { + "set": True, + "parameter": ctrl_panel.get("cmp-par-val"), + "value": trigger.value + } + on_draw = True + elif trigger.type == "ctrl-cl": + ctrl_panel.set({f"{trigger.idx}-val": trigger.value}) + if all((ctrl_panel.get("core-val"), + ctrl_panel.get("frmsize-val"), + ctrl_panel.get("ttype-val"), )): + opts = list() + for itm, label in CMP_PARAMS.items(): + if len(ctrl_panel.get(f"{itm}-opt")) > 1: + if isinstance(ctrl_panel.get(f"{itm}-val"), list): + if len(ctrl_panel.get(f"{itm}-opt")) == \ + len(ctrl_panel.get(f"{itm}-val")): + continue + opts.append({"label": label, "value": itm}) + ctrl_panel.set({ + "cmp-par-opt": opts, + "cmp-par-dis": False, + "cmp-par-val": str(), + "cmp-val-opt": list(), + "cmp-val-dis": True, + "cmp-val-val": str() + }) + else: + ctrl_panel.set({ + "cmp-par-opt": list(), + "cmp-par-dis": True, + "cmp-par-val": str(), + "cmp-val-opt": list(), + "cmp-val-dis": True, + "cmp-val-val": str() + }) + + if all((on_draw, selected["reference"]["set"], + selected["compare"]["set"], )): + plotting_area = self._get_plotting_area( + selected=selected, + normalize=bool(normalize), + url=gen_new_url( + parsed_url, + params={"selected": selected, "norm": normalize} + ) + ) + + ret_val = [ctrl_panel.panel, selected, plotting_area] + ret_val.extend(ctrl_panel.values) + return ret_val + + @app.callback( + Output("plot-mod-url", "is_open"), + Input("plot-btn-url", "n_clicks"), + State("plot-mod-url", "is_open") + ) + def toggle_plot_mod_url(n, is_open): + """Toggle the modal window with url. + """ + if n: + return not is_open + return is_open + + @app.callback( + Output("download-iterative-data", "data"), + State("store-selected", "data"), + State("normalize", "value"), + Input("plot-btn-download", "n_clicks"), + prevent_initial_call=True + ) + def _download_trending_data(selected: dict, normalize: list, _: int): + """Download the data. + + :param selected: List of tests selected by user stored in the + browser. + :param normalize: If set, the data is normalized to 2GHz CPU + frequency. + :type selected: list + :type normalize: list + :returns: dict of data frame content (base64 encoded) and meta data + used by the Download component. + :rtype: dict + """ + + if not selected: + raise PreventUpdate + + _, table = comparison_table(self._data, selected, normalize, "csv") + + return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME) diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py new file mode 100644 index 0000000000..14d5d552af --- /dev/null +++ b/csit.infra.dash/app/cdash/comparisons/tables.py @@ -0,0 +1,283 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The comparison tables. +""" + +import pandas as pd + +from numpy import mean, std +from copy import deepcopy +from ..utils.constants import Constants as C +from ..utils.utils import relative_change_stdev + + +def select_comparison_data( + data: pd.DataFrame, + selected: dict, + normalize: bool=False + ) -> pd.DataFrame: + """Select data for a comparison table. + + :param data: Data to be filtered for the comparison table. + :param selected: A dictionary with parameters and their values selected by + the user. + :param normalize: If True, the data is normalized to CPU frequency + Constants.NORM_FREQUENCY. + :type data: pandas.DataFrame + :type selected: dict + :type normalize: bool + :returns: A data frame with selected data. + :rtype: pandas.DataFrame + """ + + def _calculate_statistics( + data_in: pd.DataFrame, + ttype: str, + drv: str, + norm_factor: float + ) -> pd.DataFrame: + """Calculates mean value and standard deviation for provided data. + + :param data_in: Input data for calculations. + :param ttype: The test type. + :param drv: The driver. + :param norm_factor: The data normalization factor. + :type data_in: pandas.DataFrame + :type ttype: str + :type drv: str + :type norm_factor: float + :returns: A pandas dataframe with: test name, mean value, standard + deviation and unit. + :rtype: pandas.DataFrame + """ + d_data = { + "name": list(), + "mean": list(), + "stdev": list(), + "unit": list() + } + for itm in data_in["test_id"].unique().tolist(): + itm_lst = itm.split(".") + test = itm_lst[-1].rsplit("-", 1)[0] + df = data_in.loc[(data_in["test_id"] == itm)] + l_df = df[C.VALUE_ITER[ttype]].to_list() + if len(l_df) and isinstance(l_df[0], list): + tmp_df = list() + for l_itm in l_df: + tmp_df.extend(l_itm) + l_df = tmp_df + d_data["name"].append(f"{test.replace(f'{drv}-', '')}-{ttype}") + d_data["mean"].append(int(mean(l_df) * norm_factor)) + d_data["stdev"].append(int(std(l_df) * norm_factor)) + d_data["unit"].append(df[C.UNIT[ttype]].to_list()[0]) + return pd.DataFrame(d_data) + + lst_df = list() + for itm in selected: + if itm["ttype"] in ("NDR", "PDR"): + test_type = "ndrpdr" + else: + test_type = itm["ttype"].lower() + + dutver = itm["dutver"].split("-", 1) # 0 -> release, 1 -> dut version + tmp_df = pd.DataFrame(data.loc[( + (data["passed"] == True) & + (data["dut_type"] == itm["dut"]) & + (data["dut_version"] == dutver[1]) & + (data["test_type"] == test_type) & + (data["release"] == dutver[0]) + )]) + + drv = "" if itm["driver"] == "dpdk" else itm["driver"].replace("_", "-") + core = str() if itm["dut"] == "trex" else itm["core"].lower() + reg_id = \ + f"^.*[.|-]{itm['nic']}.*{itm['frmsize'].lower()}-{core}-{drv}.*$" + tmp_df = tmp_df[ + (tmp_df.job.str.endswith(itm["tbed"])) & + (tmp_df.test_id.str.contains(reg_id, regex=True)) + ] + if itm["driver"] == "dpdk": + for drv in C.DRIVERS: + tmp_df.drop( + tmp_df[tmp_df.test_id.str.contains(f"-{drv}-")].index, + inplace=True + ) + + # Change the data type from ndrpdr to one of ("NDR", "PDR") + if test_type == "ndrpdr": + tmp_df = tmp_df.assign(test_type=itm["ttype"].lower()) + + if not tmp_df.empty: + tmp_df = _calculate_statistics( + tmp_df, + itm["ttype"].lower(), + itm["driver"], + C.NORM_FREQUENCY / C.FREQUENCY[itm["tbed"]] if normalize else 1 + ) + + lst_df.append(tmp_df) + + if len(lst_df) == 1: + df = lst_df[0] + elif len(lst_df) > 1: + df = pd.concat( + lst_df, + ignore_index=True, + copy=False + ) + else: + df = pd.DataFrame() + + return df + + +def comparison_table( + data: pd.DataFrame, + selected: dict, + normalize: bool, + format: str="html" + ) -> tuple: + """Generate a comparison table. + + :param data: Iterative data for the comparison table. + :param selected: A dictionary with parameters and their values selected by + the user. + :param normalize: If True, the data is normalized to CPU frequency + Constants.NORM_FREQUENCY. + :param format: The output format of the table: + - html: To be displayed on html page, the values are shown in millions + of the unit. + - csv: To be downloaded as a CSV file the values are stored in base + units. + :type data: pandas.DataFrame + :type selected: dict + :type normalize: bool + :type format: str + :returns: A tuple with the tabe title and the comparison table. + :rtype: tuple[str, pandas.DataFrame] + """ + + def _create_selection(sel: dict) -> list: + """Transform the complex dictionary with user selection to list + of simple items. + + :param sel: A complex dictionary with user selection. + :type sel: dict + :returns: A list of simple items. + :rtype: list + """ + l_infra = sel["infra"].split("-") + selection = list() + for core in sel["core"]: + for fsize in sel["frmsize"]: + for ttype in sel["ttype"]: + selection.append({ + "dut": sel["dut"], + "dutver": sel["dutver"], + "tbed": f"{l_infra[0]}-{l_infra[1]}", + "nic": l_infra[2], + "driver": l_infra[-1].replace("_", "-"), + "core": core, + "frmsize": fsize, + "ttype": ttype + }) + return selection + + unit_factor, s_unit_factor = (1e6, "M") if format == "html" else (1, str()) + + r_sel = deepcopy(selected["reference"]["selection"]) + c_params = selected["compare"] + r_selection = _create_selection(r_sel) + + # Create Table title and titles of columns with data + params = list(r_sel) + params.remove(c_params["parameter"]) + lst_title = list() + for param in params: + value = r_sel[param] + if isinstance(value, list): + lst_title.append("|".join(value)) + else: + lst_title.append(value) + title = "Comparison for: " + "-".join(lst_title) + r_name = r_sel[c_params["parameter"]] + if isinstance(r_name, list): + r_name = "|".join(r_name) + c_name = c_params["value"] + + # Select reference data + r_data = select_comparison_data(data, r_selection, normalize) + + # Select compare data + c_sel = deepcopy(selected["reference"]["selection"]) + if c_params["parameter"] in ("core", "frmsize", "ttype"): + c_sel[c_params["parameter"]] = [c_params["value"], ] + else: + c_sel[c_params["parameter"]] = c_params["value"] + + c_selection = _create_selection(c_sel) + c_data = select_comparison_data(data, c_selection, normalize) + + if r_data.empty or c_data.empty: + return str(), pd.DataFrame() + + l_name, l_r_mean, l_r_std, l_c_mean, l_c_std, l_rc_mean, l_rc_std, unit = \ + list(), list(), list(), list(), list(), list(), list(), set() + for _, row in r_data.iterrows(): + if c_params["parameter"] in ("core", "frmsize", "ttype"): + l_cmp = row["name"].split("-") + if c_params["parameter"] == "core": + c_row = c_data[ + (c_data.name.str.contains(l_cmp[0])) & + (c_data.name.str.contains("-".join(l_cmp[2:]))) + ] + elif c_params["parameter"] == "frmsize": + c_row = c_data[c_data.name.str.contains("-".join(l_cmp[1:]))] + elif c_params["parameter"] == "ttype": + regex = r"^" + f"{'-'.join(l_cmp[:-1])}" + r"-.{3}$" + c_row = c_data[c_data.name.str.contains(regex, regex=True)] + else: + c_row = c_data[c_data["name"] == row["name"]] + if not c_row.empty: + unit.add(f"{s_unit_factor}{row['unit']}") + r_mean = row["mean"] + r_std = row["stdev"] + c_mean = c_row["mean"].values[0] + c_std = c_row["stdev"].values[0] + l_name.append(row["name"]) + l_r_mean.append(r_mean / unit_factor) + l_r_std.append(r_std / unit_factor) + l_c_mean.append(c_mean / unit_factor) + l_c_std.append(c_std / unit_factor) + delta, d_stdev = relative_change_stdev(r_mean, c_mean, r_std, c_std) + l_rc_mean.append(delta) + l_rc_std.append(d_stdev) + + s_unit = "|".join(unit) + df_cmp = pd.DataFrame.from_dict({ + "Test Name": l_name, + f"{r_name} Mean [{s_unit}]": l_r_mean, + f"{r_name} Stdev [{s_unit}]": l_r_std, + f"{c_name} Mean [{s_unit}]": l_c_mean, + f"{c_name} Stdev [{s_unit}]": l_c_std, + "Relative Change Mean [%]": l_rc_mean, + "Relative Change Stdev [%]": l_rc_std + }) + df_cmp.sort_values( + by="Relative Change Mean [%]", + ascending=False, + inplace=True + ) + + return (title, df_cmp) diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py index 870f16a533..9d10efc4f0 100644 --- a/csit.infra.dash/app/cdash/report/graphs.py +++ b/csit.infra.dash/app/cdash/report/graphs.py @@ -80,7 +80,7 @@ def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict, :param data: Data frame with iterative data. :param sel: Selected tests. :param layout: Layout of plot.ly graph. - :param normalize: If True, the data is normalized to CPU frquency + :param normalize: If True, the data is normalized to CPU frequency Constants.NORM_FREQUENCY. :param data: pandas.DataFrame :param sel: dict diff --git a/csit.infra.dash/app/cdash/routes.py b/csit.infra.dash/app/cdash/routes.py index 1906d00ee5..71c13edd6d 100644 --- a/csit.infra.dash/app/cdash/routes.py +++ b/csit.infra.dash/app/cdash/routes.py @@ -30,6 +30,7 @@ def home(): description=C.DESCRIPTION, trending_title=C.TREND_TITLE, report_title=C.REPORT_TITLE, + comp_title=C.COMP_TITLE, stats_title=C.STATS_TITLE, news_title=C.NEWS_TITLE ) diff --git a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 b/csit.infra.dash/app/cdash/templates/base_layout.jinja2 index ff31c72178..b799bda3a1 100644 --- a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 +++ b/csit.infra.dash/app/cdash/templates/base_layout.jinja2 @@ -31,6 +31,11 @@ </a> </p> <p> + <a href="/comparisons/" class="btn btn-primary fw-bold w-25"> + {{ comp_title }} + </a> + </p> + <p> <a href="/stats/" class="btn btn-primary fw-bold w-25"> {{ stats_title }} </a> diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py index 12d7ee5702..1aedb9b96f 100644 --- a/csit.infra.dash/app/cdash/utils/constants.py +++ b/csit.infra.dash/app/cdash/utils/constants.py @@ -142,6 +142,7 @@ class Constants: NORM_FREQUENCY = 2.0 # [GHz] FREQUENCY = { # [GHz] + "1n-aws": 1.000, "2n-aws": 1.000, "2n-dnv": 2.000, "2n-clx": 2.300, @@ -291,6 +292,18 @@ class Constants: REPORT_DOWNLOAD_FILE_NAME = "iterative_data.csv" ############################################################################ + # Comparisons. + + # The title. + COMP_TITLE = "Per Release Performance Comparisons" + + # The pathname prefix for the application. + COMP_ROUTES_PATHNAME_PREFIX = "/comparisons/" + + # Default name of downloaded file with selected data. + COMP_DOWNLOAD_FILE_NAME = "comparison_data.csv" + + ############################################################################ # Statistics. # The title. diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py index 424456088d..68099987d2 100644 --- a/csit.infra.dash/app/cdash/utils/utils.py +++ b/csit.infra.dash/app/cdash/utils/utils.py @@ -17,6 +17,7 @@ import pandas as pd import dash_bootstrap_components as dbc +from math import sqrt from numpy import isnan from dash import dcc from datetime import datetime @@ -391,3 +392,28 @@ def get_list_group_items( ) return children + +def relative_change_stdev(mean1, mean2, std1, std2): + """Compute relative standard deviation of change of two values. + + The "1" values are the base for comparison. + Results are returned as percentage (and percentual points for stdev). + Linearized theory is used, so results are wrong for relatively large stdev. + + :param mean1: Mean of the first number. + :param mean2: Mean of the second number. + :param std1: Standard deviation estimate of the first number. + :param std2: Standard deviation estimate of the second number. + :type mean1: float + :type mean2: float + :type std1: float + :type std2: float + :returns: Relative change and its stdev. + :rtype: float + """ + mean1, mean2 = float(mean1), float(mean2) + quotient = mean2 / mean1 + first = std1 / mean1 + second = std2 / mean2 + std = quotient * sqrt(first * first + second * second) + return (quotient - 1) * 100, std * 100 |