diff options
| -rw-r--r-- | resources/tools/dash/app/pal/news/layout.py | 240 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/news/news.py | 2 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/news/tables.py | 13 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/stats/layout.py | 275 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/stats/stats.py | 2 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/utils/constants.py | 6 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/utils/utils.py | 154 |
7 files changed, 413 insertions, 279 deletions
diff --git a/resources/tools/dash/app/pal/news/layout.py b/resources/tools/dash/app/pal/news/layout.py index 527710fa4b..91710ef3d9 100644 --- a/resources/tools/dash/app/pal/news/layout.py +++ b/resources/tools/dash/app/pal/news/layout.py @@ -28,7 +28,9 @@ from copy import deepcopy from ..data.data import Data from ..utils.constants import Constants as C -from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url +from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url, \ + get_ttypes, get_cadences, get_test_beds, get_job, generate_options, \ + set_job_params from ..utils.url_processing import url_decode from ..data.data import Data from .tables import table_news @@ -43,7 +45,7 @@ class Layout: """Initialization: - save the input parameters, - read and pre-process the data, - - prepare data fro the control panel, + - prepare data for the control panel, - read HTML layout file, - read tooltips from the tooltip file. @@ -76,7 +78,7 @@ class Layout: # Prepare information for the control panel: jobs = sorted(list(df_tst_info["job"].unique())) - job_info = { + d_job_info = { "job": list(), "dut": list(), "ttype": list(), @@ -85,14 +87,14 @@ class Layout: } for job in jobs: lst_job = job.split("-") - job_info["job"].append(job) - job_info["dut"].append(lst_job[1]) - job_info["ttype"].append(lst_job[3]) - job_info["cadence"].append(lst_job[4]) - job_info["tbed"].append("-".join(lst_job[-2:])) - self.df_job_info = pd.DataFrame.from_dict(job_info) + d_job_info["job"].append(job) + d_job_info["dut"].append(lst_job[1]) + d_job_info["ttype"].append(lst_job[3]) + d_job_info["cadence"].append(lst_job[4]) + d_job_info["tbed"].append("-".join(lst_job[-2:])) + self.job_info = pd.DataFrame.from_dict(d_job_info) - self._default = self._set_job_params(C.NEWS_DEFAULT_JOB) + self._default = set_job_params(self.job_info, C.NEWS_DEFAULT_JOB) # Pre-process the data: @@ -270,126 +272,6 @@ class Layout: def default(self) -> dict: return self._default - def _get_duts(self) -> list: - """Get the list of DUTs from the pre-processed information about jobs. - - :returns: Alphabeticaly sorted list of DUTs. - :rtype: list - """ - return sorted(list(self.df_job_info["dut"].unique())) - - def _get_ttypes(self, dut: str) -> list: - """Get the list of test types from the pre-processed information about - jobs. - - :param dut: The DUT for which the list of test types will be populated. - :type dut: str - :returns: Alphabeticaly sorted list of test types. - :rtype: list - """ - return sorted(list(self.df_job_info.loc[( - self.df_job_info["dut"] == dut - )]["ttype"].unique())) - - def _get_cadences(self, dut: str, ttype: str) -> list: - """Get the list of cadences from the pre-processed information about - jobs. - - :param dut: The DUT for which the list of cadences will be populated. - :param ttype: The test type for which the list of cadences will be - populated. - :type dut: str - :type ttype: str - :returns: Alphabeticaly sorted list of cadences. - :rtype: list - """ - return sorted(list(self.df_job_info.loc[( - (self.df_job_info["dut"] == dut) & - (self.df_job_info["ttype"] == ttype) - )]["cadence"].unique())) - - def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list: - """Get the list of test beds from the pre-processed information about - jobs. - - :param dut: The DUT for which the list of test beds will be populated. - :param ttype: The test type for which the list of test beds will be - populated. - :param cadence: The cadence for which the list of test beds will be - populated. - :type dut: str - :type ttype: str - :type cadence: str - :returns: Alphabeticaly sorted list of test beds. - :rtype: list - """ - return sorted(list(self.df_job_info.loc[( - (self.df_job_info["dut"] == dut) & - (self.df_job_info["ttype"] == ttype) & - (self.df_job_info["cadence"] == cadence) - )]["tbed"].unique())) - - def _get_job(self, dut, ttype, cadence, testbed): - """Get the name of a job defined by dut, ttype, cadence, test bed. - Input information comes from the control panel. - - :param dut: The DUT for which the job name will be created. - :param ttype: The test type for which the job name will be created. - :param cadence: The cadence for which the job name will be created. - :param testbed: The test bed for which the job name will be created. - :type dut: str - :type ttype: str - :type cadence: str - :type testbed: str - :returns: Job name. - :rtype: str - """ - return self.df_job_info.loc[( - (self.df_job_info["dut"] == dut) & - (self.df_job_info["ttype"] == ttype) & - (self.df_job_info["cadence"] == cadence) & - (self.df_job_info["tbed"] == testbed) - )]["job"].item() - - @staticmethod - def _generate_options(opts: list) -> list: - """Return list of options for radio items in control panel. The items in - the list are dictionaries with keys "label" and "value". - - :params opts: List of options (str) to be used for the generated list. - :type opts: list - :returns: List of options (dict). - :rtype: list - """ - return [{"label": i, "value": i} for i in opts] - - def _set_job_params(self, job: str) -> dict: - """Create a dictionary with all options and values for (and from) the - given job. - - :params job: The name of job for and from which the dictionary will be - created. - :type job: str - :returns: Dictionary with all options and values for (and from) the - given job. - :rtype: dict - """ - - lst_job = job.split("-") - return { - "job": job, - "dut": lst_job[1], - "ttype": lst_job[3], - "cadence": lst_job[4], - "tbed": "-".join(lst_job[-2:]), - "duts": self._generate_options(self._get_duts()), - "ttypes": self._generate_options(self._get_ttypes(lst_job[1])), - "cadences": self._generate_options(self._get_cadences( - lst_job[1], lst_job[3])), - "tbeds": self._generate_options(self._get_test_beds( - lst_job[1], lst_job[3], lst_job[4])) - } - def add_content(self): """Top level method which generated the web page. @@ -470,7 +352,7 @@ class Layout: """Add column with control panel. It is placed on the left side. :returns: Column with the control panel. - :rtype: dbc.col + :rtype: dbc.Col """ return dbc.Col( @@ -484,7 +366,7 @@ class Layout: """Add column with tables. It is placed on the right side. :returns: Column with tables. - :rtype: dbc.col + :rtype: dbc.Col """ return dbc.Col( @@ -616,16 +498,23 @@ class Layout: ] ) ] - ), + ) ] ) class ControlPanel: - """ + """A class representing the control panel. """ def __init__(self, panel: dict, default: dict) -> None: - """ + """Initialisation of the control pannel by default values. If + particular values are provided (parameter "panel") they are set + afterwards. + + :param panel: Custom values to be set to the control panel. + :param default: Default values to be set to the control panel. + :type panel: dict + :type defaults: dict """ self._defaults = { @@ -644,6 +533,13 @@ class Layout: self._panel[key] = panel[key] def set(self, kwargs: dict) -> None: + """Set the values of the Control panel. + + :param kwargs: key - value pairs to be set. + :type kwargs: dict + :raises KeyError: If the key in kwargs is not present in the Control + panel. + """ for key, val in kwargs.items(): if key in self._panel: self._panel[key] = val @@ -659,12 +555,31 @@ class Layout: return self._panel def get(self, key: str) -> any: + """Returns the value of a key from the Control panel. + + :param key: The key which value should be returned. + :type key: str + :returns: The value of the key. + :rtype: any + :raises KeyError: If the key in kwargs is not present in the Control + panel. + """ return self._panel[key] def values(self) -> list: + """Returns the values from the Control panel as a list. + + :returns: The values from the Control panel. + :rtype: list + """ return list(self._panel.values()) def callbacks(self, app): + """Callbacks for the whole application. + + :param app: The application. + :type app: Flask + """ @app.callback( Output("control-panel", "data"), # Store @@ -685,9 +600,25 @@ class Layout: Input("dd-tbeds", "value"), Input("url", "href") ) - def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str, - tbed: str, href: str) -> tuple: - """ + def _update_application(cp_data: dict, dut: str, ttype: str, + cadence:str, tbed: str, href: str) -> tuple: + """Update the application when the event is detected. + + :param cp_data: Current status of the control panel stored in + browser. + :param dut: Input - DUT name. + :param ttype: Input - Test type. + :param cadence: Input - The cadence of the job. + :param tbed: Input - The test bed. + :param href: Input - The URL provided by the browser. + :type cp_data: dict + :type dut: str + :type ttype: str + :type cadence: str + :type tbed: str + :type href: str + :returns: New values for web page elements. + :rtype: tuple """ ctrl_panel = self.ControlPanel(cp_data, self.default) @@ -701,13 +632,13 @@ class Layout: trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0] if trigger_id == "ri-duts": - ttype_opts = self._generate_options(self._get_ttypes(dut)) + ttype_opts = generate_options(get_ttypes(self.job_info, dut)) ttype_val = ttype_opts[0]["value"] - cad_opts = self._generate_options( - self._get_cadences(dut, ttype_val)) + cad_opts = generate_options( + get_cadences(self.job_info, dut, ttype_val)) cad_val = cad_opts[0]["value"] - tbed_opts = self._generate_options( - self._get_test_beds(dut, ttype_val, cad_val)) + tbed_opts = generate_options(get_test_beds( + self.job_info, dut, ttype_val, cad_val)) tbed_val = tbed_opts[0]["value"] ctrl_panel.set({ "ri-duts-value": dut, @@ -719,11 +650,11 @@ class Layout: "dd-tbeds-value": tbed_val }) elif trigger_id == "ri-ttypes": - cad_opts = self._generate_options( - self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype)) + cad_opts = generate_options(get_cadences( + self.job_info, ctrl_panel.get("ri-duts-value"), ttype)) cad_val = cad_opts[0]["value"] - tbed_opts = self._generate_options( - self._get_test_beds(ctrl_panel.get("ri-duts-value"), + tbed_opts = generate_options(get_test_beds( + self.job_info, ctrl_panel.get("ri-duts-value"), ttype, cad_val)) tbed_val = tbed_opts[0]["value"] ctrl_panel.set({ @@ -734,8 +665,8 @@ class Layout: "dd-tbeds-value": tbed_val }) elif trigger_id == "ri-cadences": - tbed_opts = self._generate_options( - self._get_test_beds(ctrl_panel.get("ri-duts-value"), + tbed_opts = generate_options(get_test_beds( + self.job_info, ctrl_panel.get("ri-duts-value"), ctrl_panel.get("ri-ttypes-value"), cadence)) tbed_val = tbed_opts[0]["value"] ctrl_panel.set({ @@ -752,18 +683,13 @@ class Layout: if url_params: new_job = url_params.get("job", list())[0] if new_job: - job_params = self._set_job_params(new_job) + job_params = set_job_params(self.job_info, new_job) ctrl_panel = self.ControlPanel(None, job_params) else: ctrl_panel = self.ControlPanel(cp_data, self.default) - job = self._get_job( - ctrl_panel.get("ri-duts-value"), - ctrl_panel.get("ri-ttypes-value"), - ctrl_panel.get("ri-cadences-value"), - ctrl_panel.get("dd-tbeds-value") - ) - job = self._get_job( + job = get_job( + self.job_info, ctrl_panel.get("ri-duts-value"), ctrl_panel.get("ri-ttypes-value"), ctrl_panel.get("ri-cadences-value"), diff --git a/resources/tools/dash/app/pal/news/news.py b/resources/tools/dash/app/pal/news/news.py index 5e65b62281..a0d05f1483 100644 --- a/resources/tools/dash/app/pal/news/news.py +++ b/resources/tools/dash/app/pal/news/news.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Instantiate the Statistics Dash applocation. +"""Instantiate the News Dash application. """ import dash diff --git a/resources/tools/dash/app/pal/news/tables.py b/resources/tools/dash/app/pal/news/tables.py index 6272814165..1a6c7d2556 100644 --- a/resources/tools/dash/app/pal/news/tables.py +++ b/resources/tools/dash/app/pal/news/tables.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" +"""The tables with news. """ import pandas as pd @@ -21,7 +21,16 @@ from ..utils.constants import Constants as C def table_news(data: pd.DataFrame, job: str) -> list: - """ + """Generates the tables with news: + 1. Falied tests from the last run + 2. Regressions and progressions calculated from the last C.NEWS_TIME_PERIOD + days. + + :param data: Trending data with calculated annomalies to be displayed in the + tables. + :param job: The job name. + :type data: pandas.DataFrame + :type job: str """ job_data = data.loc[(data["job"] == job)] diff --git a/resources/tools/dash/app/pal/stats/layout.py b/resources/tools/dash/app/pal/stats/layout.py index 2e74fb8b3c..7bd2dee620 100644 --- a/resources/tools/dash/app/pal/stats/layout.py +++ b/resources/tools/dash/app/pal/stats/layout.py @@ -29,20 +29,44 @@ from datetime import datetime, timedelta from copy import deepcopy from ..utils.constants import Constants as C -from ..utils.utils import show_tooltip, gen_new_url +from ..utils.utils import show_tooltip, gen_new_url, get_date, get_ttypes, \ + get_cadences, get_test_beds, get_job, generate_options, set_job_params from ..utils.url_processing import url_decode from ..data.data import Data from .graphs import graph_statistics, select_data class Layout: - """ + """The layout of the dash app and the callbacks. """ def __init__(self, app: Flask, html_layout_file: str, graph_layout_file: str, data_spec_file: str, tooltip_file: str, time_period: int=None) -> None: - """ + """Initialization: + - save the input parameters, + - read and pre-process the data, + - prepare data for the control panel, + - read HTML layout file, + - read tooltips from the tooltip file. + + :param app: Flask application running the dash application. + :param html_layout_file: Path and name of the file specifying the HTML + layout of the dash application. + :param graph_layout_file: Path and name of the file with layout of + plot.ly graphs. + :param data_spec_file: Path and name of the file specifying the data to + be read from parquets for this application. + :param tooltip_file: Path and name of the yaml file specifying the + tooltips. + :param time_period: It defines the time period for data read from the + parquets in days from now back to the past. + :type app: Flask + :type html_layout_file: str + :type graph_layout_file: str + :type data_spec_file: str + :type tooltip_file: str + :type time_period: int """ # Inputs @@ -73,7 +97,7 @@ class Layout: self._time_period = data_time_period jobs = sorted(list(data_stats["job"].unique())) - job_info = { + d_job_info = { "job": list(), "dut": list(), "ttype": list(), @@ -82,14 +106,14 @@ class Layout: } for job in jobs: lst_job = job.split("-") - job_info["job"].append(job) - job_info["dut"].append(lst_job[1]) - job_info["ttype"].append(lst_job[3]) - job_info["cadence"].append(lst_job[4]) - job_info["tbed"].append("-".join(lst_job[-2:])) - self.df_job_info = pd.DataFrame.from_dict(job_info) + d_job_info["job"].append(job) + d_job_info["dut"].append(lst_job[1]) + d_job_info["ttype"].append(lst_job[3]) + d_job_info["cadence"].append(lst_job[4]) + d_job_info["tbed"].append("-".join(lst_job[-2:])) + self.job_info = pd.DataFrame.from_dict(d_job_info) - self._default = self._set_job_params(C.STATS_DEFAULT_JOB) + self._default = set_job_params(self.job_info, C.STATS_DEFAULT_JOB) tst_info = { "job": list(), @@ -203,68 +227,20 @@ class Layout: def default(self) -> any: return self._default - def _get_duts(self) -> list: - """ - """ - return sorted(list(self.df_job_info["dut"].unique())) - - def _get_ttypes(self, dut: str) -> list: - """ - """ - return sorted(list(self.df_job_info.loc[( - self.df_job_info["dut"] == dut - )]["ttype"].unique())) - - def _get_cadences(self, dut: str, ttype: str) -> list: - """ - """ - return sorted(list(self.df_job_info.loc[( - (self.df_job_info["dut"] == dut) & - (self.df_job_info["ttype"] == ttype) - )]["cadence"].unique())) + def add_content(self): + """Top level method which generated the web page. - def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list: - """ - """ - return sorted(list(self.df_job_info.loc[( - (self.df_job_info["dut"] == dut) & - (self.df_job_info["ttype"] == ttype) & - (self.df_job_info["cadence"] == cadence) - )]["tbed"].unique())) + It generates: + - Store for user input data, + - Navigation bar, + - Main area with control panel and ploting area. - def _get_job(self, dut, ttype, cadence, testbed): - """Get the name of a job defined by dut, ttype, cadence, testbed. + If no HTML layout is provided, an error message is displayed instead. - Input information comes from control panel. - """ - return self.df_job_info.loc[( - (self.df_job_info["dut"] == dut) & - (self.df_job_info["ttype"] == ttype) & - (self.df_job_info["cadence"] == cadence) & - (self.df_job_info["tbed"] == testbed) - )]["job"].item() - - def _set_job_params(self, job: str) -> dict: - """ + :returns: The HTML div with teh whole page. + :rtype: html.Div """ - lst_job = job.split("-") - return { - "job": job, - "dut": lst_job[1], - "ttype": lst_job[3], - "cadence": lst_job[4], - "tbed": "-".join(lst_job[-2:]), - "duts": self._generate_options(self._get_duts()), - "ttypes": self._generate_options(self._get_ttypes(lst_job[1])), - "cadences": self._generate_options(self._get_cadences( - lst_job[1], lst_job[3])), - "tbeds": self._generate_options(self._get_test_beds( - lst_job[1], lst_job[3], lst_job[4])) - } - def add_content(self): - """ - """ if self.html_layout: return html.Div( id="div-main", @@ -315,6 +291,9 @@ class Layout: def _add_navbar(self): """Add nav element with navigation panel. It is placed on the top. + + :returns: Navigation bar. + :rtype: dbc.NavbarSimple """ return dbc.NavbarSimple( id="navbarsimple-main", @@ -337,6 +316,9 @@ class Layout: def _add_ctrl_col(self) -> dbc.Col: """Add column with controls. It is placed on the left side. + + :returns: Column with the control panel. + :rtype: dbc.Col """ return dbc.Col( id="col-controls", @@ -347,6 +329,9 @@ class Layout: def _add_plotting_col(self) -> dbc.Col: """Add column with plots and tables. It is placed on the right side. + + :returns: Column with tables. + :rtype: dbc.Col """ return dbc.Col( id="col-plotting-area", @@ -427,7 +412,10 @@ class Layout: ) def _add_ctrl_panel(self) -> dbc.Row: - """ + """Add control panel. + + :returns: Control panel. + :rtype: dbc.Row """ return dbc.Row( id="row-ctrl-panel", @@ -542,7 +530,20 @@ class Layout: ) class ControlPanel: + """A class representing the control panel. + """ + def __init__(self, panel: dict, default: dict) -> None: + """Initialisation of the control pannel by default values. If + particular values are provided (parameter "panel") they are set + afterwards. + + :param panel: Custom values to be set to the control panel. + :param default: Default values to be set to the control panel. + :type panel: dict + :type defaults: dict + """ + self._defaults = { "ri-ttypes-options": default["ttypes"], "ri-cadences-options": default["cadences"], @@ -559,6 +560,13 @@ class Layout: self._panel[key] = panel[key] def set(self, kwargs: dict) -> None: + """Set the values of the Control panel. + + :param kwargs: key - value pairs to be set. + :type kwargs: dict + :raises KeyError: If the key in kwargs is not present in the Control + panel. + """ for key, val in kwargs.items(): if key in self._panel: self._panel[key] = val @@ -574,20 +582,32 @@ class Layout: return self._panel def get(self, key: str) -> any: + """Returns the value of a key from the Control panel. + + :param key: The key which value should be returned. + :type key: str + :returns: The value of the key. + :rtype: any + :raises KeyError: If the key in kwargs is not present in the Control + panel. + """ return self._panel[key] def values(self) -> list: - return list(self._panel.values()) + """Returns the values from the Control panel as a list. - @staticmethod - def _generate_options(opts: list) -> list: - return [{"label": i, "value": i} for i in opts] + :returns: The values from the Control panel. + :rtype: list + """ + return list(self._panel.values()) - @staticmethod - def _get_date(s_date: str) -> datetime: - return datetime(int(s_date[0:4]), int(s_date[5:7]), int(s_date[8:10])) def callbacks(self, app): + """Callbacks for the whole application. + + :param app: The application. + :type app: Flask + """ @app.callback( Output("control-panel", "data"), # Store @@ -611,15 +631,35 @@ class Layout: Input("dpr-period", "end_date"), Input("url", "href") ) - def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str, + def _update_ctrl_panel(cp_data: dict, dut: str, ttype: str, cadence:str, tbed: str, start: str, end: str, href: str) -> tuple: - """ + """Update the application when the event is detected. + + :param cp_data: Current status of the control panel stored in + browser. + :param dut: Input - DUT name. + :param ttype: Input - Test type. + :param cadence: Input - The cadence of the job. + :param tbed: Input - The test bed. + :param start: Date and time where the data processing starts. + :param end: Date and time where the data processing ends. + :param href: Input - The URL provided by the browser. + :type cp_data: dict + :type dut: str + :type ttype: str + :type cadence: str + :type tbed: str + :type start: str + :type end: str + :type href: str + :returns: New values for web page elements. + :rtype: tuple """ ctrl_panel = self.ControlPanel(cp_data, self.default) - start = self._get_date(start) - end = self._get_date(end) + start = get_date(start) + end = get_date(end) # Parse the url: parsed_url = url_decode(href) @@ -630,13 +670,13 @@ class Layout: trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0] if trigger_id == "ri-duts": - ttype_opts = self._generate_options(self._get_ttypes(dut)) + ttype_opts = generate_options(get_ttypes(self.job_info, dut)) ttype_val = ttype_opts[0]["value"] - cad_opts = self._generate_options( - self._get_cadences(dut, ttype_val)) + cad_opts = generate_options(get_cadences( + self.job_info, dut, ttype_val)) cad_val = cad_opts[0]["value"] - tbed_opts = self._generate_options( - self._get_test_beds(dut, ttype_val, cad_val)) + tbed_opts = generate_options(get_test_beds( + self.job_info, dut, ttype_val, cad_val)) tbed_val = tbed_opts[0]["value"] ctrl_panel.set({ "ri-duts-value": dut, @@ -648,12 +688,12 @@ class Layout: "dd-tbeds-value": tbed_val }) elif trigger_id == "ri-ttypes": - cad_opts = self._generate_options( - self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype)) + cad_opts = generate_options(get_cadences( + self.job_info, ctrl_panel.get("ri-duts-value"), ttype)) cad_val = cad_opts[0]["value"] - tbed_opts = self._generate_options( - self._get_test_beds(ctrl_panel.get("ri-duts-value"), - ttype, cad_val)) + tbed_opts = generate_options(get_test_beds( + self.job_info, ctrl_panel.get("ri-duts-value"), ttype, + cad_val)) tbed_val = tbed_opts[0]["value"] ctrl_panel.set({ "ri-ttypes-value": ttype, @@ -663,8 +703,8 @@ class Layout: "dd-tbeds-value": tbed_val }) elif trigger_id == "ri-cadences": - tbed_opts = self._generate_options( - self._get_test_beds(ctrl_panel.get("ri-duts-value"), + tbed_opts = generate_options(get_test_beds( + self.job_info, ctrl_panel.get("ri-duts-value"), ctrl_panel.get("ri-ttypes-value"), cadence)) tbed_val = tbed_opts[0]["value"] ctrl_panel.set({ @@ -685,20 +725,15 @@ class Layout: new_start = url_params.get("start", list())[0] new_end = url_params.get("end", list())[0] if new_job and new_start and new_end: - start = self._get_date(new_start) - end = self._get_date(new_end) - job_params = self._set_job_params(new_job) + start = get_date(new_start) + end = get_date(new_end) + job_params = set_job_params(self.job_info, new_job) ctrl_panel = self.ControlPanel(None, job_params) else: ctrl_panel = self.ControlPanel(cp_data, self.default) - job = self._get_job( - ctrl_panel.get("ri-duts-value"), - ctrl_panel.get("ri-ttypes-value"), - ctrl_panel.get("ri-cadences-value"), - ctrl_panel.get("dd-tbeds-value") - ) - job = self._get_job( + job = get_job( + self.job_info, ctrl_panel.get("ri-duts-value"), ctrl_panel.get("ri-ttypes-value"), ctrl_panel.get("ri-cadences-value"), @@ -734,23 +769,35 @@ class Layout: prevent_initial_call=True ) def _download_data(cp_data: dict, start: str, end: str, n_clicks: int): - """ + """Download the data + + :param cp_data: Current status of the control panel stored in + browser. + :param start: Date and time where the data processing starts. + :param end: Date and time where the data processing ends. + :param n_clicks: Number of clicks on the button "Download". + :type cp_data: dict + :type start: str + :type end: str + :type n_clicks: int + :returns: dict of data frame content (base64 encoded) and meta data + used by the Download component. + :rtype: dict """ if not (n_clicks): raise PreventUpdate ctrl_panel = self.ControlPanel(cp_data, self.default) - job = self._get_job( + job = get_job( + self.job_info, ctrl_panel.get("ri-duts-value"), ctrl_panel.get("ri-ttypes-value"), ctrl_panel.get("ri-cadences-value"), ctrl_panel.get("dd-tbeds-value") ) - start = datetime(int(start[0:4]), int(start[5:7]), int(start[8:10])) - end = datetime(int(end[0:4]), int(end[5:7]), int(end[8:10])) - data = select_data(self.data, job, start, end) + data = select_data(self.data, job, get_date(start), get_date(end)) data = data.drop(columns=["job", ]) return dcc.send_data_frame(data.T.to_csv, f"{job}-stats.csv") @@ -764,7 +811,19 @@ class Layout: ) def _show_metadata_from_graphs( passed_data: dict, duration_data: dict) -> tuple: - """ + """Generates the data for the offcanvas displayed when a particular + point in a graph is clicked on. + + :param passed_data: The data from the clicked point in the graph + displaying the pass/fail data. + :param duration_data: The data from the clicked point in the graph + displaying the duration data. + :type passed_data: dict + :type duration data: dict + :returns: The data to be displayed on the offcanvas (job statistics + and the list of failed tests) and the information to show the + offcanvas. + :rtype: tuple(list, bool) """ if not (passed_data or duration_data): diff --git a/resources/tools/dash/app/pal/stats/stats.py b/resources/tools/dash/app/pal/stats/stats.py index 560ec53f14..5b31faca44 100644 --- a/resources/tools/dash/app/pal/stats/stats.py +++ b/resources/tools/dash/app/pal/stats/stats.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Instantiate the Statistics Dash applocation. +"""Instantiate the Statistics Dash application. """ import dash diff --git a/resources/tools/dash/app/pal/utils/constants.py b/resources/tools/dash/app/pal/utils/constants.py index 1c84ba14a6..7a836e79f3 100644 --- a/resources/tools/dash/app/pal/utils/constants.py +++ b/resources/tools/dash/app/pal/utils/constants.py @@ -253,7 +253,11 @@ class Constants: # The default job displayed when the page is loaded first time. NEWS_DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx" - # Time period for regressions and progressions. + # Time period for regressions and progressions. Be CAREFULL with this + # number. Setting it too high causes long processing time during the + # application start-up. + # If NEWS_TIME_PERIOD = 180, it takes approx. 35 minutes to calculate + # annomalies for all tests. NEWS_TIME_PERIOD = 21 # [days] ############################################################################ diff --git a/resources/tools/dash/app/pal/utils/utils.py b/resources/tools/dash/app/pal/utils/utils.py index 70fb02e189..9e4eeeb892 100644 --- a/resources/tools/dash/app/pal/utils/utils.py +++ b/resources/tools/dash/app/pal/utils/utils.py @@ -14,6 +14,7 @@ """Function used by Dash applications. """ +import pandas as pd import dash_bootstrap_components as dbc from numpy import isnan @@ -80,7 +81,7 @@ def get_color(idx: int) -> str: its index. :param idx: Index of the color. - :type idex: int + :type idx: int :returns: Color defined by hex code. :trype: str """ @@ -183,26 +184,161 @@ def get_date(s_date: str) -> datetime: return datetime(int(s_date[0:4]), int(s_date[5:7]), int(s_date[8:10])) -def gen_new_url(parsed_url: dict, params: dict) -> str: +def gen_new_url(url_components: dict, params: dict) -> str: """Generate a new URL with encoded parameters. - :param parsed_url: Dictionary with URL elements. It should contain "scheme", - "netloc" and "path". - :param params: URL parameters to be encoded to the URL. + :param url_components: Dictionary with URL elements. It should contain + "scheme", "netloc" and "path". + :param url_components: URL parameters to be encoded to the URL. :type parsed_url: dict :type params: dict :returns Encoded URL with parameters. :rtype: str """ - if parsed_url: + if url_components: return url_encode( { - "scheme": parsed_url.get("scheme", ""), - "netloc": parsed_url.get("netloc", ""), - "path": parsed_url.get("path", ""), + "scheme": url_components.get("scheme", ""), + "netloc": url_components.get("netloc", ""), + "path": url_components.get("path", ""), "params": params } ) else: return str() + + +def get_duts(df: pd.DataFrame) -> list: + """Get the list of DUTs from the pre-processed information about jobs. + + :param df: DataFrame with information about jobs. + :type df: pandas.DataFrame + :returns: Alphabeticaly sorted list of DUTs. + :rtype: list + """ + return sorted(list(df["dut"].unique())) + + +def get_ttypes(df: pd.DataFrame, dut: str) -> list: + """Get the list of test types from the pre-processed information about + jobs. + + :param df: DataFrame with information about jobs. + :param dut: The DUT for which the list of test types will be populated. + :type df: pandas.DataFrame + :type dut: str + :returns: Alphabeticaly sorted list of test types. + :rtype: list + """ + return sorted(list(df.loc[(df["dut"] == dut)]["ttype"].unique())) + + +def get_cadences(df: pd.DataFrame, dut: str, ttype: str) -> list: + """Get the list of cadences from the pre-processed information about + jobs. + + :param df: DataFrame with information about jobs. + :param dut: The DUT for which the list of cadences will be populated. + :param ttype: The test type for which the list of cadences will be + populated. + :type df: pandas.DataFrame + :type dut: str + :type ttype: str + :returns: Alphabeticaly sorted list of cadences. + :rtype: list + """ + return sorted(list(df.loc[( + (df["dut"] == dut) & + (df["ttype"] == ttype) + )]["cadence"].unique())) + + +def get_test_beds(df: pd.DataFrame, dut: str, ttype: str, cadence: str) -> list: + """Get the list of test beds from the pre-processed information about + jobs. + + :param df: DataFrame with information about jobs. + :param dut: The DUT for which the list of test beds will be populated. + :param ttype: The test type for which the list of test beds will be + populated. + :param cadence: The cadence for which the list of test beds will be + populated. + :type df: pandas.DataFrame + :type dut: str + :type ttype: str + :type cadence: str + :returns: Alphabeticaly sorted list of test beds. + :rtype: list + """ + return sorted(list(df.loc[( + (df["dut"] == dut) & + (df["ttype"] == ttype) & + (df["cadence"] == cadence) + )]["tbed"].unique())) + + +def get_job(df: pd.DataFrame, dut, ttype, cadence, testbed): + """Get the name of a job defined by dut, ttype, cadence, test bed. + Input information comes from the control panel. + + :param df: DataFrame with information about jobs. + :param dut: The DUT for which the job name will be created. + :param ttype: The test type for which the job name will be created. + :param cadence: The cadence for which the job name will be created. + :param testbed: The test bed for which the job name will be created. + :type df: pandas.DataFrame + :type dut: str + :type ttype: str + :type cadence: str + :type testbed: str + :returns: Job name. + :rtype: str + """ + return df.loc[( + (df["dut"] == dut) & + (df["ttype"] == ttype) & + (df["cadence"] == cadence) & + (df["tbed"] == testbed) + )]["job"].item() + + +def generate_options(opts: list) -> list: + """Return list of options for radio items in control panel. The items in + the list are dictionaries with keys "label" and "value". + + :params opts: List of options (str) to be used for the generated list. + :type opts: list + :returns: List of options (dict). + :rtype: list + """ + return [{"label": i, "value": i} for i in opts] + + +def set_job_params(df: pd.DataFrame, job: str) -> dict: + """Create a dictionary with all options and values for (and from) the + given job. + + :param df: DataFrame with information about jobs. + :params job: The name of job for and from which the dictionary will be + created. + :type df: pandas.DataFrame + :type job: str + :returns: Dictionary with all options and values for (and from) the + given job. + :rtype: dict + """ + + l_job = job.split("-") + return { + "job": job, + "dut": l_job[1], + "ttype": l_job[3], + "cadence": l_job[4], + "tbed": "-".join(l_job[-2:]), + "duts": generate_options(get_duts(df)), + "ttypes": generate_options(get_ttypes(df, l_job[1])), + "cadences": generate_options(get_cadences(df, l_job[1], l_job[3])), + "tbeds": generate_options( + get_test_beds(df, l_job[1], l_job[3], l_job[4])) + } |