diff options
| -rw-r--r-- | resources/tools/dash/app/pal/__init__.py | 6 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/news/layout.py | 73 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/news/tables.py | 173 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/utils/constants.py | 10 |
4 files changed, 196 insertions, 66 deletions
diff --git a/resources/tools/dash/app/pal/__init__.py b/resources/tools/dash/app/pal/__init__.py index 0eb2a4e79e..1ea6db0fac 100644 --- a/resources/tools/dash/app/pal/__init__.py +++ b/resources/tools/dash/app/pal/__init__.py @@ -27,9 +27,9 @@ def init_app(): """ logging.basicConfig( - format=u"%(asctime)s: %(levelname)s: %(message)s", - datefmt=u"%Y/%m/%d %H:%M:%S", - level=logging.INFO + format=C.LOG_FORMAT, + datefmt=C.LOG_DATE_FORMAT, + level=C.LOG_LEVEL ) logging.info("Application started.") diff --git a/resources/tools/dash/app/pal/news/layout.py b/resources/tools/dash/app/pal/news/layout.py index 73fabdf884..6e598315e5 100644 --- a/resources/tools/dash/app/pal/news/layout.py +++ b/resources/tools/dash/app/pal/news/layout.py @@ -33,7 +33,7 @@ from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url, \ set_job_params from ..utils.url_processing import url_decode from ..data.data import Data -from .tables import table_news +from .tables import table_news, table_summary class Layout: @@ -77,7 +77,7 @@ class Layout: df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True) # Prepare information for the control panel: - jobs = sorted(list(df_tst_info["job"].unique())) + self._jobs = sorted(list(df_tst_info["job"].unique())) d_job_info = { "job": list(), "dut": list(), @@ -85,7 +85,7 @@ class Layout: "cadence": list(), "tbed": list() } - for job in jobs: + for job in self._jobs: lst_job = job.split("-") d_job_info["job"].append(job) d_job_info["dut"].append(lst_job[1]) @@ -118,7 +118,9 @@ class Layout: "regressions": list(), "progressions": list() } - for job in jobs: + logging.debug("Processing jobs ...") + for job in self._jobs: + logging.debug(f"+ {job}") # Create lists of failed tests: df_job = df_tst_info.loc[(df_tst_info["job"] == job)] last_build = max(df_job["build"].unique()) @@ -254,7 +256,8 @@ class Layout: f"{self._tooltip_file}\n{err}" ) - self._default_tab_failed = table_news(self.data, self._default["job"]) + self._default_tab_failed = \ + table_news(self.data, self._default["job"], C.NEWS_TIME_PERIOD) # Callbacks: if self._app is not None and hasattr(self, 'callbacks'): @@ -488,6 +491,25 @@ class Layout: ] ), dbc.Row( + class_name="gy-1 p-0", + children=[ + dbc.ButtonGroup( + [ + dbc.Button( + id="btn-summary", + children=( + f"Show Summary from the last " + f"{C.NEWS_SUMMARY_PERIOD} Days" + ), + class_name="me-1", + color="info" + ) + ], + size="md", + ) + ] + ), + dbc.Row( class_name="gy-1", children=[ dbc.Alert( @@ -598,10 +620,11 @@ class Layout: Input("ri-ttypes", "value"), Input("ri-cadences", "value"), Input("dd-tbeds", "value"), - Input("url", "href") + Input("url", "href"), + Input("btn-summary", "n_clicks") ) def _update_application(cp_data: dict, dut: str, ttype: str, - cadence:str, tbed: str, href: str) -> tuple: + cadence:str, tbed: str, href: str, btn_all: int) -> tuple: """Update the application when the event is detected. :param cp_data: Current status of the control panel stored in @@ -630,6 +653,8 @@ class Layout: else: url_params = None + show_summary = False + trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0] if trigger_id == "ri-duts": ttype_opts = generate_options(get_ttypes(self.job_info, dut)) @@ -682,25 +707,37 @@ class Layout: # TODO: Add verification if url_params: new_job = url_params.get("job", list())[0] - if new_job: + if new_job and new_job != "all": job_params = set_job_params(self.job_info, new_job) ctrl_panel = self.ControlPanel(None, job_params) + if new_job and new_job == "all": + show_summary = True else: ctrl_panel = self.ControlPanel(cp_data, self.default) + elif trigger_id == "btn-summary": + show_summary = True - job = get_job( - self.job_info, - ctrl_panel.get("ri-duts-value"), - ctrl_panel.get("ri-ttypes-value"), - ctrl_panel.get("ri-cadences-value"), - ctrl_panel.get("dd-tbeds-value") - ) - ctrl_panel.set({"al-job-children": job}) - tab_failed = table_news(self.data, job) + if show_summary: + ctrl_panel.set({ + "al-job-children": \ + f"Summary from the last {C.NEWS_SUMMARY_PERIOD} days" + }) + job = "all" + tables = table_summary(self.data, self._jobs) + else: + job = get_job( + self.job_info, + ctrl_panel.get("ri-duts-value"), + ctrl_panel.get("ri-ttypes-value"), + ctrl_panel.get("ri-cadences-value"), + ctrl_panel.get("dd-tbeds-value") + ) + ctrl_panel.set({"al-job-children": job}) + tables = table_news(self.data, job, C.NEWS_TIME_PERIOD) ret_val = [ ctrl_panel.panel, - tab_failed, + tables, gen_new_url(parsed_url, {"job": job}) ] ret_val.extend(ctrl_panel.values()) diff --git a/resources/tools/dash/app/pal/news/tables.py b/resources/tools/dash/app/pal/news/tables.py index 1a6c7d2556..04b4fc975e 100644 --- a/resources/tools/dash/app/pal/news/tables.py +++ b/resources/tools/dash/app/pal/news/tables.py @@ -17,10 +17,88 @@ import pandas as pd import dash_bootstrap_components as dbc +from datetime import datetime, timedelta +from dash import html + from ..utils.constants import Constants as C -def table_news(data: pd.DataFrame, job: str) -> list: +def _table_info(job_data: pd.DataFrame) -> dbc.Table: + """Generates table with info about the job. + + :param job_data: Dataframe with information about the job. + :type job_data: pandas.DataFrame + :returns: Table with job info. + :rtype: dbc.Table + """ + return dbc.Table.from_dataframe( + pd.DataFrame.from_dict( + { + "Job": job_data["job"], + "Last Build": job_data["build"], + "Date": job_data["start"], + "DUT": job_data["dut_type"], + "DUT Version": job_data["dut_version"], + "Hosts": ", ".join(job_data["hosts"].to_list()[0]) + } + ), + bordered=True, + striped=True, + hover=True, + size="sm", + color="info" + ) + + +def _table_failed(job_data: pd.DataFrame, failed: list) -> dbc.Table: + """Generates table with failed tests from the last run of the job. + + :param job_data: Dataframe with information about the job. + :param failed: List of failed tests. + :type job_data: pandas.DataFrame + :type failed: list + :returns: Table with fialed tests. + :rtype: dbc.Table + """ + return dbc.Table.from_dataframe( + pd.DataFrame.from_dict( + { + ( + f"Last Failed Tests on " + f"{job_data['start'].values[0]} ({len(failed)})" + ): failed + } + ), + bordered=True, + striped=True, + hover=True, + size="sm", + color="danger" + ) + + +def _table_gressions(itms: dict, color: str) -> dbc.Table: + """Generates table with regressions. + + :param itms: Dictionary with items (regressions or progressions) and their + last occurence. + :param color: Color of the table. + :type regressions: dict + :type color: str + :returns: The table with regressions. + :rtype: dbc.Table + """ + return dbc.Table.from_dataframe( + pd.DataFrame.from_dict(itms), + bordered=True, + striped=True, + hover=True, + size="sm", + color=color + ) + + +def table_news(data: pd.DataFrame, job: str, period: int) -> list: """Generates the tables with news: 1. Falied tests from the last run 2. Regressions and progressions calculated from the last C.NEWS_TIME_PERIOD @@ -29,57 +107,64 @@ def table_news(data: pd.DataFrame, job: str) -> list: :param data: Trending data with calculated annomalies to be displayed in the tables. :param job: The job name. + :param period: The time period (nr of days from now) taken into account. :type data: pandas.DataFrame :type job: str + :type period: int + :returns: List of tables. + :rtype: list """ + last_day = datetime.utcnow() - timedelta(days=period) + r_list = list() job_data = data.loc[(data["job"] == job)] + r_list.append(_table_info(job_data)) + failed = job_data["failed"].to_list()[0] - regressions = {"Test Name": list(), "Last Regression": list()} + if failed: + r_list.append(_table_failed(job_data, failed)) + + title = f"Regressions in the last {period} days" + regressions = {title: list(), "Last Regression": list()} for itm in job_data["regressions"].to_list()[0]: - regressions["Test Name"].append(itm[0]) - regressions["Last Regression"].append(itm[1].strftime('%Y-%m-%d %H:%M')) - progressions = {"Test Name": list(), "Last Progression": list()} + if itm[1] < last_day: + break + regressions[title].append(itm[0]) + regressions["Last Regression"].append( + itm[1].strftime('%Y-%m-%d %H:%M')) + if regressions["Last Regression"]: + r_list.append(_table_gressions(regressions, "warning")) + + title = f"Progressions in the last {period} days" + progressions = {title: list(), "Last Progression": list()} for itm in job_data["progressions"].to_list()[0]: - progressions["Test Name"].append(itm[0]) + if itm[1] < last_day: + break + progressions[title].append(itm[0]) progressions["Last Progression"].append( itm[1].strftime('%Y-%m-%d %H:%M')) + if progressions["Last Progression"]: + r_list.append(_table_gressions(progressions, "success")) - return [ - dbc.Table.from_dataframe(pd.DataFrame.from_dict({ - "Job": job_data["job"], - "Last Build": job_data["build"], - "Date": job_data["start"], - "DUT": job_data["dut_type"], - "DUT Version": job_data["dut_version"], - "Hosts": ", ".join(job_data["hosts"].to_list()[0]) - }), bordered=True, striped=True, hover=True, size="sm", color="light"), - dbc.Table.from_dataframe(pd.DataFrame.from_dict({ - ( - f"Last Failed Tests on " - f"{job_data['start'].values[0]} ({len(failed)})" - ): failed - }), bordered=True, striped=True, hover=True, size="sm", color="light"), - dbc.Label( - class_name="p-0", - size="lg", - children=( - f"Regressions during the last {C.NEWS_TIME_PERIOD} days " - f"({len(regressions['Test Name'])})" - ) - ), - dbc.Table.from_dataframe( - pd.DataFrame.from_dict(regressions), - bordered=True, striped=True, hover=True, size="sm", color="light"), - dbc.Label( - class_name="p-0", - size="lg", - children=( - f"Progressions during the last {C.NEWS_TIME_PERIOD} days " - f"({len(progressions['Test Name'])})" - ) - ), - dbc.Table.from_dataframe( - pd.DataFrame.from_dict(progressions), - bordered=True, striped=True, hover=True, size="sm", color="light") - ] + return r_list + + +def table_summary(data: pd.DataFrame, jobs: list) -> list: + """Generates summary (failed tests, regressions and progressions) from the + last week. + + :param data: Trending data with calculated annomalies to be displayed in the + tables. + :param jobs: List of jobs. + :type data: pandas.DataFrame + :type job: str + :returns: List of tables. + :rtype: list + """ + + r_list = list() + for job in jobs: + r_list.extend(table_news(data, job, C.NEWS_SUMMARY_PERIOD)) + r_list.append(html.Div(html.P(" "))) + + return r_list diff --git a/resources/tools/dash/app/pal/utils/constants.py b/resources/tools/dash/app/pal/utils/constants.py index b95a8f5456..1f31185f71 100644 --- a/resources/tools/dash/app/pal/utils/constants.py +++ b/resources/tools/dash/app/pal/utils/constants.py @@ -17,7 +17,7 @@ does not need to be hard coded here, but can be read from environment variables. """ - +import logging import dash_bootstrap_components as dbc from dash import html @@ -30,6 +30,11 @@ class Constants: ############################################################################ # General, application wide constants. + # Logging settings. + LOG_LEVEL = logging.INFO + LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s" + LOG_DATE_FORMAT = "%Y/%m/%d %H:%M:%S" + # The application title. TITLE = "FD.io CSIT" @@ -256,6 +261,9 @@ class Constants: # Time period for regressions and progressions. NEWS_TIME_PERIOD = TIME_PERIOD # [days] + # Time period for summary tables. + NEWS_SUMMARY_PERIOD = 7 # [days] + ############################################################################ # Report. |