aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2022-08-19 14:42:23 +0200
committerTibor Frank <tifrank@cisco.com>2022-08-19 13:25:46 +0000
commit1efb9c180524bdb0bdccc612e182364821810fa5 (patch)
tree9f5e8c1b6fe238ffef09118d037e2f60190a3fac
parented0473f9ac01522470fb8a2260720214b72ca47f (diff)
UTI: Add Summary to News
Change-Id: Ifa3882b36976183f628d2e3171061945ab33bf29 Signed-off-by: Tibor Frank <tifrank@cisco.com>
-rw-r--r--resources/tools/dash/app/pal/__init__.py6
-rw-r--r--resources/tools/dash/app/pal/news/layout.py73
-rw-r--r--resources/tools/dash/app/pal/news/tables.py173
-rw-r--r--resources/tools/dash/app/pal/utils/constants.py10
4 files changed, 196 insertions, 66 deletions
diff --git a/resources/tools/dash/app/pal/__init__.py b/resources/tools/dash/app/pal/__init__.py
index 0eb2a4e79e..1ea6db0fac 100644
--- a/resources/tools/dash/app/pal/__init__.py
+++ b/resources/tools/dash/app/pal/__init__.py
@@ -27,9 +27,9 @@ def init_app():
"""
logging.basicConfig(
- format=u"%(asctime)s: %(levelname)s: %(message)s",
- datefmt=u"%Y/%m/%d %H:%M:%S",
- level=logging.INFO
+ format=C.LOG_FORMAT,
+ datefmt=C.LOG_DATE_FORMAT,
+ level=C.LOG_LEVEL
)
logging.info("Application started.")
diff --git a/resources/tools/dash/app/pal/news/layout.py b/resources/tools/dash/app/pal/news/layout.py
index 73fabdf884..6e598315e5 100644
--- a/resources/tools/dash/app/pal/news/layout.py
+++ b/resources/tools/dash/app/pal/news/layout.py
@@ -33,7 +33,7 @@ from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url, \
set_job_params
from ..utils.url_processing import url_decode
from ..data.data import Data
-from .tables import table_news
+from .tables import table_news, table_summary
class Layout:
@@ -77,7 +77,7 @@ class Layout:
df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
# Prepare information for the control panel:
- jobs = sorted(list(df_tst_info["job"].unique()))
+ self._jobs = sorted(list(df_tst_info["job"].unique()))
d_job_info = {
"job": list(),
"dut": list(),
@@ -85,7 +85,7 @@ class Layout:
"cadence": list(),
"tbed": list()
}
- for job in jobs:
+ for job in self._jobs:
lst_job = job.split("-")
d_job_info["job"].append(job)
d_job_info["dut"].append(lst_job[1])
@@ -118,7 +118,9 @@ class Layout:
"regressions": list(),
"progressions": list()
}
- for job in jobs:
+ logging.debug("Processing jobs ...")
+ for job in self._jobs:
+ logging.debug(f"+ {job}")
# Create lists of failed tests:
df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
last_build = max(df_job["build"].unique())
@@ -254,7 +256,8 @@ class Layout:
f"{self._tooltip_file}\n{err}"
)
- self._default_tab_failed = table_news(self.data, self._default["job"])
+ self._default_tab_failed = \
+ table_news(self.data, self._default["job"], C.NEWS_TIME_PERIOD)
# Callbacks:
if self._app is not None and hasattr(self, 'callbacks'):
@@ -488,6 +491,25 @@ class Layout:
]
),
dbc.Row(
+ class_name="gy-1 p-0",
+ children=[
+ dbc.ButtonGroup(
+ [
+ dbc.Button(
+ id="btn-summary",
+ children=(
+ f"Show Summary from the last "
+ f"{C.NEWS_SUMMARY_PERIOD} Days"
+ ),
+ class_name="me-1",
+ color="info"
+ )
+ ],
+ size="md",
+ )
+ ]
+ ),
+ dbc.Row(
class_name="gy-1",
children=[
dbc.Alert(
@@ -598,10 +620,11 @@ class Layout:
Input("ri-ttypes", "value"),
Input("ri-cadences", "value"),
Input("dd-tbeds", "value"),
- Input("url", "href")
+ Input("url", "href"),
+ Input("btn-summary", "n_clicks")
)
def _update_application(cp_data: dict, dut: str, ttype: str,
- cadence:str, tbed: str, href: str) -> tuple:
+ cadence:str, tbed: str, href: str, btn_all: int) -> tuple:
"""Update the application when the event is detected.
:param cp_data: Current status of the control panel stored in
@@ -630,6 +653,8 @@ class Layout:
else:
url_params = None
+ show_summary = False
+
trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
if trigger_id == "ri-duts":
ttype_opts = generate_options(get_ttypes(self.job_info, dut))
@@ -682,25 +707,37 @@ class Layout:
# TODO: Add verification
if url_params:
new_job = url_params.get("job", list())[0]
- if new_job:
+ if new_job and new_job != "all":
job_params = set_job_params(self.job_info, new_job)
ctrl_panel = self.ControlPanel(None, job_params)
+ if new_job and new_job == "all":
+ show_summary = True
else:
ctrl_panel = self.ControlPanel(cp_data, self.default)
+ elif trigger_id == "btn-summary":
+ show_summary = True
- job = get_job(
- self.job_info,
- ctrl_panel.get("ri-duts-value"),
- ctrl_panel.get("ri-ttypes-value"),
- ctrl_panel.get("ri-cadences-value"),
- ctrl_panel.get("dd-tbeds-value")
- )
- ctrl_panel.set({"al-job-children": job})
- tab_failed = table_news(self.data, job)
+ if show_summary:
+ ctrl_panel.set({
+ "al-job-children": \
+ f"Summary from the last {C.NEWS_SUMMARY_PERIOD} days"
+ })
+ job = "all"
+ tables = table_summary(self.data, self._jobs)
+ else:
+ job = get_job(
+ self.job_info,
+ ctrl_panel.get("ri-duts-value"),
+ ctrl_panel.get("ri-ttypes-value"),
+ ctrl_panel.get("ri-cadences-value"),
+ ctrl_panel.get("dd-tbeds-value")
+ )
+ ctrl_panel.set({"al-job-children": job})
+ tables = table_news(self.data, job, C.NEWS_TIME_PERIOD)
ret_val = [
ctrl_panel.panel,
- tab_failed,
+ tables,
gen_new_url(parsed_url, {"job": job})
]
ret_val.extend(ctrl_panel.values())
diff --git a/resources/tools/dash/app/pal/news/tables.py b/resources/tools/dash/app/pal/news/tables.py
index 1a6c7d2556..04b4fc975e 100644
--- a/resources/tools/dash/app/pal/news/tables.py
+++ b/resources/tools/dash/app/pal/news/tables.py
@@ -17,10 +17,88 @@
import pandas as pd
import dash_bootstrap_components as dbc
+from datetime import datetime, timedelta
+from dash import html
+
from ..utils.constants import Constants as C
-def table_news(data: pd.DataFrame, job: str) -> list:
+def _table_info(job_data: pd.DataFrame) -> dbc.Table:
+ """Generates table with info about the job.
+
+ :param job_data: Dataframe with information about the job.
+ :type job_data: pandas.DataFrame
+ :returns: Table with job info.
+ :rtype: dbc.Table
+ """
+ return dbc.Table.from_dataframe(
+ pd.DataFrame.from_dict(
+ {
+ "Job": job_data["job"],
+ "Last Build": job_data["build"],
+ "Date": job_data["start"],
+ "DUT": job_data["dut_type"],
+ "DUT Version": job_data["dut_version"],
+ "Hosts": ", ".join(job_data["hosts"].to_list()[0])
+ }
+ ),
+ bordered=True,
+ striped=True,
+ hover=True,
+ size="sm",
+ color="info"
+ )
+
+
+def _table_failed(job_data: pd.DataFrame, failed: list) -> dbc.Table:
+ """Generates table with failed tests from the last run of the job.
+
+ :param job_data: Dataframe with information about the job.
+ :param failed: List of failed tests.
+ :type job_data: pandas.DataFrame
+ :type failed: list
+ :returns: Table with fialed tests.
+ :rtype: dbc.Table
+ """
+ return dbc.Table.from_dataframe(
+ pd.DataFrame.from_dict(
+ {
+ (
+ f"Last Failed Tests on "
+ f"{job_data['start'].values[0]} ({len(failed)})"
+ ): failed
+ }
+ ),
+ bordered=True,
+ striped=True,
+ hover=True,
+ size="sm",
+ color="danger"
+ )
+
+
+def _table_gressions(itms: dict, color: str) -> dbc.Table:
+ """Generates table with regressions.
+
+ :param itms: Dictionary with items (regressions or progressions) and their
+ last occurence.
+ :param color: Color of the table.
+ :type regressions: dict
+ :type color: str
+ :returns: The table with regressions.
+ :rtype: dbc.Table
+ """
+ return dbc.Table.from_dataframe(
+ pd.DataFrame.from_dict(itms),
+ bordered=True,
+ striped=True,
+ hover=True,
+ size="sm",
+ color=color
+ )
+
+
+def table_news(data: pd.DataFrame, job: str, period: int) -> list:
"""Generates the tables with news:
1. Falied tests from the last run
2. Regressions and progressions calculated from the last C.NEWS_TIME_PERIOD
@@ -29,57 +107,64 @@ def table_news(data: pd.DataFrame, job: str) -> list:
:param data: Trending data with calculated annomalies to be displayed in the
tables.
:param job: The job name.
+ :param period: The time period (nr of days from now) taken into account.
:type data: pandas.DataFrame
:type job: str
+ :type period: int
+ :returns: List of tables.
+ :rtype: list
"""
+ last_day = datetime.utcnow() - timedelta(days=period)
+ r_list = list()
job_data = data.loc[(data["job"] == job)]
+ r_list.append(_table_info(job_data))
+
failed = job_data["failed"].to_list()[0]
- regressions = {"Test Name": list(), "Last Regression": list()}
+ if failed:
+ r_list.append(_table_failed(job_data, failed))
+
+ title = f"Regressions in the last {period} days"
+ regressions = {title: list(), "Last Regression": list()}
for itm in job_data["regressions"].to_list()[0]:
- regressions["Test Name"].append(itm[0])
- regressions["Last Regression"].append(itm[1].strftime('%Y-%m-%d %H:%M'))
- progressions = {"Test Name": list(), "Last Progression": list()}
+ if itm[1] < last_day:
+ break
+ regressions[title].append(itm[0])
+ regressions["Last Regression"].append(
+ itm[1].strftime('%Y-%m-%d %H:%M'))
+ if regressions["Last Regression"]:
+ r_list.append(_table_gressions(regressions, "warning"))
+
+ title = f"Progressions in the last {period} days"
+ progressions = {title: list(), "Last Progression": list()}
for itm in job_data["progressions"].to_list()[0]:
- progressions["Test Name"].append(itm[0])
+ if itm[1] < last_day:
+ break
+ progressions[title].append(itm[0])
progressions["Last Progression"].append(
itm[1].strftime('%Y-%m-%d %H:%M'))
+ if progressions["Last Progression"]:
+ r_list.append(_table_gressions(progressions, "success"))
- return [
- dbc.Table.from_dataframe(pd.DataFrame.from_dict({
- "Job": job_data["job"],
- "Last Build": job_data["build"],
- "Date": job_data["start"],
- "DUT": job_data["dut_type"],
- "DUT Version": job_data["dut_version"],
- "Hosts": ", ".join(job_data["hosts"].to_list()[0])
- }), bordered=True, striped=True, hover=True, size="sm", color="light"),
- dbc.Table.from_dataframe(pd.DataFrame.from_dict({
- (
- f"Last Failed Tests on "
- f"{job_data['start'].values[0]} ({len(failed)})"
- ): failed
- }), bordered=True, striped=True, hover=True, size="sm", color="light"),
- dbc.Label(
- class_name="p-0",
- size="lg",
- children=(
- f"Regressions during the last {C.NEWS_TIME_PERIOD} days "
- f"({len(regressions['Test Name'])})"
- )
- ),
- dbc.Table.from_dataframe(
- pd.DataFrame.from_dict(regressions),
- bordered=True, striped=True, hover=True, size="sm", color="light"),
- dbc.Label(
- class_name="p-0",
- size="lg",
- children=(
- f"Progressions during the last {C.NEWS_TIME_PERIOD} days "
- f"({len(progressions['Test Name'])})"
- )
- ),
- dbc.Table.from_dataframe(
- pd.DataFrame.from_dict(progressions),
- bordered=True, striped=True, hover=True, size="sm", color="light")
- ]
+ return r_list
+
+
+def table_summary(data: pd.DataFrame, jobs: list) -> list:
+ """Generates summary (failed tests, regressions and progressions) from the
+ last week.
+
+ :param data: Trending data with calculated annomalies to be displayed in the
+ tables.
+ :param jobs: List of jobs.
+ :type data: pandas.DataFrame
+ :type job: str
+ :returns: List of tables.
+ :rtype: list
+ """
+
+ r_list = list()
+ for job in jobs:
+ r_list.extend(table_news(data, job, C.NEWS_SUMMARY_PERIOD))
+ r_list.append(html.Div(html.P(" ")))
+
+ return r_list
diff --git a/resources/tools/dash/app/pal/utils/constants.py b/resources/tools/dash/app/pal/utils/constants.py
index b95a8f5456..1f31185f71 100644
--- a/resources/tools/dash/app/pal/utils/constants.py
+++ b/resources/tools/dash/app/pal/utils/constants.py
@@ -17,7 +17,7 @@
does not need to be hard coded here, but can be read from environment variables.
"""
-
+import logging
import dash_bootstrap_components as dbc
from dash import html
@@ -30,6 +30,11 @@ class Constants:
############################################################################
# General, application wide constants.
+ # Logging settings.
+ LOG_LEVEL = logging.INFO
+ LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s"
+ LOG_DATE_FORMAT = "%Y/%m/%d %H:%M:%S"
+
# The application title.
TITLE = "FD.io CSIT"
@@ -256,6 +261,9 @@ class Constants:
# Time period for regressions and progressions.
NEWS_TIME_PERIOD = TIME_PERIOD # [days]
+ # Time period for summary tables.
+ NEWS_SUMMARY_PERIOD = 7 # [days]
+
############################################################################
# Report.