UTI: Add Summary to News

Change-Id: Ifa3882b36976183f628d2e3171061945ab33bf29
Signed-off-by: Tibor Frank <tifrank@cisco.com>

4 files changed, 196 insertions, 66 deletions

diff --git a/resources/tools/dash/app/pal/__init__.py b/resources/tools/dash/app/pal/__init__.py
index 0eb2a4e79e..1ea6db0fac 100644
--- a/resources/tools/dash/app/pal/__init__.py
+++ b/resources/tools/dash/app/pal/__init__.py
@@ -27,9 +27,9 @@ def init_app():
     """
 
     logging.basicConfig(
-        format=u"%(asctime)s: %(levelname)s: %(message)s",
-        datefmt=u"%Y/%m/%d %H:%M:%S",
-        level=logging.INFO
+        format=C.LOG_FORMAT,
+        datefmt=C.LOG_DATE_FORMAT,
+        level=C.LOG_LEVEL
     )
 
     logging.info("Application started.")
diff --git a/resources/tools/dash/app/pal/news/layout.py b/resources/tools/dash/app/pal/news/layout.py
index 73fabdf884..6e598315e5 100644
--- a/resources/tools/dash/app/pal/news/layout.py
+++ b/resources/tools/dash/app/pal/news/layout.py
@@ -33,7 +33,7 @@ from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url, \
     set_job_params
 from ..utils.url_processing import url_decode
 from ..data.data import Data
-from .tables import table_news
+from .tables import table_news, table_summary
 
 
 class Layout:
@@ -77,7 +77,7 @@ class Layout:
         df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True)
 
         # Prepare information for the control panel:
-        jobs = sorted(list(df_tst_info["job"].unique()))
+        self._jobs = sorted(list(df_tst_info["job"].unique()))
         d_job_info = {
             "job": list(),
             "dut": list(),
@@ -85,7 +85,7 @@ class Layout:
             "cadence": list(),
             "tbed": list()
         }
-        for job in jobs:
+        for job in self._jobs:
             lst_job = job.split("-")
             d_job_info["job"].append(job)
             d_job_info["dut"].append(lst_job[1])
@@ -118,7 +118,9 @@ class Layout:
             "regressions": list(),
             "progressions": list()
         }
-        for job in jobs:
+        logging.debug("Processing jobs ...")
+        for job in self._jobs:
+            logging.debug(f"+ {job}")
             # Create lists of failed tests:
             df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
             last_build = max(df_job["build"].unique())
@@ -254,7 +256,8 @@ class Layout:
                 f"{self._tooltip_file}\n{err}"
             )
 
-        self._default_tab_failed = table_news(self.data, self._default["job"])
+        self._default_tab_failed = \
+            table_news(self.data, self._default["job"], C.NEWS_TIME_PERIOD)
 
         # Callbacks:
         if self._app is not None and hasattr(self, 'callbacks'):
@@ -488,6 +491,25 @@ class Layout:
                             ]
                         ),
                         dbc.Row(
+                            class_name="gy-1 p-0",
+                            children=[
+                                dbc.ButtonGroup(
+                                    [
+                                        dbc.Button(
+                                            id="btn-summary",
+                                            children=(
+                                                f"Show Summary from the last "
+                                                f"{C.NEWS_SUMMARY_PERIOD} Days"
+                                            ),
+                                            class_name="me-1",
+                                            color="info"
+                                        )
+                                    ],
+                                    size="md",
+                                )
+                            ]
+                        ),
+                        dbc.Row(
                             class_name="gy-1",
                             children=[
                                 dbc.Alert(
@@ -598,10 +620,11 @@ class Layout:
             Input("ri-ttypes", "value"),
             Input("ri-cadences", "value"),
             Input("dd-tbeds", "value"),
-            Input("url", "href")
+            Input("url", "href"),
+            Input("btn-summary", "n_clicks")
         )
         def _update_application(cp_data: dict, dut: str, ttype: str,
-                cadence:str, tbed: str, href: str) -> tuple:
+                cadence:str, tbed: str, href: str, btn_all: int) -> tuple:
             """Update the application when the event is detected.
 
             :param cp_data: Current status of the control panel stored in
@@ -630,6 +653,8 @@ class Layout:
             else:
                 url_params = None
 
+            show_summary = False
+
             trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
             if trigger_id == "ri-duts":
                 ttype_opts = generate_options(get_ttypes(self.job_info, dut))
@@ -682,25 +707,37 @@ class Layout:
                 # TODO: Add verification
                 if url_params:
                     new_job = url_params.get("job", list())[0]
-                    if new_job:
+                    if new_job and new_job != "all":
                         job_params = set_job_params(self.job_info, new_job)
                         ctrl_panel = self.ControlPanel(None, job_params)
+                    if new_job and new_job == "all":
+                        show_summary = True
                 else:
                     ctrl_panel = self.ControlPanel(cp_data, self.default)
+            elif trigger_id == "btn-summary":
+                show_summary = True
 
-            job = get_job(
-                self.job_info,
-                ctrl_panel.get("ri-duts-value"),
-                ctrl_panel.get("ri-ttypes-value"),
-                ctrl_panel.get("ri-cadences-value"),
-                ctrl_panel.get("dd-tbeds-value")
-            )
-            ctrl_panel.set({"al-job-children": job})
-            tab_failed = table_news(self.data, job)
+            if show_summary:
+                ctrl_panel.set({
+                    "al-job-children": \
+                        f"Summary from the last {C.NEWS_SUMMARY_PERIOD} days"
+                })
+                job = "all"
+                tables = table_summary(self.data, self._jobs)
+            else:
+                job = get_job(
+                    self.job_info,
+                    ctrl_panel.get("ri-duts-value"),
+                    ctrl_panel.get("ri-ttypes-value"),
+                    ctrl_panel.get("ri-cadences-value"),
+                    ctrl_panel.get("dd-tbeds-value")
+                )
+                ctrl_panel.set({"al-job-children": job})
+                tables = table_news(self.data, job, C.NEWS_TIME_PERIOD)
 
             ret_val = [
                 ctrl_panel.panel,
-                tab_failed,
+                tables,
                 gen_new_url(parsed_url, {"job": job})
             ]
             ret_val.extend(ctrl_panel.values())
diff --git a/resources/tools/dash/app/pal/news/tables.py b/resources/tools/dash/app/pal/news/tables.py
index 1a6c7d2556..04b4fc975e 100644
--- a/resources/tools/dash/app/pal/news/tables.py
+++ b/resources/tools/dash/app/pal/news/tables.py
@@ -17,10 +17,88 @@
 import pandas as pd
 import dash_bootstrap_components as dbc
 
+from datetime import datetime, timedelta
+from dash import html
+
 from ..utils.constants import Constants as C
 
 
-def table_news(data: pd.DataFrame, job: str) -> list:
+def _table_info(job_data: pd.DataFrame) -> dbc.Table:
+    """Generates table with info about the job.
+
+    :param job_data: Dataframe with information about the job.
+    :type job_data: pandas.DataFrame
+    :returns: Table with job info.
+    :rtype: dbc.Table
+    """
+    return dbc.Table.from_dataframe(
+        pd.DataFrame.from_dict(
+            {
+                "Job": job_data["job"],
+                "Last Build": job_data["build"],
+                "Date": job_data["start"],
+                "DUT": job_data["dut_type"],
+                "DUT Version": job_data["dut_version"],
+                "Hosts": ", ".join(job_data["hosts"].to_list()[0])
+            }
+        ),
+        bordered=True,
+        striped=True,
+        hover=True,
+        size="sm",
+        color="info"
+    )
+
+
+def _table_failed(job_data: pd.DataFrame, failed: list) -> dbc.Table:
+    """Generates table with failed tests from the last run of the job.
+
+    :param job_data: Dataframe with information about the job.
+    :param failed: List of failed tests.
+    :type job_data: pandas.DataFrame
+    :type failed: list
+    :returns: Table with fialed tests.
+    :rtype: dbc.Table
+    """
+    return dbc.Table.from_dataframe(
+        pd.DataFrame.from_dict(
+            {
+                (
+                    f"Last Failed Tests on "
+                    f"{job_data['start'].values[0]} ({len(failed)})"
+                ): failed
+            }
+        ),
+        bordered=True,
+        striped=True,
+        hover=True,
+        size="sm",
+        color="danger"
+    )
+
+
+def _table_gressions(itms: dict, color: str) -> dbc.Table:
+    """Generates table with regressions.
+
+    :param itms: Dictionary with items (regressions or progressions) and their
+        last occurence.
+    :param color: Color of the table.
+    :type regressions: dict
+    :type color: str
+    :returns: The table with regressions.
+    :rtype: dbc.Table
+    """
+    return dbc.Table.from_dataframe(
+        pd.DataFrame.from_dict(itms),
+        bordered=True,
+        striped=True,
+        hover=True,
+        size="sm",
+        color=color
+    )
+
+
+def table_news(data: pd.DataFrame, job: str, period: int) -> list:
     """Generates the tables with news:
     1. Falied tests from the last run
     2. Regressions and progressions calculated from the last C.NEWS_TIME_PERIOD
@@ -29,57 +107,64 @@ def table_news(data: pd.DataFrame, job: str) -> list:
     :param data: Trending data with calculated annomalies to be displayed in the
         tables.
     :param job: The job name.
+    :param period: The time period (nr of days from now) taken into account.
     :type data: pandas.DataFrame
     :type job: str
+    :type period: int
+    :returns: List of tables.
+    :rtype: list
     """
 
+    last_day = datetime.utcnow() - timedelta(days=period)
+    r_list = list()
     job_data = data.loc[(data["job"] == job)]
+    r_list.append(_table_info(job_data))
+
     failed = job_data["failed"].to_list()[0]
-    regressions = {"Test Name": list(), "Last Regression": list()}
+    if failed:
+        r_list.append(_table_failed(job_data, failed))
+
+    title = f"Regressions in the last {period} days"
+    regressions = {title: list(), "Last Regression": list()}
     for itm in job_data["regressions"].to_list()[0]:
-        regressions["Test Name"].append(itm[0])
-        regressions["Last Regression"].append(itm[1].strftime('%Y-%m-%d %H:%M'))
-    progressions = {"Test Name": list(), "Last Progression": list()}
+        if itm[1] < last_day:
+            break
+        regressions[title].append(itm[0])
+        regressions["Last Regression"].append(
+            itm[1].strftime('%Y-%m-%d %H:%M'))
+    if regressions["Last Regression"]:
+        r_list.append(_table_gressions(regressions, "warning"))
+
+    title = f"Progressions in the last {period} days"
+    progressions = {title: list(), "Last Progression": list()}
     for itm in job_data["progressions"].to_list()[0]:
-        progressions["Test Name"].append(itm[0])
+        if itm[1] < last_day:
+            break
+        progressions[title].append(itm[0])
         progressions["Last Progression"].append(
             itm[1].strftime('%Y-%m-%d %H:%M'))
+    if progressions["Last Progression"]:
+        r_list.append(_table_gressions(progressions, "success"))
 
-    return [
-        dbc.Table.from_dataframe(pd.DataFrame.from_dict({
-            "Job": job_data["job"],
-            "Last Build": job_data["build"],
-            "Date": job_data["start"],
-            "DUT": job_data["dut_type"],
-            "DUT Version": job_data["dut_version"],
-            "Hosts": ", ".join(job_data["hosts"].to_list()[0])
-        }), bordered=True, striped=True, hover=True, size="sm", color="light"),
-        dbc.Table.from_dataframe(pd.DataFrame.from_dict({
-            (
-                f"Last Failed Tests on "
-                f"{job_data['start'].values[0]} ({len(failed)})"
-            ): failed
-        }), bordered=True, striped=True, hover=True, size="sm", color="light"),
-        dbc.Label(
-            class_name="p-0",
-            size="lg",
-            children=(
-                f"Regressions during the last {C.NEWS_TIME_PERIOD} days "
-                f"({len(regressions['Test Name'])})"
-            )
-        ),
-        dbc.Table.from_dataframe(
-            pd.DataFrame.from_dict(regressions),
-            bordered=True, striped=True, hover=True, size="sm", color="light"),
-        dbc.Label(
-            class_name="p-0",
-            size="lg",
-            children=(
-                f"Progressions during the last {C.NEWS_TIME_PERIOD} days "
-                f"({len(progressions['Test Name'])})"
-            )
-        ),
-        dbc.Table.from_dataframe(
-            pd.DataFrame.from_dict(progressions),
-            bordered=True, striped=True, hover=True, size="sm", color="light")
-    ]
+    return r_list
+
+
+def table_summary(data: pd.DataFrame, jobs: list) -> list:
+    """Generates summary (failed tests, regressions and progressions) from the
+    last week.
+
+    :param data: Trending data with calculated annomalies to be displayed in the
+        tables.
+    :param jobs: List of jobs.
+    :type data: pandas.DataFrame
+    :type job: str
+    :returns: List of tables.
+    :rtype: list
+    """
+
+    r_list = list()
+    for job in jobs:
+        r_list.extend(table_news(data, job, C.NEWS_SUMMARY_PERIOD))
+        r_list.append(html.Div(html.P(" ")))
+
+    return r_list
diff --git a/resources/tools/dash/app/pal/utils/constants.py b/resources/tools/dash/app/pal/utils/constants.py
index b95a8f5456..1f31185f71 100644
--- a/resources/tools/dash/app/pal/utils/constants.py
+++ b/resources/tools/dash/app/pal/utils/constants.py
@@ -17,7 +17,7 @@
 does not need to be hard coded here, but can be read from environment variables.
 """
 
-
+import logging
 import dash_bootstrap_components as dbc
 
 from dash import html
@@ -30,6 +30,11 @@ class Constants:
     ############################################################################
     # General, application wide constants.
 
+    # Logging settings.
+    LOG_LEVEL = logging.INFO
+    LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s"
+    LOG_DATE_FORMAT = "%Y/%m/%d %H:%M:%S"
+
     # The application title.
     TITLE = "FD.io CSIT"
 
@@ -256,6 +261,9 @@ class Constants:
     # Time period for regressions and progressions.
     NEWS_TIME_PERIOD = TIME_PERIOD  # [days]
 
+    # Time period for summary tables.
+    NEWS_SUMMARY_PERIOD = 7  # [days]
+
     ############################################################################
     # Report.