feat(uti): Add structured menu for stats

Change-Id: I907606b1b994e991975f887611750d23fd7f1634
Signed-off-by: Tibor Frank <tifrank@cisco.com>

3 files changed, 287 insertions, 28 deletions

diff --git a/resources/tools/dash/app/pal/stats/graphs.py b/resources/tools/dash/app/pal/stats/graphs.py
index 2fabf8e6ae..37fc1b2e73 100644
--- a/resources/tools/dash/app/pal/stats/graphs.py
+++ b/resources/tools/dash/app/pal/stats/graphs.py
@@ -40,8 +40,10 @@ def graph_statistics(df: pd.DataFrame, job:str, layout: dict,
     if data.empty:
         return None, None
 
-    x_axis = [d for d in data["start_time"] if d >= start and d <= end]
-    if not x_axis:
+    data = data.loc[(
+        (data["start_time"] >= start) & (data["start_time"] <= end)
+    )]
+    if data.empty:
         return None, None
 
     hover = list()
@@ -62,7 +64,7 @@ def graph_statistics(df: pd.DataFrame, job:str, layout: dict,
     # Job durations:
     fig_duration = go.Figure(
         data=go.Scatter(
-            x=x_axis,
+            x=data["start_time"],
             y=data["duration"],
             name=u"Duration",
             text=hover,
@@ -87,14 +89,14 @@ def graph_statistics(df: pd.DataFrame, job:str, layout: dict,
     fig_passed = go.Figure(
         data=[
             go.Bar(
-                x=x_axis,
+                x=data["start_time"],
                 y=data["passed"],
                 name=u"Passed",
                 hovertext=hover,
                 hoverinfo=u"text"
             ),
             go.Bar(
-                x=x_axis,
+                x=data["start_time"],
                 y=data["failed"],
                 name=u"Failed",
                 hovertext=hover,
diff --git a/resources/tools/dash/app/pal/stats/layout.py b/resources/tools/dash/app/pal/stats/layout.py
index 0c5c22e81b..405fd8b1a3 100644
--- a/resources/tools/dash/app/pal/stats/layout.py
+++ b/resources/tools/dash/app/pal/stats/layout.py
@@ -21,10 +21,11 @@ from flask import Flask
 from dash import dcc
 from dash import html
 from dash import callback_context, no_update
-from dash import Input, Output
+from dash import Input, Output, State
 from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from datetime import datetime, timedelta
+from copy import deepcopy
 
 from ..data.data import Data
 from .graphs import graph_statistics
@@ -34,6 +35,8 @@ class Layout:
     """
     """
 
+    DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
+
     def __init__(self, app: Flask, html_layout_file: str, spec_file: str,
         graph_layout_file: str, data_spec_file: str,
         time_period: int=None) -> None:
@@ -68,6 +71,36 @@ class Layout:
             self._time_period = data_time_period
 
         self._jobs = sorted(list(data_stats["job"].unique()))
+        job_info = {
+            "job": list(),
+            "dut": list(),
+            "ttype": list(),
+            "cadence": list(),
+            "tbed": list()
+        }
+        for job in self._jobs:
+            lst_job = job.split("-")
+            job_info["job"].append(job)
+            job_info["dut"].append(lst_job[1])
+            job_info["ttype"].append(lst_job[3])
+            job_info["cadence"].append(lst_job[4])
+            job_info["tbed"].append("-".join(lst_job[-2:]))
+        self.df_job_info = pd.DataFrame.from_dict(job_info)
+
+        lst_job = self.DEFAULT_JOB.split("-")
+        self._default = {
+            "job": self.DEFAULT_JOB,
+            "dut": lst_job[1],
+            "ttype": lst_job[3],
+            "cadence": lst_job[4],
+            "tbed": "-".join(lst_job[-2:]),
+            "duts": self._generate_options(self._get_duts()),
+            "ttypes": self._generate_options(self._get_ttypes(lst_job[1])),
+            "cadences": self._generate_options(self._get_cadences(
+                lst_job[1], lst_job[3])),
+            "tbeds": self._generate_options(self._get_test_beds(
+                lst_job[1], lst_job[3], lst_job[4]))
+        }
 
         tst_info = {
             "job": list(),
@@ -129,7 +162,7 @@ class Layout:
             )
 
         self._default_fig_passed, self._default_fig_duration = graph_statistics(
-            self.data, self.jobs[0], self.layout
+            self.data, self._default["job"], self.layout
         )
 
         # Callbacks:
@@ -149,12 +182,54 @@ class Layout:
         return self._graph_layout
 
     @property
-    def jobs(self) -> list:
-        return self._jobs
+    def time_period(self) -> int:
+        return self._time_period
 
     @property
-    def time_period(self):
-        return self._time_period
+    def default(self) -> any:
+        return self._default
+
+    def _get_duts(self) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info["dut"].unique()))
+
+    def _get_ttypes(self, dut: str) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info.loc[(
+            self.df_job_info["dut"] == dut
+        )]["ttype"].unique()))
+
+    def _get_cadences(self, dut: str, ttype: str) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info.loc[(
+            (self.df_job_info["dut"] == dut) &
+            (self.df_job_info["ttype"] == ttype)
+        )]["cadence"].unique()))
+
+    def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list:
+        """
+        """
+        return sorted(list(self.df_job_info.loc[(
+            (self.df_job_info["dut"] == dut) &
+            (self.df_job_info["ttype"] == ttype) &
+            (self.df_job_info["cadence"] == cadence)
+        )]["tbed"].unique()))
+
+    def _get_job(self, dut, ttype, cadence, testbed):
+        """Get the name of a job defined by dut, ttype, cadence, testbed.
+
+        Input information comes from control panel.
+        """
+        return self.df_job_info.loc[(
+            (self.df_job_info["dut"] == dut) &
+            (self.df_job_info["ttype"] == ttype) &
+            (self.df_job_info["cadence"] == cadence) &
+            (self.df_job_info["tbed"] == testbed)
+        )]["job"].item()
+
 
     def add_content(self):
         """
@@ -163,6 +238,9 @@ class Layout:
             return html.Div(
                 id="div-main",
                 children=[
+                    dcc.Store(
+                        id="control-panel"
+                    ),
                     dbc.Row(
                         id="row-navbar",
                         class_name="g-0",
@@ -296,12 +374,74 @@ class Layout:
                 dbc.Row(
                     class_name="g-0 p-2",
                     children=[
-                        dbc.Label("Choose the Trending Job"),
-                        dbc.RadioItems(
-                            id="ri_job",
-                            value=self.jobs[0],
-                            options=[
-                                {"label": i, "value": i} for i in self.jobs
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    "Device under Test",
+                                    class_name="p-0"
+                                ),
+                                dbc.RadioItems(
+                                    id="ri-duts",
+                                    inline=True,
+                                    value=self.default["dut"],
+                                    options=self.default["duts"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    "Test Type",
+                                    class_name="p-0"
+                                ),
+                                dbc.RadioItems(
+                                    id="ri-ttypes",
+                                    inline=True,
+                                    value=self.default["ttype"],
+                                    options=self.default["ttypes"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    "Cadence",
+                                    class_name="p-0"
+                                ),
+                                dbc.RadioItems(
+                                    id="ri-cadences",
+                                    inline=True,
+                                    value=self.default["cadence"],
+                                    options=self.default["cadences"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Label(
+                                    "Test Bed",
+                                    class_name="p-0"
+                                ),
+                                dbc.Select(
+                                    id="dd-tbeds",
+                                    placeholder="Select a test bed...",
+                                    value=self.default["tbed"],
+                                    options=self.default["tbeds"]
+                                )
+                            ]
+                        ),
+                        dbc.Row(
+                            class_name="gy-1",
+                            children=[
+                                dbc.Alert(
+                                    id="al-job",
+                                    color="info",
+                                    children=self.default["job"]
+                                )
                             ]
                         )
                     ]
@@ -329,29 +469,148 @@ class Layout:
             ]
         )
 
+    class ControlPanel:
+        def __init__(self, panel: dict, default: dict) -> None:
+            self._defaults = {
+                "ri-ttypes-options": default["ttypes"],
+                "ri-cadences-options": default["cadences"],
+                "dd-tbeds-options": default["tbeds"],
+                "ri-duts-value": default["dut"],
+                "ri-ttypes-value": default["ttype"],
+                "ri-cadences-value": default["cadence"],
+                "dd-tbeds-value": default["tbed"],
+                "al-job-children": default["job"]
+            }
+            self._panel = deepcopy(self._defaults)
+            if panel:
+                for key in self._defaults:
+                    self._panel[key] = panel[key]
+
+        def set(self, kwargs: dict) -> None:
+            for key, val in kwargs.items():
+                if key in self._panel:
+                    self._panel[key] = val
+                else:
+                    raise KeyError(f"The key {key} is not defined.")
+
+        @property
+        def defaults(self) -> dict:
+            return self._defaults
+
+        @property
+        def panel(self) -> dict:
+            return self._panel
+
+        def get(self, key: str) -> any:
+            return self._panel[key]
+
+        def values(self) -> list:
+            return list(self._panel.values())
+
+    @staticmethod
+    def _generate_options(opts: list) -> list:
+        """
+        """
+        return [{"label": i, "value": i} for i in opts]
+
     def callbacks(self, app):
 
         @app.callback(
+            Output("control-panel", "data"),  # Store
             Output("graph-passed", "figure"),
             Output("graph-duration", "figure"),
-            Input("ri_job", "value"),
+            Output("ri-ttypes", "options"),
+            Output("ri-cadences", "options"),
+            Output("dd-tbeds", "options"),
+            Output("ri-duts", "value"),
+            Output("ri-ttypes", "value"),
+            Output("ri-cadences", "value"),
+            Output("dd-tbeds", "value"),
+            Output("al-job", "children"),
+            State("control-panel", "data"),  # Store
+            Input("ri-duts", "value"),
+            Input("ri-ttypes", "value"),
+            Input("ri-cadences", "value"),
+            Input("dd-tbeds", "value"),
             Input("dpr-period", "start_date"),
             Input("dpr-period", "end_date"),
             prevent_initial_call=True
         )
-        def _update_ctrl_panel(job:str, d_start: str, d_end: str) -> tuple:
+        def _update_ctrl_panel(cp_data: dict, dut:str, ttype: str, cadence:str,
+                tbed: str, d_start: str, d_end: str) -> tuple:
             """
             """
 
+            ctrl_panel = self.ControlPanel(cp_data, self.default)
+
             d_start = datetime(int(d_start[0:4]), int(d_start[5:7]),
                 int(d_start[8:10]))
             d_end = datetime(int(d_end[0:4]), int(d_end[5:7]), int(d_end[8:10]))
 
-            fig_passed, fig_duration = graph_statistics(
-                self.data, job, self.layout, d_start, d_end
+            trigger_id = callback_context.triggered[0]["prop_id"].split(".")[0]
+            if trigger_id == "ri-duts":
+                ttype_opts = self._generate_options(self._get_ttypes(dut))
+                ttype_val = ttype_opts[0]["value"]
+                cad_opts = self._generate_options(
+                    self._get_cadences(dut, ttype_val))
+                cad_val = cad_opts[0]["value"]
+                tbed_opts = self._generate_options(
+                    self._get_test_beds(dut, ttype_val, cad_val))
+                tbed_val = tbed_opts[0]["value"]
+                ctrl_panel.set({
+                    "ri-duts-value": dut,
+                    "ri-ttypes-options": ttype_opts,
+                    "ri-ttypes-value": ttype_val,
+                    "ri-cadences-options": cad_opts,
+                    "ri-cadences-value": cad_val,
+                    "dd-tbeds-options": tbed_opts,
+                    "dd-tbeds-value": tbed_val
+                })
+            elif trigger_id == "ri-ttypes":
+                cad_opts = self._generate_options(
+                    self._get_cadences(ctrl_panel.get("ri-duts-value"), ttype))
+                cad_val = cad_opts[0]["value"]
+                tbed_opts = self._generate_options(
+                    self._get_test_beds(ctrl_panel.get("ri-duts-value"),
+                    ttype, cad_val))
+                tbed_val = tbed_opts[0]["value"]
+                ctrl_panel.set({
+                    "ri-ttypes-value": ttype,
+                    "ri-cadences-options": cad_opts,
+                    "ri-cadences-value": cad_val,
+                    "dd-tbeds-options": tbed_opts,
+                    "dd-tbeds-value": tbed_val
+                })
+            elif trigger_id == "ri-cadences":
+                tbed_opts = self._generate_options(
+                    self._get_test_beds(ctrl_panel.get("ri-duts-value"),
+                    ctrl_panel.get("ri-ttypes-value"), cadence))
+                tbed_val = tbed_opts[0]["value"]
+                ctrl_panel.set({
+                    "ri-cadences-value": cadence,
+                    "dd-tbeds-options": tbed_opts,
+                    "dd-tbeds-value": tbed_val
+                })
+            elif trigger_id == "dd-tbeds":
+                ctrl_panel.set({
+                    "dd-tbeds-value": tbed
+                })
+            elif trigger_id == "dpr-period":
+                pass
+
+            job = self._get_job(
+                ctrl_panel.get("ri-duts-value"),
+                ctrl_panel.get("ri-ttypes-value"),
+                ctrl_panel.get("ri-cadences-value"),
+                ctrl_panel.get("dd-tbeds-value")
             )
+            ctrl_panel.set({"al-job-children": job})
+            fig_passed, fig_duration = graph_statistics(
+                self.data, job, self.layout, d_start, d_end)
 
-            return fig_passed, fig_duration
+            ret_val = [ctrl_panel.panel, fig_passed, fig_duration]
+            ret_val.extend(ctrl_panel.values())
+            return ret_val
 
         @app.callback(
             Output("download-data", "data"),
diff --git a/resources/tools/dash/app/pal/trending/graphs.py b/resources/tools/dash/app/pal/trending/graphs.py
index 16cb5a2cb3..1d9fd1ccfa 100644
--- a/resources/tools/dash/app/pal/trending/graphs.py
+++ b/resources/tools/dash/app/pal/trending/graphs.py
@@ -201,11 +201,12 @@ def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
     df = df.dropna(subset=[_VALUE[ttype], ])
     if df.empty:
         return list()
-
-    x_axis = [d for d in df["start_time"] if d >= start and d <= end]
-    if not x_axis:
+    df = df.loc[((df["start_time"] >= start) & (df["start_time"] <= end))]
+    if df.empty:
         return list()
 
+    x_axis = df["start_time"].tolist()
+
     anomalies, trend_avg, trend_stdev = _classify_anomalies(
         {k: v for k, v in zip(x_axis, df[_VALUE[ttype]])}
     )
@@ -327,9 +328,6 @@ def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
                         u"len": 0.8,
                         u"title": u"Circles Marking Data Classification",
                         u"titleside": u"right",
-                        # u"titlefont": {
-                        #     u"size": 14
-                        # },
                         u"tickmode": u"array",
                         u"tickvals": [0.167, 0.500, 0.833],
                         u"ticktext": _TICK_TEXT_LAT \