C-Dash: Pre-load the data from parquets

Signed-off-by: Tibor Frank <tifrank@cisco.com>
Change-Id: I20792792469c10d1db2e891b76879ec8ced1b7d3

2 files changed, 27 insertions, 47 deletions

diff --git a/csit.infra.dash/app/cdash/news/layout.py b/csit.infra.dash/app/cdash/news/layout.py
index 11151d727a..da36b1430c 100644
--- a/csit.infra.dash/app/cdash/news/layout.py
+++ b/csit.infra.dash/app/cdash/news/layout.py
@@ -14,7 +14,6 @@
 """Plotly Dash HTML layout override.
 """
 
-import logging
 import pandas as pd
 import dash_bootstrap_components as dbc
 
@@ -23,13 +22,10 @@ from dash import dcc
 from dash import html
 from dash import callback_context
 from dash import Input, Output, State
-from yaml import load, FullLoader, YAMLError
 
-from ..data.data import Data
 from ..utils.constants import Constants as C
-from ..utils.utils import classify_anomalies, show_tooltip, gen_new_url
+from ..utils.utils import classify_anomalies, gen_new_url
 from ..utils.url_processing import url_decode
-from ..data.data import Data
 from .tables import table_summary
 
 
@@ -37,8 +33,13 @@ class Layout:
     """The layout of the dash app and the callbacks.
     """
 
-    def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str,
-        tooltip_file: str) -> None:
+    def __init__(
+            self,
+            app: Flask,
+            data_stats: pd.DataFrame,
+            data_trending: pd.DataFrame,
+            html_layout_file: str
+        ) -> None:
         """Initialization:
         - save the input parameters,
         - read and pre-process the data,
@@ -47,38 +48,22 @@ class Layout:
         - read tooltips from the tooltip file.
 
         :param app: Flask application running the dash application.
+        :param data_stats: Pandas dataframe with staistical data.
+        :param data_trending: Pandas dataframe with trending data.
         :param html_layout_file: Path and name of the file specifying the HTML
             layout of the dash application.
-        :param data_spec_file: Path and name of the file specifying the data to
-            be read from parquets for this application.
-        :param tooltip_file: Path and name of the yaml file specifying the
-            tooltips.
         :type app: Flask
+        :type data_stats: pandas.DataFrame
+        :type data_trending: pandas.DataFrame
         :type html_layout_file: str
-        :type data_spec_file: str
-        :type tooltip_file: str
         """
 
         # Inputs
         self._app = app
         self._html_layout_file = html_layout_file
-        self._data_spec_file = data_spec_file
-        self._tooltip_file = tooltip_file
-
-        # Read the data:
-        data_stats, data_mrr, data_ndrpdr = Data(
-            data_spec_file=self._data_spec_file,
-            debug=True
-        ).read_stats(days=C.NEWS_TIME_PERIOD)
-
-        df_tst_info = pd.concat(
-            [data_mrr, data_ndrpdr],
-            ignore_index=True,
-            copy=False
-        )
 
         # Prepare information for the control panel:
-        self._jobs = sorted(list(df_tst_info["job"].unique()))
+        self._jobs = sorted(list(data_trending["job"].unique()))
         d_job_info = {
             "job": list(),
             "dut": list(),
@@ -119,7 +104,7 @@ class Layout:
         }
         for job in self._jobs:
             # Create lists of failed tests:
-            df_job = df_tst_info.loc[(df_tst_info["job"] == job)]
+            df_job = data_trending.loc[(data_trending["job"] == job)]
             last_build = str(max(pd.to_numeric(df_job["build"].unique())))
             df_build = df_job.loc[(df_job["build"] == last_build)]
             tst_info["job"].append(job)
@@ -230,7 +215,6 @@ class Layout:
 
         # Read from files:
         self._html_layout = str()
-        self._tooltips = dict()
 
         try:
             with open(self._html_layout_file, "r") as file_read:
@@ -240,19 +224,6 @@ class Layout:
                 f"Not possible to open the file {self._html_layout_file}\n{err}"
             )
 
-        try:
-            with open(self._tooltip_file, "r") as file_read:
-                self._tooltips = load(file_read, Loader=FullLoader)
-        except IOError as err:
-            logging.warning(
-                f"Not possible to open the file {self._tooltip_file}\n{err}"
-            )
-        except YAMLError as err:
-            logging.warning(
-                f"An error occurred while parsing the specification file "
-                f"{self._tooltip_file}\n{err}"
-            )
-
         self._default_period = C.NEWS_SHORT
         self._default_active = (False, True, False)
 
diff --git a/csit.infra.dash/app/cdash/news/news.py b/csit.infra.dash/app/cdash/news/news.py
index eda70d385f..b5cc5483a8 100644
--- a/csit.infra.dash/app/cdash/news/news.py
+++ b/csit.infra.dash/app/cdash/news/news.py
@@ -14,16 +14,25 @@
 """Instantiate the News Dash application.
 """
 import dash
+import pandas as pd
 
 from ..utils.constants import Constants as C
 from .layout import Layout
 
 
-def init_news(server):
+def init_news(
+        server,
+        data_stats: pd.DataFrame,
+        data_trending: pd.DataFrame
+    ) -> dash.Dash:
     """Create a Plotly Dash dashboard.
 
     :param server: Flask server.
     :type server: Flask
+    :param data_stats: Pandas dataframe with staistical data.
+    :param data_trending: Pandas dataframe with trending data.
+    :type data_stats: pandas.DataFrame
+    :type data_trending: pandas.DataFrame
     :returns: Dash app server.
     :rtype: Dash
     """
@@ -37,9 +46,9 @@ def init_news(server):
 
     layout = Layout(
         app=dash_app,
-        html_layout_file=C.HTML_LAYOUT_FILE,
-        data_spec_file=C.DATA_SPEC_FILE,
-        tooltip_file=C.TOOLTIP_FILE,
+        data_stats=data_stats,
+        data_trending=data_trending,
+        html_layout_file=C.HTML_LAYOUT_FILE
     )
     dash_app.index_string = layout.html_layout
     dash_app.layout = layout.add_content()