From c31372861134f29ae6eec8d98874e030e57ab5f1 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Fri, 27 Jan 2023 08:26:25 +0100 Subject: C-Dash: Pre-load the data from parquets Signed-off-by: Tibor Frank Change-Id: I20792792469c10d1db2e891b76879ec8ced1b7d3 --- csit.infra.dash/app/cdash/stats/layout.py | 45 ++++++++++--------------------- csit.infra.dash/app/cdash/stats/stats.py | 17 +++++++++--- 2 files changed, 27 insertions(+), 35 deletions(-) (limited to 'csit.infra.dash/app/cdash/stats') diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py index 116185d62c..ecd81bacbe 100644 --- a/csit.infra.dash/app/cdash/stats/layout.py +++ b/csit.infra.dash/app/cdash/stats/layout.py @@ -25,14 +25,12 @@ from dash import callback_context, no_update from dash import Input, Output, State from dash.exceptions import PreventUpdate from yaml import load, FullLoader, YAMLError -from datetime import datetime from ..utils.constants import Constants as C from ..utils.control_panel import ControlPanel from ..utils.utils import show_tooltip, gen_new_url, get_ttypes, get_cadences, \ get_test_beds, get_job, generate_options, set_job_params from ..utils.url_processing import url_decode -from ..data.data import Data from .graphs import graph_statistics, select_data @@ -40,9 +38,15 @@ class Layout: """The layout of the dash app and the callbacks. """ - def __init__(self, app: Flask, html_layout_file: str, - graph_layout_file: str, data_spec_file: str, tooltip_file: str, - time_period: int=None) -> None: + def __init__( + self, + app: Flask, + data_stats: pd.DataFrame, + data_trending: pd.DataFrame, + html_layout_file: str, + graph_layout_file: str, + tooltip_file: str + ) -> None: """Initialization: - save the input parameters, - read and pre-process the data, @@ -51,43 +55,27 @@ class Layout: - read tooltips from the tooltip file. :param app: Flask application running the dash application. + :param data_stats: Pandas dataframe with staistical data. + :param data_trending: Pandas dataframe with trending data. :param html_layout_file: Path and name of the file specifying the HTML layout of the dash application. :param graph_layout_file: Path and name of the file with layout of plot.ly graphs. - :param data_spec_file: Path and name of the file specifying the data to - be read from parquets for this application. :param tooltip_file: Path and name of the yaml file specifying the tooltips. - :param time_period: It defines the time period for data read from the - parquets in days from now back to the past. :type app: Flask + :type data_stats: pandas.DataFrame + :type data_trending: pandas.DataFrame :type html_layout_file: str :type graph_layout_file: str - :type data_spec_file: str :type tooltip_file: str - :type time_period: int """ # Inputs self._app = app self._html_layout_file = html_layout_file self._graph_layout_file = graph_layout_file - self._data_spec_file = data_spec_file self._tooltip_file = tooltip_file - self._time_period = time_period - - # Read the data: - data_stats, data_mrr, data_ndrpdr = Data( - data_spec_file=self._data_spec_file, - debug=True - ).read_stats(days=self._time_period) - - df_tst_info = pd.concat( - [data_mrr, data_ndrpdr], - ignore_index=True, - copy=False - ) # Pre-process the data: data_stats = data_stats[~data_stats.job.str.contains("-verify-")] @@ -95,11 +83,6 @@ class Layout: data_stats = data_stats[~data_stats.job.str.contains("-iterative-")] data_stats = data_stats[["job", "build", "start_time", "duration"]] - data_time_period = \ - (datetime.utcnow() - data_stats["start_time"].min()).days - if self._time_period > data_time_period: - self._time_period = data_time_period - jobs = sorted(list(data_stats["job"].unique())) d_job_info = { "job": list(), @@ -130,7 +113,7 @@ class Layout: "lst_failed": list() } for job in jobs: - df_job = df_tst_info.loc[(df_tst_info["job"] == job)] + df_job = data_trending.loc[(data_trending["job"] == job)] builds = df_job["build"].unique() for build in builds: df_build = df_job.loc[(df_job["build"] == build)] diff --git a/csit.infra.dash/app/cdash/stats/stats.py b/csit.infra.dash/app/cdash/stats/stats.py index 062e6b0bba..fdeef8b2f7 100644 --- a/csit.infra.dash/app/cdash/stats/stats.py +++ b/csit.infra.dash/app/cdash/stats/stats.py @@ -14,16 +14,25 @@ """Instantiate the Statistics Dash application. """ import dash +import pandas as pd from ..utils.constants import Constants as C from .layout import Layout -def init_stats(server, time_period=None): +def init_stats( + server, + data_stats: pd.DataFrame, + data_trending: pd.DataFrame + ) -> dash.Dash: """Create a Plotly Dash dashboard. :param server: Flask server. + :param data_stats: Pandas dataframe with staistical data. + :param data_trending: Pandas dataframe with trending data. :type server: Flask + :type data_stats: pandas.DataFrame + :type data_trending: pandas.DataFrame :returns: Dash app server. :rtype: Dash """ @@ -37,11 +46,11 @@ def init_stats(server, time_period=None): layout = Layout( app=dash_app, + data_stats=data_stats, + data_trending=data_trending, html_layout_file=C.HTML_LAYOUT_FILE, graph_layout_file=C.STATS_GRAPH_LAYOUT_FILE, - data_spec_file=C.DATA_SPEC_FILE, - tooltip_file=C.TOOLTIP_FILE, - time_period=time_period + tooltip_file=C.TOOLTIP_FILE ) dash_app.index_string = layout.html_layout dash_app.layout = layout.add_content() -- cgit 1.2.3-korg