diff options
| -rw-r--r-- | resources/tools/dash/app/pal/__init__.py | 1 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/data/data.py | 88 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/data/data.yaml | 103 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/data/url_processing.py | 22 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/news/layout.py | 175 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/routes.py | 10 |
6 files changed, 254 insertions, 145 deletions
diff --git a/resources/tools/dash/app/pal/__init__.py b/resources/tools/dash/app/pal/__init__.py index f66edceafc..9f80c5fcaa 100644 --- a/resources/tools/dash/app/pal/__init__.py +++ b/resources/tools/dash/app/pal/__init__.py @@ -57,6 +57,7 @@ def init_app(): assets = Environment() assets.init_app(app) + # Set the time period for Trending if TIME_PERIOD is None or TIME_PERIOD > MAX_TIME_PERIOD: time_period = MAX_TIME_PERIOD else: diff --git a/resources/tools/dash/app/pal/data/data.py b/resources/tools/dash/app/pal/data/data.py index efe2a2d1b6..f2c02acc63 100644 --- a/resources/tools/dash/app/pal/data/data.py +++ b/resources/tools/dash/app/pal/data/data.py @@ -11,7 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Prepare data for Plotly Dash.""" +"""Prepare data for Plotly Dash applications. +""" import logging @@ -27,11 +28,20 @@ from awswrangler.exceptions import EmptyDataFrame, NoFilesFound class Data: - """ + """Gets the data from parquets and stores it for further use by dash + applications. """ def __init__(self, data_spec_file: str, debug: bool=False) -> None: - """ + """Initialize the Data object. + + :param data_spec_file: Path to file specifying the data to be read from + parquets. + :param debug: If True, the debuf information is printed to stdout. + :type data_spec_file: str + :type debug: bool + :raises RuntimeError: if it is not possible to open data_spec_file or it + is not a valid yaml file. """ # Inputs: @@ -64,6 +74,17 @@ class Data: return self._data def _get_columns(self, parquet: str) -> list: + """Get the list of columns from the data specification file to be read + from parquets. + + :param parquet: The parquet's name. + :type parquet: str + :raises RuntimeError: if the parquet is not defined in the data + specification file or it does not have any columns specified. + :returns: List of columns. + :rtype: list + """ + try: return self._data_spec[parquet]["columns"] except KeyError as err: @@ -74,6 +95,17 @@ class Data: ) def _get_path(self, parquet: str) -> str: + """Get the path from the data specification file to be read from + parquets. + + :param parquet: The parquet's name. + :type parquet: str + :raises RuntimeError: if the parquet is not defined in the data + specification file or it does not have the path specified. + :returns: Path. + :rtype: str + """ + try: return self._data_spec[parquet]["path"] except KeyError as err: @@ -84,9 +116,12 @@ class Data: ) def _create_dataframe_from_parquet(self, - path, partition_filter=None, columns=None, - validate_schema=False, last_modified_begin=None, - last_modified_end=None, days=None) -> DataFrame: + path, partition_filter=None, + columns=None, + validate_schema=False, + last_modified_begin=None, + last_modified_end=None, + days=None) -> DataFrame: """Read parquet stored in S3 compatible storage and returns Pandas Dataframe. @@ -151,8 +186,21 @@ class Data: return df def read_stats(self, days: int=None) -> tuple: - """Read Suite Result Analysis data partition from parquet. + """Read statistics from parquet. + + It reads from: + - Suite Result Analysis (SRA) partition, + - NDRPDR trending partition, + - MRR trending partition. + + :param days: Number of days back to the past for which the data will be + read. + :type days: int + :returns: tuple of pandas DataFrame-s with data read from specified + parquets. + :rtype: tuple of pandas DataFrame-s """ + l_stats = lambda part: True if part["stats_type"] == "sra" else False l_mrr = lambda part: True if part["test_type"] == "mrr" else False l_ndrpdr = lambda part: True if part["test_type"] == "ndrpdr" else False @@ -180,7 +228,14 @@ class Data: def read_trending_mrr(self, days: int=None) -> DataFrame: """Read MRR data partition from parquet. + + :param days: Number of days back to the past for which the data will be + read. + :type days: int + :returns: Pandas DataFrame with read data. + :rtype: DataFrame """ + lambda_f = lambda part: True if part["test_type"] == "mrr" else False return self._create_dataframe_from_parquet( @@ -192,7 +247,14 @@ class Data: def read_trending_ndrpdr(self, days: int=None) -> DataFrame: """Read NDRPDR data partition from iterative parquet. + + :param days: Number of days back to the past for which the data will be + read. + :type days: int + :returns: Pandas DataFrame with read data. + :rtype: DataFrame """ + lambda_f = lambda part: True if part["test_type"] == "ndrpdr" else False return self._create_dataframe_from_parquet( @@ -204,7 +266,13 @@ class Data: def read_iterative_mrr(self, release: str) -> DataFrame: """Read MRR data partition from iterative parquet. + + :param release: The CSIT release from which the data will be read. + :type release: str + :returns: Pandas DataFrame with read data. + :rtype: DataFrame """ + lambda_f = lambda part: True if part["test_type"] == "mrr" else False return self._create_dataframe_from_parquet( @@ -215,7 +283,13 @@ class Data: def read_iterative_ndrpdr(self, release: str) -> DataFrame: """Read NDRPDR data partition from parquet. + + :param release: The CSIT release from which the data will be read. + :type release: str + :returns: Pandas DataFrame with read data. + :rtype: DataFrame """ + lambda_f = lambda part: True if part["test_type"] == "ndrpdr" else False return self._create_dataframe_from_parquet( diff --git a/resources/tools/dash/app/pal/data/data.yaml b/resources/tools/dash/app/pal/data/data.yaml index 69f7165dc4..2585ef0e84 100644 --- a/resources/tools/dash/app/pal/data/data.yaml +++ b/resources/tools/dash/app/pal/data/data.yaml @@ -26,13 +26,10 @@ trending-mrr: - start_time - passed - test_id - # - test_name_long - # - test_name_short - version - result_receive_rate_rate_avg - result_receive_rate_rate_stdev - result_receive_rate_rate_unit - # - result_receive_rate_rate_values trending-ndrpdr: path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/trending columns: @@ -44,65 +41,21 @@ trending-ndrpdr: - start_time - passed - test_id - # - test_name_long - # - test_name_short - version - # - result_pdr_upper_rate_unit - # - result_pdr_upper_rate_value - # - result_pdr_upper_bandwidth_unit - # - result_pdr_upper_bandwidth_value - result_pdr_lower_rate_unit - result_pdr_lower_rate_value - # - result_pdr_lower_bandwidth_unit - # - result_pdr_lower_bandwidth_value - # - result_ndr_upper_rate_unit - # - result_ndr_upper_rate_value - # - result_ndr_upper_bandwidth_unit - # - result_ndr_upper_bandwidth_value - result_ndr_lower_rate_unit - result_ndr_lower_rate_value - # - result_ndr_lower_bandwidth_unit - # - result_ndr_lower_bandwidth_value - # - result_latency_reverse_pdr_90_avg - result_latency_reverse_pdr_90_hdrh - # - result_latency_reverse_pdr_90_max - # - result_latency_reverse_pdr_90_min - # - result_latency_reverse_pdr_90_unit - # - result_latency_reverse_pdr_50_avg - result_latency_reverse_pdr_50_hdrh - # - result_latency_reverse_pdr_50_max - # - result_latency_reverse_pdr_50_min - # - result_latency_reverse_pdr_50_unit - # - result_latency_reverse_pdr_10_avg - result_latency_reverse_pdr_10_hdrh - # - result_latency_reverse_pdr_10_max - # - result_latency_reverse_pdr_10_min - # - result_latency_reverse_pdr_10_unit - # - result_latency_reverse_pdr_0_avg - result_latency_reverse_pdr_0_hdrh - # - result_latency_reverse_pdr_0_max - # - result_latency_reverse_pdr_0_min - # - result_latency_reverse_pdr_0_unit - # - result_latency_forward_pdr_90_avg - result_latency_forward_pdr_90_hdrh - # - result_latency_forward_pdr_90_max - # - result_latency_forward_pdr_90_min - # - result_latency_forward_pdr_90_unit - result_latency_forward_pdr_50_avg - result_latency_forward_pdr_50_hdrh - # - result_latency_forward_pdr_50_max - # - result_latency_forward_pdr_50_min - result_latency_forward_pdr_50_unit - # - result_latency_forward_pdr_10_avg - result_latency_forward_pdr_10_hdrh - # - result_latency_forward_pdr_10_max - # - result_latency_forward_pdr_10_min - # - result_latency_forward_pdr_10_unit - # - result_latency_forward_pdr_0_avg - result_latency_forward_pdr_0_hdrh - # - result_latency_forward_pdr_0_max - # - result_latency_forward_pdr_0_min - # - result_latency_forward_pdr_0_unit iterative-mrr: path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_{release} columns: @@ -114,8 +67,6 @@ iterative-mrr: - start_time - passed - test_id - # - test_name_long - # - test_name_short - version - result_receive_rate_rate_avg - result_receive_rate_rate_stdev @@ -132,66 +83,14 @@ iterative-ndrpdr: - start_time - passed - test_id - # - test_name_long - # - test_name_short - version - # - result_pdr_upper_rate_unit - # - result_pdr_upper_rate_value - # - result_pdr_upper_bandwidth_unit - # - result_pdr_upper_bandwidth_value - result_pdr_lower_rate_unit - result_pdr_lower_rate_value - # - result_pdr_lower_bandwidth_unit - # - result_pdr_lower_bandwidth_value - # - result_ndr_upper_rate_unit - # - result_ndr_upper_rate_value - # - result_ndr_upper_bandwidth_unit - # - result_ndr_upper_bandwidth_value - result_ndr_lower_rate_unit - result_ndr_lower_rate_value - # - result_ndr_lower_bandwidth_unit - # - result_ndr_lower_bandwidth_value - # - result_latency_reverse_pdr_90_avg - ## - result_latency_reverse_pdr_90_hdrh - # - result_latency_reverse_pdr_90_max - # - result_latency_reverse_pdr_90_min - # - result_latency_reverse_pdr_90_unit - # - result_latency_reverse_pdr_50_avg - ## - result_latency_reverse_pdr_50_hdrh - # - result_latency_reverse_pdr_50_max - # - result_latency_reverse_pdr_50_min - # - result_latency_reverse_pdr_50_unit - # - result_latency_reverse_pdr_10_avg - ## - result_latency_reverse_pdr_10_hdrh - # - result_latency_reverse_pdr_10_max - # - result_latency_reverse_pdr_10_min - # - result_latency_reverse_pdr_10_unit - # - result_latency_reverse_pdr_0_avg - ## - result_latency_reverse_pdr_0_hdrh - # - result_latency_reverse_pdr_0_max - # - result_latency_reverse_pdr_0_min - # - result_latency_reverse_pdr_0_unit - # - result_latency_forward_pdr_90_avg - ## - result_latency_forward_pdr_90_hdrh - # - result_latency_forward_pdr_90_max - # - result_latency_forward_pdr_90_min - # - result_latency_forward_pdr_90_unit - result_latency_forward_pdr_50_avg - ## - result_latency_forward_pdr_50_hdrh - # - result_latency_forward_pdr_50_max - # - result_latency_forward_pdr_50_min - result_latency_forward_pdr_50_unit - # - result_latency_forward_pdr_10_avg - ## - result_latency_forward_pdr_10_hdrh - # - result_latency_forward_pdr_10_max - # - result_latency_forward_pdr_10_min - # - result_latency_forward_pdr_10_unit - # - result_latency_forward_pdr_0_avg - ## - result_latency_forward_pdr_0_hdrh - # - result_latency_forward_pdr_0_max - # - result_latency_forward_pdr_0_min - # - result_latency_forward_pdr_0_unit # coverage-ndrpdr: # path: str # columns: -# - list
\ No newline at end of file +# - list diff --git a/resources/tools/dash/app/pal/data/url_processing.py b/resources/tools/dash/app/pal/data/url_processing.py index 22cd034dfd..9307015d0d 100644 --- a/resources/tools/dash/app/pal/data/url_processing.py +++ b/resources/tools/dash/app/pal/data/url_processing.py @@ -24,8 +24,20 @@ from binascii import Error as BinasciiErr def url_encode(params: dict) -> str: + """Encode the URL parameters and zip them and create the whole URL using + given data. + + :param params: All data necessary to create the URL: + - scheme, + - network location, + - path, + - query, + - parameters. + :type params: dict + :returns: Encoded URL. + :rtype: str """ - """ + url_params = params.get("params", None) if url_params: encoded_params = urlsafe_b64encode( @@ -45,8 +57,14 @@ def url_encode(params: dict) -> str: def url_decode(url: str) -> dict: + """Parse the given URL and decode the parameters. + + :param url: URL to be parsed and decoded. + :type url: str + :returns: Paresed URL. + :rtype: dict """ - """ + try: parsed_url = urlparse(url) except ValueError as err: diff --git a/resources/tools/dash/app/pal/news/layout.py b/resources/tools/dash/app/pal/news/layout.py index c34575b75a..b8edb7a683 100644 --- a/resources/tools/dash/app/pal/news/layout.py +++ b/resources/tools/dash/app/pal/news/layout.py @@ -31,20 +31,32 @@ from .tables import table_failed class Layout: - """ + """The layout of the dash app and the callbacks. """ + # The default job displayed when the page is loaded first time. DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx" - URL_STYLE = { - "background-color": "#d2ebf5", - "border-color": "#bce1f1", - "color": "#135d7c" - } - - def __init__(self, app: Flask, html_layout_file: str, - data_spec_file: str, tooltip_file: str) -> None: - """ + def __init__(self, app: Flask, html_layout_file: str, data_spec_file: str, + tooltip_file: str) -> None: + """Initialization: + - save the input parameters, + - read and pre-process the data, + - prepare data fro the control panel, + - read HTML layout file, + - read tooltips from the tooltip file. + + :param app: Flask application running the dash application. + :param html_layout_file: Path and name of the file specifying the HTML + layout of the dash application. + :param data_spec_file: Path and name of the file specifying the data to + be read from parquets for this application. + :param tooltip_file: Path and name of the yaml file specifying the + tooltips. + :type app: Flask + :type html_layout_file: str + :type data_spec_file: str + :type tooltip_file: str """ # Inputs @@ -61,6 +73,7 @@ class Layout: df_tst_info = pd.concat([data_mrr, data_ndrpdr], ignore_index=True) + # Prepare information for the control panel: jobs = sorted(list(df_tst_info["job"].unique())) job_info = { "job": list(), @@ -80,6 +93,7 @@ class Layout: self._default = self._set_job_params(self.DEFAULT_JOB) + # Pre-process the data: tst_info = { "job": list(), "build": list(), @@ -118,7 +132,7 @@ class Layout: self._data = pd.DataFrame.from_dict(tst_info) # Read from files: - self._html_layout = "" + self._html_layout = str() self._tooltips = dict() try: @@ -157,23 +171,41 @@ class Layout: return self._data @property - def default(self) -> any: + def default(self) -> dict: return self._default def _get_duts(self) -> list: - """ + """Get the list of DUTs from the pre-processed information about jobs. + + :returns: Alphabeticaly sorted list of DUTs. + :rtype: list """ return sorted(list(self.df_job_info["dut"].unique())) def _get_ttypes(self, dut: str) -> list: - """ + """Get the list of test types from the pre-processed information about + jobs. + + :param dut: The DUT for which the list of test types will be populated. + :type dut: str + :returns: Alphabeticaly sorted list of test types. + :rtype: list """ return sorted(list(self.df_job_info.loc[( self.df_job_info["dut"] == dut )]["ttype"].unique())) def _get_cadences(self, dut: str, ttype: str) -> list: - """ + """Get the list of cadences from the pre-processed information about + jobs. + + :param dut: The DUT for which the list of cadences will be populated. + :param ttype: The test type for which the list of cadences will be + populated. + :type dut: str + :type ttype: str + :returns: Alphabeticaly sorted list of cadences. + :rtype: list """ return sorted(list(self.df_job_info.loc[( (self.df_job_info["dut"] == dut) & @@ -181,7 +213,19 @@ class Layout: )]["cadence"].unique())) def _get_test_beds(self, dut: str, ttype: str, cadence: str) -> list: - """ + """Get the list of test beds from the pre-processed information about + jobs. + + :param dut: The DUT for which the list of test beds will be populated. + :param ttype: The test type for which the list of test beds will be + populated. + :param cadence: The cadence for which the list of test beds will be + populated. + :type dut: str + :type ttype: str + :type cadence: str + :returns: Alphabeticaly sorted list of test beds. + :rtype: list """ return sorted(list(self.df_job_info.loc[( (self.df_job_info["dut"] == dut) & @@ -190,9 +234,19 @@ class Layout: )]["tbed"].unique())) def _get_job(self, dut, ttype, cadence, testbed): - """Get the name of a job defined by dut, ttype, cadence, testbed. - - Input information comes from control panel. + """Get the name of a job defined by dut, ttype, cadence, test bed. + Input information comes from the control panel. + + :param dut: The DUT for which the job name will be created. + :param ttype: The test type for which the job name will be created. + :param cadence: The cadence for which the job name will be created. + :param testbed: The test bed for which the job name will be created. + :type dut: str + :type ttype: str + :type cadence: str + :type testbed: str + :returns: Job name. + :rtype: str """ return self.df_job_info.loc[( (self.df_job_info["dut"] == dut) & @@ -201,9 +255,30 @@ class Layout: (self.df_job_info["tbed"] == testbed) )]["job"].item() - def _set_job_params(self, job: str) -> dict: + @staticmethod + def _generate_options(opts: list) -> list: + """Return list of options for radio items in control panel. The items in + the list are dictionaries with keys "label" and "value". + + :params opts: List of options (str) to be used for the generated list. + :type opts: list + :returns: List of options (dict). + :rtype: list """ + return [{"label": i, "value": i} for i in opts] + + def _set_job_params(self, job: str) -> dict: + """Create a dictionary with all options and values for (and from) the + given job. + + :params job: The name of job for and from which the dictionary will be + created. + :type job: str + :returns: Dictionary with all options and values for (and from) the + given job. + :rtype: dict """ + lst_job = job.split("-") return { "job": job, @@ -221,8 +296,22 @@ class Layout: def _show_tooltip(self, id: str, title: str, clipboard_id: str=None) -> list: + """Generate list of elements to display a text (e.g. a title) with a + tooltip and optionaly with Copy&Paste icon and the clipboard + functionality enabled. + + :param id: Tooltip ID. + :param title: A text for which the tooltip will be displayed. + :param clipboard_id: If defined, a Copy&Paste icon is displayed and the + clipboard functionality is enabled. + :type id: str + :type title: str + :type clipboard_id: str + :returns: List of elements to display a text with a tooltip and + optionaly with Copy&Paste icon. + :rtype: list """ - """ + return [ dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \ if clipboard_id else str(), @@ -243,8 +332,19 @@ class Layout: ] def add_content(self): + """Top level method which generated the web page. + + It generates: + - Store for user input data, + - Navigation bar, + - Main area with control panel and ploting area. + + If no HTML layout is provided, an error message is displayed instead. + + :returns: The HTML div with teh whole page. + :rtype: html.Div """ - """ + if self.html_layout: return html.Div( id="div-main", @@ -282,7 +382,11 @@ class Layout: def _add_navbar(self): """Add nav element with navigation panel. It is placed on the top. + + :returns: Navigation bar. + :rtype: dbc.NavbarSimple """ + return dbc.NavbarSimple( id="navbarsimple-main", children=[ @@ -303,8 +407,12 @@ class Layout: ) def _add_ctrl_col(self) -> dbc.Col: - """Add column with controls. It is placed on the left side. + """Add column with control panel. It is placed on the left side. + + :returns: Column with the control panel. + :rtype: dbc.col """ + return dbc.Col( id="col-controls", children=[ @@ -313,8 +421,12 @@ class Layout: ) def _add_plotting_col(self) -> dbc.Col: - """Add column with plots and tables. It is placed on the right side. + """Add column with tables. It is placed on the right side. + + :returns: Column with tables. + :rtype: dbc.col """ + return dbc.Col( id="col-plotting-area", children=[ @@ -328,7 +440,10 @@ class Layout: ) def _add_ctrl_panel(self) -> dbc.Row: - """ + """Add control panel. + + :returns: Control panel. + :rtype: dbc.Row """ return dbc.Row( id="row-ctrl-panel", @@ -419,7 +534,13 @@ class Layout: ) class ControlPanel: + """ + """ + def __init__(self, panel: dict, default: dict) -> None: + """ + """ + self._defaults = { "ri-ttypes-options": default["ttypes"], "ri-cadences-options": default["cadences"], @@ -456,10 +577,6 @@ class Layout: def values(self) -> list: return list(self._panel.values()) - @staticmethod - def _generate_options(opts: list) -> list: - return [{"label": i, "value": i} for i in opts] - def callbacks(self, app): @app.callback( diff --git a/resources/tools/dash/app/pal/routes.py b/resources/tools/dash/app/pal/routes.py index 63bbb304f0..d4cd88ffce 100644 --- a/resources/tools/dash/app/pal/routes.py +++ b/resources/tools/dash/app/pal/routes.py @@ -18,13 +18,13 @@ from flask import current_app as app from flask import render_template -@app.route(u"/") +@app.route("/") def home(): """Landing page. """ return render_template( - u"index_layout.jinja2", - title=u"FD.io CSIT", - description=u"Performance Dashboard", - template=u"d-flex h-100 text-center text-white bg-dark" + "index_layout.jinja2", + title="FD.io CSIT", + description="Performance Dashboard", + template="d-flex h-100 text-center text-white bg-dark" ) |