diff options
Diffstat (limited to 'resources/tools/dash/app/pal/data')
| -rw-r--r-- | resources/tools/dash/app/pal/data/data.py | 36 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/data/data.yaml | 88 | ||||
| -rw-r--r-- | resources/tools/dash/app/pal/data/tooltips.yaml | 4 |
3 files changed, 66 insertions, 62 deletions
diff --git a/resources/tools/dash/app/pal/data/data.py b/resources/tools/dash/app/pal/data/data.py index 3d9b8b1664..efe2a2d1b6 100644 --- a/resources/tools/dash/app/pal/data/data.py +++ b/resources/tools/dash/app/pal/data/data.py @@ -13,14 +13,16 @@ """Prepare data for Plotly Dash.""" -from datetime import datetime, timedelta import logging + +from yaml import load, FullLoader, YAMLError +from datetime import datetime, timedelta from time import time +from pytz import UTC +from pandas import DataFrame import awswrangler as wr -from pytz import UTC -from yaml import load, FullLoader, YAMLError from awswrangler.exceptions import EmptyDataFrame, NoFilesFound @@ -28,7 +30,7 @@ class Data: """ """ - def __init__(self, data_spec_file, debug=False): + def __init__(self, data_spec_file: str, debug: bool=False) -> None: """ """ @@ -61,7 +63,7 @@ class Data: def data(self): return self._data - def _get_columns(self, parquet): + def _get_columns(self, parquet: str) -> list: try: return self._data_spec[parquet]["columns"] except KeyError as err: @@ -71,7 +73,7 @@ class Data: f"specified.\n{err}" ) - def _get_path(self, parquet): + def _get_path(self, parquet: str) -> str: try: return self._data_spec[parquet]["path"] except KeyError as err: @@ -84,7 +86,7 @@ class Data: def _create_dataframe_from_parquet(self, path, partition_filter=None, columns=None, validate_schema=False, last_modified_begin=None, - last_modified_end=None, days=None): + last_modified_end=None, days=None) -> DataFrame: """Read parquet stored in S3 compatible storage and returns Pandas Dataframe. @@ -148,7 +150,7 @@ class Data: self._data = df return df - def read_stats(self, days=None): + def read_stats(self, days: int=None) -> tuple: """Read Suite Result Analysis data partition from parquet. """ l_stats = lambda part: True if part["stats_type"] == "sra" else False @@ -176,7 +178,7 @@ class Data: ) ) - def read_trending_mrr(self, days=None): + def read_trending_mrr(self, days: int=None) -> DataFrame: """Read MRR data partition from parquet. """ lambda_f = lambda part: True if part["test_type"] == "mrr" else False @@ -188,7 +190,7 @@ class Data: days=days ) - def read_trending_ndrpdr(self, days=None): + def read_trending_ndrpdr(self, days: int=None) -> DataFrame: """Read NDRPDR data partition from iterative parquet. """ lambda_f = lambda part: True if part["test_type"] == "ndrpdr" else False @@ -200,26 +202,24 @@ class Data: days=days ) - def read_iterative_mrr(self, days=None): + def read_iterative_mrr(self, release: str) -> DataFrame: """Read MRR data partition from iterative parquet. """ lambda_f = lambda part: True if part["test_type"] == "mrr" else False return self._create_dataframe_from_parquet( - path=self._get_path("iterative-mrr"), + path=self._get_path("iterative-mrr").format(release=release), partition_filter=lambda_f, - columns=self._get_columns("iterative-mrr"), - days=days + columns=self._get_columns("iterative-mrr") ) - def read_iterative_ndrpdr(self, days=None): + def read_iterative_ndrpdr(self, release: str) -> DataFrame: """Read NDRPDR data partition from parquet. """ lambda_f = lambda part: True if part["test_type"] == "ndrpdr" else False return self._create_dataframe_from_parquet( - path=self._get_path("iterative-ndrpdr"), + path=self._get_path("iterative-ndrpdr").format(release=release), partition_filter=lambda_f, - columns=self._get_columns("iterative-ndrpdr"), - days=days + columns=self._get_columns("iterative-ndrpdr") ) diff --git a/resources/tools/dash/app/pal/data/data.yaml b/resources/tools/dash/app/pal/data/data.yaml index 92cd659f48..15fad711ba 100644 --- a/resources/tools/dash/app/pal/data/data.yaml +++ b/resources/tools/dash/app/pal/data/data.yaml @@ -104,7 +104,7 @@ trending-ndrpdr: # - result_latency_forward_pdr_0_min # - result_latency_forward_pdr_0_unit iterative-mrr: - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2202 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_{release} columns: - job - build @@ -122,7 +122,7 @@ iterative-mrr: - result_receive_rate_rate_unit - result_receive_rate_rate_values iterative-ndrpdr: - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2202 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_{release} columns: - job - build @@ -135,62 +135,62 @@ iterative-ndrpdr: - test_name_long - test_name_short - version - - result_pdr_upper_rate_unit - - result_pdr_upper_rate_value - - result_pdr_upper_bandwidth_unit - - result_pdr_upper_bandwidth_value + # - result_pdr_upper_rate_unit + # - result_pdr_upper_rate_value + # - result_pdr_upper_bandwidth_unit + # - result_pdr_upper_bandwidth_value - result_pdr_lower_rate_unit - result_pdr_lower_rate_value - - result_pdr_lower_bandwidth_unit - - result_pdr_lower_bandwidth_value - - result_ndr_upper_rate_unit - - result_ndr_upper_rate_value - - result_ndr_upper_bandwidth_unit - - result_ndr_upper_bandwidth_value + # - result_pdr_lower_bandwidth_unit + # - result_pdr_lower_bandwidth_value + # - result_ndr_upper_rate_unit + # - result_ndr_upper_rate_value + # - result_ndr_upper_bandwidth_unit + # - result_ndr_upper_bandwidth_value - result_ndr_lower_rate_unit - result_ndr_lower_rate_value - - result_ndr_lower_bandwidth_unit - - result_ndr_lower_bandwidth_value - - result_latency_reverse_pdr_90_avg + # - result_ndr_lower_bandwidth_unit + # - result_ndr_lower_bandwidth_value + # - result_latency_reverse_pdr_90_avg - result_latency_reverse_pdr_90_hdrh - - result_latency_reverse_pdr_90_max - - result_latency_reverse_pdr_90_min - - result_latency_reverse_pdr_90_unit - - result_latency_reverse_pdr_50_avg + # - result_latency_reverse_pdr_90_max + # - result_latency_reverse_pdr_90_min + # - result_latency_reverse_pdr_90_unit + # - result_latency_reverse_pdr_50_avg - result_latency_reverse_pdr_50_hdrh - - result_latency_reverse_pdr_50_max - - result_latency_reverse_pdr_50_min - - result_latency_reverse_pdr_50_unit - - result_latency_reverse_pdr_10_avg + # - result_latency_reverse_pdr_50_max + # - result_latency_reverse_pdr_50_min + # - result_latency_reverse_pdr_50_unit + # - result_latency_reverse_pdr_10_avg - result_latency_reverse_pdr_10_hdrh - - result_latency_reverse_pdr_10_max - - result_latency_reverse_pdr_10_min - - result_latency_reverse_pdr_10_unit - - result_latency_reverse_pdr_0_avg + # - result_latency_reverse_pdr_10_max + # - result_latency_reverse_pdr_10_min + # - result_latency_reverse_pdr_10_unit + # - result_latency_reverse_pdr_0_avg - result_latency_reverse_pdr_0_hdrh - - result_latency_reverse_pdr_0_max - - result_latency_reverse_pdr_0_min - - result_latency_reverse_pdr_0_unit - - result_latency_forward_pdr_90_avg + # - result_latency_reverse_pdr_0_max + # - result_latency_reverse_pdr_0_min + # - result_latency_reverse_pdr_0_unit + # - result_latency_forward_pdr_90_avg - result_latency_forward_pdr_90_hdrh - - result_latency_forward_pdr_90_max - - result_latency_forward_pdr_90_min - - result_latency_forward_pdr_90_unit + # - result_latency_forward_pdr_90_max + # - result_latency_forward_pdr_90_min + # - result_latency_forward_pdr_90_unit - result_latency_forward_pdr_50_avg - result_latency_forward_pdr_50_hdrh - - result_latency_forward_pdr_50_max - - result_latency_forward_pdr_50_min + # - result_latency_forward_pdr_50_max + # - result_latency_forward_pdr_50_min - result_latency_forward_pdr_50_unit - - result_latency_forward_pdr_10_avg + # - result_latency_forward_pdr_10_avg - result_latency_forward_pdr_10_hdrh - - result_latency_forward_pdr_10_max - - result_latency_forward_pdr_10_min - - result_latency_forward_pdr_10_unit - - result_latency_forward_pdr_0_avg + # - result_latency_forward_pdr_10_max + # - result_latency_forward_pdr_10_min + # - result_latency_forward_pdr_10_unit + # - result_latency_forward_pdr_0_avg - result_latency_forward_pdr_0_hdrh - - result_latency_forward_pdr_0_max - - result_latency_forward_pdr_0_min - - result_latency_forward_pdr_0_unit + # - result_latency_forward_pdr_0_max + # - result_latency_forward_pdr_0_min + # - result_latency_forward_pdr_0_unit # coverage-ndrpdr: # path: str # columns: diff --git a/resources/tools/dash/app/pal/data/tooltips.yaml b/resources/tools/dash/app/pal/data/tooltips.yaml index e5259b4d7b..5a830e4b68 100644 --- a/resources/tools/dash/app/pal/data/tooltips.yaml +++ b/resources/tools/dash/app/pal/data/tooltips.yaml @@ -10,6 +10,8 @@ help-dut: Device Under Test (DUT) - In software networking, “device” denotes a specific piece of software tasked with packet processing. Such device is surrounded with other software components (such as operating system kernel). +help-dut-ver: + The version of the Device under Test. help-framesize: Frame size - size of an Ethernet Layer-2 frame on the wire, including any VLAN tags (dot1q, dot1ad) and Ethernet FCS, but excluding Ethernet preamble and @@ -17,6 +19,8 @@ help-framesize: help-infra: Infrastructure is defined by the toplology (number of nodes), processor architecture, NIC and driver. +help-release: + The CSIT release. help-tbed: The test bed is defined by toplology (number of nodes) and processor architecture. |