feat(uti): Add iterative data

Change-Id: Iaa7253b377f019235289f6bbf48eafd850a2dfc8
Signed-off-by: Tibor Frank <tifrank@cisco.com>

3 files changed, 66 insertions, 62 deletions

diff --git a/resources/tools/dash/app/pal/data/data.py b/resources/tools/dash/app/pal/data/data.py
index 3d9b8b1664..efe2a2d1b6 100644
--- a/resources/tools/dash/app/pal/data/data.py
+++ b/resources/tools/dash/app/pal/data/data.py
@@ -13,14 +13,16 @@
 
 """Prepare data for Plotly Dash."""
 
-from datetime import datetime, timedelta
 import logging
+
+from yaml import load, FullLoader, YAMLError
+from datetime import datetime, timedelta
 from time import time
+from pytz import UTC
+from pandas import DataFrame
 
 import awswrangler as wr
-from pytz import UTC
 
-from yaml import load, FullLoader, YAMLError
 from awswrangler.exceptions import EmptyDataFrame, NoFilesFound
 
 
@@ -28,7 +30,7 @@ class Data:
     """
     """
 
-    def __init__(self, data_spec_file, debug=False):
+    def __init__(self, data_spec_file: str, debug: bool=False) -> None:
         """
         """
 
@@ -61,7 +63,7 @@ class Data:
     def data(self):
         return self._data
 
-    def _get_columns(self, parquet):
+    def _get_columns(self, parquet: str) -> list:
         try:
             return self._data_spec[parquet]["columns"]
         except KeyError as err:
@@ -71,7 +73,7 @@ class Data:
                 f"specified.\n{err}"
             )
 
-    def _get_path(self, parquet):
+    def _get_path(self, parquet: str) -> str:
         try:
             return self._data_spec[parquet]["path"]
         except KeyError as err:
@@ -84,7 +86,7 @@ class Data:
     def _create_dataframe_from_parquet(self,
         path, partition_filter=None, columns=None,
         validate_schema=False, last_modified_begin=None,
-        last_modified_end=None, days=None):
+        last_modified_end=None, days=None) -> DataFrame:
         """Read parquet stored in S3 compatible storage and returns Pandas
         Dataframe.
 
@@ -148,7 +150,7 @@ class Data:
         self._data = df
         return df
 
-    def read_stats(self, days=None):
+    def read_stats(self, days: int=None) -> tuple:
         """Read Suite Result Analysis data partition from parquet.
         """
         l_stats = lambda part: True if part["stats_type"] == "sra" else False
@@ -176,7 +178,7 @@ class Data:
             )
         )
 
-    def read_trending_mrr(self, days=None):
+    def read_trending_mrr(self, days: int=None) -> DataFrame:
         """Read MRR data partition from parquet.
         """
         lambda_f = lambda part: True if part["test_type"] == "mrr" else False
@@ -188,7 +190,7 @@ class Data:
             days=days
         )
 
-    def read_trending_ndrpdr(self, days=None):
+    def read_trending_ndrpdr(self, days: int=None) -> DataFrame:
         """Read NDRPDR data partition from iterative parquet.
         """
         lambda_f = lambda part: True if part["test_type"] == "ndrpdr" else False
@@ -200,26 +202,24 @@ class Data:
             days=days
         )
 
-    def read_iterative_mrr(self, days=None):
+    def read_iterative_mrr(self, release: str) -> DataFrame:
         """Read MRR data partition from iterative parquet.
         """
         lambda_f = lambda part: True if part["test_type"] == "mrr" else False
 
         return self._create_dataframe_from_parquet(
-            path=self._get_path("iterative-mrr"),
+            path=self._get_path("iterative-mrr").format(release=release),
             partition_filter=lambda_f,
-            columns=self._get_columns("iterative-mrr"),
-            days=days
+            columns=self._get_columns("iterative-mrr")
         )
 
-    def read_iterative_ndrpdr(self, days=None):
+    def read_iterative_ndrpdr(self, release: str) -> DataFrame:
         """Read NDRPDR data partition from parquet.
         """
         lambda_f = lambda part: True if part["test_type"] == "ndrpdr" else False
 
         return self._create_dataframe_from_parquet(
-            path=self._get_path("iterative-ndrpdr"),
+            path=self._get_path("iterative-ndrpdr").format(release=release),
             partition_filter=lambda_f,
-            columns=self._get_columns("iterative-ndrpdr"),
-            days=days
+            columns=self._get_columns("iterative-ndrpdr")
         )
diff --git a/resources/tools/dash/app/pal/data/data.yaml b/resources/tools/dash/app/pal/data/data.yaml
index 92cd659f48..15fad711ba 100644
--- a/resources/tools/dash/app/pal/data/data.yaml
+++ b/resources/tools/dash/app/pal/data/data.yaml
@@ -104,7 +104,7 @@ trending-ndrpdr:
     # - result_latency_forward_pdr_0_min
     # - result_latency_forward_pdr_0_unit
 iterative-mrr:
-  path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2202
+  path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_{release}
   columns:
     - job
     - build
@@ -122,7 +122,7 @@ iterative-mrr:
     - result_receive_rate_rate_unit
     - result_receive_rate_rate_values
 iterative-ndrpdr:
-  path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2202
+  path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_{release}
   columns:
     - job
     - build
@@ -135,62 +135,62 @@ iterative-ndrpdr:
     - test_name_long
     - test_name_short
     - version
-    - result_pdr_upper_rate_unit
-    - result_pdr_upper_rate_value
-    - result_pdr_upper_bandwidth_unit
-    - result_pdr_upper_bandwidth_value
+    # - result_pdr_upper_rate_unit
+    # - result_pdr_upper_rate_value
+    # - result_pdr_upper_bandwidth_unit
+    # - result_pdr_upper_bandwidth_value
     - result_pdr_lower_rate_unit
     - result_pdr_lower_rate_value
-    - result_pdr_lower_bandwidth_unit
-    - result_pdr_lower_bandwidth_value
-    - result_ndr_upper_rate_unit
-    - result_ndr_upper_rate_value
-    - result_ndr_upper_bandwidth_unit
-    - result_ndr_upper_bandwidth_value
+    # - result_pdr_lower_bandwidth_unit
+    # - result_pdr_lower_bandwidth_value
+    # - result_ndr_upper_rate_unit
+    # - result_ndr_upper_rate_value
+    # - result_ndr_upper_bandwidth_unit
+    # - result_ndr_upper_bandwidth_value
     - result_ndr_lower_rate_unit
     - result_ndr_lower_rate_value
-    - result_ndr_lower_bandwidth_unit
-    - result_ndr_lower_bandwidth_value
-    - result_latency_reverse_pdr_90_avg
+    # - result_ndr_lower_bandwidth_unit
+    # - result_ndr_lower_bandwidth_value
+    # - result_latency_reverse_pdr_90_avg
     - result_latency_reverse_pdr_90_hdrh
-    - result_latency_reverse_pdr_90_max
-    - result_latency_reverse_pdr_90_min
-    - result_latency_reverse_pdr_90_unit
-    - result_latency_reverse_pdr_50_avg
+    # - result_latency_reverse_pdr_90_max
+    # - result_latency_reverse_pdr_90_min
+    # - result_latency_reverse_pdr_90_unit
+    # - result_latency_reverse_pdr_50_avg
     - result_latency_reverse_pdr_50_hdrh
-    - result_latency_reverse_pdr_50_max
-    - result_latency_reverse_pdr_50_min
-    - result_latency_reverse_pdr_50_unit
-    - result_latency_reverse_pdr_10_avg
+    # - result_latency_reverse_pdr_50_max
+    # - result_latency_reverse_pdr_50_min
+    # - result_latency_reverse_pdr_50_unit
+    # - result_latency_reverse_pdr_10_avg
     - result_latency_reverse_pdr_10_hdrh
-    - result_latency_reverse_pdr_10_max
-    - result_latency_reverse_pdr_10_min
-    - result_latency_reverse_pdr_10_unit
-    - result_latency_reverse_pdr_0_avg
+    # - result_latency_reverse_pdr_10_max
+    # - result_latency_reverse_pdr_10_min
+    # - result_latency_reverse_pdr_10_unit
+    # - result_latency_reverse_pdr_0_avg
     - result_latency_reverse_pdr_0_hdrh
-    - result_latency_reverse_pdr_0_max
-    - result_latency_reverse_pdr_0_min
-    - result_latency_reverse_pdr_0_unit
-    - result_latency_forward_pdr_90_avg
+    # - result_latency_reverse_pdr_0_max
+    # - result_latency_reverse_pdr_0_min
+    # - result_latency_reverse_pdr_0_unit
+    # - result_latency_forward_pdr_90_avg
     - result_latency_forward_pdr_90_hdrh
-    - result_latency_forward_pdr_90_max
-    - result_latency_forward_pdr_90_min
-    - result_latency_forward_pdr_90_unit
+    # - result_latency_forward_pdr_90_max
+    # - result_latency_forward_pdr_90_min
+    # - result_latency_forward_pdr_90_unit
     - result_latency_forward_pdr_50_avg
     - result_latency_forward_pdr_50_hdrh
-    - result_latency_forward_pdr_50_max
-    - result_latency_forward_pdr_50_min
+    # - result_latency_forward_pdr_50_max
+    # - result_latency_forward_pdr_50_min
     - result_latency_forward_pdr_50_unit
-    - result_latency_forward_pdr_10_avg
+    # - result_latency_forward_pdr_10_avg
     - result_latency_forward_pdr_10_hdrh
-    - result_latency_forward_pdr_10_max
-    - result_latency_forward_pdr_10_min
-    - result_latency_forward_pdr_10_unit
-    - result_latency_forward_pdr_0_avg
+    # - result_latency_forward_pdr_10_max
+    # - result_latency_forward_pdr_10_min
+    # - result_latency_forward_pdr_10_unit
+    # - result_latency_forward_pdr_0_avg
     - result_latency_forward_pdr_0_hdrh
-    - result_latency_forward_pdr_0_max
-    - result_latency_forward_pdr_0_min
-    - result_latency_forward_pdr_0_unit
+    # - result_latency_forward_pdr_0_max
+    # - result_latency_forward_pdr_0_min
+    # - result_latency_forward_pdr_0_unit
 # coverage-ndrpdr:
 #   path: str
 #   columns:
diff --git a/resources/tools/dash/app/pal/data/tooltips.yaml b/resources/tools/dash/app/pal/data/tooltips.yaml
index e5259b4d7b..5a830e4b68 100644
--- a/resources/tools/dash/app/pal/data/tooltips.yaml
+++ b/resources/tools/dash/app/pal/data/tooltips.yaml
@@ -10,6 +10,8 @@ help-dut:
   Device Under Test (DUT) - In software networking, “device” denotes a specific
   piece of software tasked with packet processing. Such device is surrounded
   with other software components (such as operating system kernel).
+help-dut-ver:
+  The version of the Device under Test.
 help-framesize:
   Frame size - size of an Ethernet Layer-2 frame on the wire, including any VLAN
   tags (dot1q, dot1ad) and Ethernet FCS, but excluding Ethernet preamble and
@@ -17,6 +19,8 @@ help-framesize:
 help-infra:
   Infrastructure is defined by the toplology (number of nodes), processor
   architecture, NIC and driver.
+help-release:
+  The CSIT release.
 help-tbed:
   The test bed is defined by toplology (number of nodes) and processor
   architecture.