UTI: Add Download for iterative data.

Change-Id: I79bcfc2449310d4d186146ce79b2143a33ed2d1f
Signed-off-by: Tibor Frank <tifrank@cisco.com>

7 files changed, 222 insertions, 52 deletions

diff --git a/resources/tools/dash/app/pal/report/graphs.py b/resources/tools/dash/app/pal/report/graphs.py
index 4cd9287f0f..36f28d09e8 100644
--- a/resources/tools/dash/app/pal/report/graphs.py
+++ b/resources/tools/dash/app/pal/report/graphs.py
@@ -25,7 +25,14 @@ from ..utils.utils import get_color
 
 
 def get_short_version(version: str, dut_type: str="vpp") -> str:
-    """
+    """Returns the short version of DUT without build number.
+
+    :param version: Original version string.
+    :param dut_type: DUT type.
+    :type version: str
+    :type dut_type: str
+    :returns: Short verion string.
+    :rtype: str
     """
 
     if dut_type in ("trex", "dpdk"):
@@ -48,7 +55,15 @@ def get_short_version(version: str, dut_type: str="vpp") -> str:
 
 
 def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
-    """
+    """Select the data for graphs and tables from the provided data frame.
+
+    :param data: Data frame with data for graphs and tables.
+    :param itm: Item (in this case job name) which data will be selected from
+        the input data frame.
+    :type data: pandas.DataFrame
+    :type itm: str
+    :returns: A data frame with selected data.
+    :rtype: pandas.DataFrame
     """
 
     phy = itm["phy"].split("-")
@@ -96,7 +111,20 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 
 def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
         normalize: bool) -> tuple:
-    """
+    """Generate the statistical box graph with iterative data (MRR, NDR and PDR,
+    for PDR also Latencies).
+
+    :param data: Data frame with iterative data.
+    :param sel: Selected tests.
+    :param layout: Layout of plot.ly graph.
+    :param normalize: If True, the data is normalized to CPU frquency
+        Constants.NORM_FREQUENCY.
+    :param data: pandas.DataFrame
+    :param sel: dict
+    :param layout: dict
+    :param normalize: bool
+    :returns: Tuple of graphs - throughput and latency.
+    :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
 
     fig_tput = None
@@ -192,46 +220,56 @@ def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
 
 def table_comparison(data: pd.DataFrame, sel:dict,
         normalize: bool) -> pd.DataFrame:
-    """
+    """Generate the comparison table with selected tests.
+
+    :param data: Data frame with iterative data.
+    :param sel: Selected tests.
+    :param normalize: If True, the data is normalized to CPU frquency
+        Constants.NORM_FREQUENCY.
+    :param data: pandas.DataFrame
+    :param sel: dict
+    :param normalize: bool
+    :returns: Comparison table.
+    :rtype: pandas.DataFrame
     """
     table = pd.DataFrame(
-        {
-            "Test Case": [
-                "64b-2t1c-avf-eth-l2xcbase-eth-2memif-1dcr",
-                "64b-2t1c-avf-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc",
-                "64b-2t1c-avf-ethip4udp-ip4base-iacl50sl-10kflows",
-                "78b-2t1c-avf-ethip6-ip6scale2m-rnd "],
-            "2106.0-8": [
-                "14.45 +- 0.08",
-                "9.63 +- 0.05",
-                "9.7 +- 0.02",
-                "8.95 +- 0.06"],
-            "2110.0-8": [
-                "14.45 +- 0.08",
-                "9.63 +- 0.05",
-                "9.7 +- 0.02",
-                "8.95 +- 0.06"],
-            "2110.0-9": [
-                "14.45 +- 0.08",
-                "9.63 +- 0.05",
-                "9.7 +- 0.02",
-                "8.95 +- 0.06"],
-            "2202.0-9": [
-                "14.45 +- 0.08",
-                "9.63 +- 0.05",
-                "9.7 +- 0.02",
-                "8.95 +- 0.06"],
-            "2110.0-9 vs 2110.0-8": [
-                "-0.23 +-  0.62",
-                "-1.37 +-   1.3",
-                "+0.08 +-   0.2",
-                "-2.16 +-  0.83"],
-            "2202.0-9 vs 2110.0-9": [
-                "+6.95 +-  0.72",
-                "+5.35 +-  1.26",
-                "+4.48 +-  1.48",
-                "+4.09 +-  0.95"]
-        }
+        # {
+        #     "Test Case": [
+        #         "64b-2t1c-avf-eth-l2xcbase-eth-2memif-1dcr",
+        #         "64b-2t1c-avf-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc",
+        #         "64b-2t1c-avf-ethip4udp-ip4base-iacl50sl-10kflows",
+        #         "78b-2t1c-avf-ethip6-ip6scale2m-rnd "],
+        #     "2106.0-8": [
+        #         "14.45 +- 0.08",
+        #         "9.63 +- 0.05",
+        #         "9.7 +- 0.02",
+        #         "8.95 +- 0.06"],
+        #     "2110.0-8": [
+        #         "14.45 +- 0.08",
+        #         "9.63 +- 0.05",
+        #         "9.7 +- 0.02",
+        #         "8.95 +- 0.06"],
+        #     "2110.0-9": [
+        #         "14.45 +- 0.08",
+        #         "9.63 +- 0.05",
+        #         "9.7 +- 0.02",
+        #         "8.95 +- 0.06"],
+        #     "2202.0-9": [
+        #         "14.45 +- 0.08",
+        #         "9.63 +- 0.05",
+        #         "9.7 +- 0.02",
+        #         "8.95 +- 0.06"],
+        #     "2110.0-9 vs 2110.0-8": [
+        #         "-0.23 +-  0.62",
+        #         "-1.37 +-   1.3",
+        #         "+0.08 +-   0.2",
+        #         "-2.16 +-  0.83"],
+        #     "2202.0-9 vs 2110.0-9": [
+        #         "+6.95 +-  0.72",
+        #         "+5.35 +-  1.26",
+        #         "+4.48 +-  1.48",
+        #         "+4.09 +-  0.95"]
+        # }
     )
 
-    return pd.DataFrame()  #table
+    return table
diff --git a/resources/tools/dash/app/pal/report/layout.py b/resources/tools/dash/app/pal/report/layout.py
index 0e8f32405c..978ab0de6c 100644
--- a/resources/tools/dash/app/pal/report/layout.py
+++ b/resources/tools/dash/app/pal/report/layout.py
@@ -23,6 +23,7 @@ from dash import dcc
 from dash import html
 from dash import callback_context, no_update, ALL
 from dash import Input, Output, State
+from dash.exceptions import PreventUpdate
 from yaml import load, FullLoader, YAMLError
 from copy import deepcopy
 from ast import literal_eval
@@ -32,7 +33,8 @@ from ..utils.utils import show_tooltip, label, sync_checklists, list_tests, \
     gen_new_url
 from ..utils.url_processing import url_decode
 from ..data.data import Data
-from .graphs import graph_iterative, table_comparison, get_short_version
+from .graphs import graph_iterative, table_comparison, get_short_version, \
+    select_iterative_data
 
 
 class Layout:
@@ -1408,3 +1410,37 @@ class Layout:
             ]
             ret_val.extend(ctrl_panel.values())
             return ret_val
+
+        @app.callback(
+            Output("download-data", "data"),
+            State("selected-tests", "data"),
+            Input("btn-download-data", "n_clicks"),
+            prevent_initial_call=True
+        )
+        def _download_data(store_sel, n_clicks):
+            """Download the data
+
+            :param store_sel: List of tests selected by user stored in the
+                browser.
+            :param n_clicks: Number of clicks on the button "Download".
+            :type store_sel: list
+            :type n_clicks: int
+            :returns: dict of data frame content (base64 encoded) and meta data
+                used by the Download component.
+            :rtype: dict
+            """
+
+            if not n_clicks:
+                raise PreventUpdate
+
+            if not store_sel:
+                raise PreventUpdate
+
+            df = pd.DataFrame()
+            for itm in store_sel:
+                sel_data = select_iterative_data(self.data, itm)
+                if sel_data is None:
+                    continue
+                df = pd.concat([df, sel_data], ignore_index=True)
+
+            return dcc.send_data_frame(df.to_csv, C.REPORT_DOWNLOAD_FILE_NAME)
diff --git a/resources/tools/dash/app/pal/stats/graphs.py b/resources/tools/dash/app/pal/stats/graphs.py
index c1f58f2405..42f23da5aa 100644
--- a/resources/tools/dash/app/pal/stats/graphs.py
+++ b/resources/tools/dash/app/pal/stats/graphs.py
@@ -21,7 +21,19 @@ from datetime import datetime, timedelta
 
 def select_data(data: pd.DataFrame, itm:str, start: datetime,
         end: datetime) -> pd.DataFrame:
-    """
+    """Select the data for graphs from the provided data frame.
+
+    :param data: Data frame with data for graphs.
+    :param itm: Item (in this case job name) which data will be selected from
+        the input data frame.
+    :param start: The date (and time) when the selected data starts.
+    :param end: The date (and time) when the selected data ends.
+    :type data: pandas.DataFrame
+    :type itm: str
+    :type start: datetime.datetime
+    :type end: datetime.datetime
+    :returns: A data frame with selected data.
+    :rtype: pandas.DataFrame
     """
 
     df = data.loc[
@@ -36,7 +48,24 @@ def select_data(data: pd.DataFrame, itm:str, start: datetime,
 def graph_statistics(df: pd.DataFrame, job:str, layout: dict,
         start: datetime=datetime.utcnow()-timedelta(days=180),
         end: datetime=datetime.utcnow()) -> tuple:
-    """
+    """Generate graphs:
+    1. Passed / failed tests,
+    2. Job durations
+    with additional information shown in hover.
+
+    :param df: Data frame with input data.
+    :param job: The name of job which data will be presented in the graphs.
+    :param layout: Layout of plot.ly graph.
+    :param start: The date (and time) when the selected data starts.
+    :param end: The date (and time) when the selected data ends.
+    :type df: pandas.DataFrame
+    :type job: str
+    :type layout: dict
+    :type start: datetime.datetime
+    :type end: datetime.datetime
+    :returns: Tuple with two generated graphs (pased/failed tests and job
+        duration).
+    :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
 
     data = select_data(df, job, start, end)
diff --git a/resources/tools/dash/app/pal/stats/layout.py b/resources/tools/dash/app/pal/stats/layout.py
index 8df6d53bac..1d271cb265 100644
--- a/resources/tools/dash/app/pal/stats/layout.py
+++ b/resources/tools/dash/app/pal/stats/layout.py
@@ -800,7 +800,8 @@ class Layout:
             data = select_data(self.data, job, get_date(start), get_date(end))
             data = data.drop(columns=["job", ])
 
-            return dcc.send_data_frame(data.T.to_csv, f"{job}-stats.csv")
+            return dcc.send_data_frame(
+                data.T.to_csv, f"{job}-{C.STATS_DOWNLOAD_FILE_NAME}")
 
         @app.callback(
             Output("row-metadata", "children"),
diff --git a/resources/tools/dash/app/pal/trending/graphs.py b/resources/tools/dash/app/pal/trending/graphs.py
index 4cd8285f72..06bea25466 100644
--- a/resources/tools/dash/app/pal/trending/graphs.py
+++ b/resources/tools/dash/app/pal/trending/graphs.py
@@ -27,7 +27,14 @@ from ..utils.utils import classify_anomalies, get_color
 
 
 def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
-    """
+    """Get the HDRH latencies from the test data.
+
+    :param row: A row fron the data frame with test data.
+    :param name: The test name to be displayed as the graph title.
+    :type row: pandas.Series
+    :type name: str
+    :returns: Dictionary with HDRH latencies.
+    :rtype: dict
     """
 
     latencies = {"name": name}
@@ -41,7 +48,15 @@ def _get_hdrh_latencies(row: pd.Series, name: str) -> dict:
 
 
 def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
-    """
+    """Select the data for graphs from the provided data frame.
+
+    :param data: Data frame with data for graphs.
+    :param itm: Item (in this case job name) which data will be selected from
+        the input data frame.
+    :type data: pandas.DataFrame
+    :type itm: str
+    :returns: A data frame with selected data.
+    :rtype: pandas.DataFrame
     """
 
     phy = itm["phy"].split("-")
@@ -85,7 +100,25 @@ def select_trending_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 
 def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
     start: datetime, end: datetime, color: str, norm_factor: float) -> list:
-    """
+    """Generate the trending traces for the trending graph.
+
+    :param ttype: Test type (MRR, NDR, PDR).
+    :param name: The test name to be displayed as the graph title.
+    :param df: Data frame with test data.
+    :param start: The date (and time) when the selected data starts.
+    :param end: The date (and time) when the selected data ends.
+    :param color: The color of the trace (samples and trend line).
+    :param norm_factor: The factor used for normalization of the results to CPU
+        frequency set to Constants.NORM_FREQUENCY.
+    :type ttype: str
+    :type name: str
+    :type df: pandas.DataFrame
+    :type start: datetime.datetime
+    :type end: datetime.datetime
+    :type color: str
+    :type norm_factor: float
+    :returns: Traces (samples, trending line, anomalies)
+    :rtype: list
     """
 
     df = df.dropna(subset=[C.VALUE[ttype], ])
@@ -242,7 +275,24 @@ def _generate_trending_traces(ttype: str, name: str, df: pd.DataFrame,
 
 def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
     start: datetime, end: datetime, normalize: bool) -> tuple:
-    """
+    """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
+    (result_latency_forward_pdr_50_avg).
+
+    :param data: Data frame with test results.
+    :param sel: Selected tests.
+    :param layout: Layout of plot.ly graph.
+    :param start: The date (and time) when the selected data starts.
+    :param end: The date (and time) when the selected data ends.
+    :param normalize: If True, the data is normalized to CPU frquency
+        Constants.NORM_FREQUENCY.
+    :type data: pandas.DataFrame
+    :type sel: dict
+    :type layout: dict
+    :type start: datetime.datetime
+    :type end: datetype.datetype
+    :type normalize: bool
+    :returns: Trending graph(s)
+    :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
 
     if not sel:
@@ -291,7 +341,14 @@ def graph_trending(data: pd.DataFrame, sel:dict, layout: dict,
 
 
 def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
-    """
+    """Generate HDR Latency histogram graphs.
+
+    :param data: HDRH data.
+    :param layout: Layout of plot.ly graph.
+    :type data: dict
+    :type layout: dict
+    :returns: HDR latency Histogram.
+    :rtype: plotly.graph_objects.Figure
     """
 
     fig = None
diff --git a/resources/tools/dash/app/pal/trending/layout.py b/resources/tools/dash/app/pal/trending/layout.py
index 797bf6a4f5..2be19f8439 100644
--- a/resources/tools/dash/app/pal/trending/layout.py
+++ b/resources/tools/dash/app/pal/trending/layout.py
@@ -1390,4 +1390,4 @@ class Layout:
                     continue
                 df = pd.concat([df, sel_data], ignore_index=True)
 
-            return dcc.send_data_frame(df.to_csv, "trending_data.csv")
+            return dcc.send_data_frame(df.to_csv, C.TREND_DOWNLOAD_FILE_NAME)
diff --git a/resources/tools/dash/app/pal/utils/constants.py b/resources/tools/dash/app/pal/utils/constants.py
index 7a836e79f3..cc4a9e0f23 100644
--- a/resources/tools/dash/app/pal/utils/constants.py
+++ b/resources/tools/dash/app/pal/utils/constants.py
@@ -273,6 +273,9 @@ class Constants:
     # Layout of plot.ly graphs.
     REPORT_GRAPH_LAYOUT_FILE = "pal/report/layout.yaml"
 
+    # Default name of downloaded file with selected data.
+    REPORT_DOWNLOAD_FILE_NAME = "iterative_data.csv"
+
     ############################################################################
     # Statistics.
 
@@ -289,6 +292,9 @@ class Constants:
     # The default job displayed when the page is loaded first time.
     STATS_DEFAULT_JOB = "csit-vpp-perf-mrr-daily-master-2n-icx"
 
+    # Default name of downloaded file with selected data.
+    STATS_DOWNLOAD_FILE_NAME = "stats.csv"
+
     ############################################################################
     # Trending.
 
@@ -301,3 +307,6 @@ class Constants:
 
     # Layout of plot.ly graphs.
     TREND_GRAPH_LAYOUT_FILE = "pal/trending/layout.yaml"
+
+    # Default name of downloaded file with selected data.
+    TREND_DOWNLOAD_FILE_NAME = "trending_data.csv"