C-Dash: Show mrr trials in trending graphs

Change-Id: I5b983aa249dd5ac62f78e6f980ab81f270234b0f
Signed-off-by: Tibor Frank <tifrank@cisco.com>

6 files changed, 290 insertions, 131 deletions

diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
index 5f6ba2db2a..a951ab85e9 100644
--- a/csit.infra.dash/app/cdash/comparisons/tables.py
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -320,7 +320,7 @@ def comparison_table(
             c_mean = c_row["mean"].values[0]
             c_std = c_row["stdev"].values[0]
             if r_mean == 0.0 or c_mean == 0.0:
-                break
+                continue
             unit.add(f"{s_unit_factor}{row['unit']}")
             l_name.append(row["name"])
             l_r_mean.append(r_mean / unit_factor)
diff --git a/csit.infra.dash/app/cdash/data/_metadata/trending_mrr b/csit.infra.dash/app/cdash/data/_metadata/trending_mrr
index 9d4e126e59..768a9ec21c 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/trending_mrr
+++ b/csit.infra.dash/app/cdash/data/_metadata/trending_mrr
diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml
index 54f9cece4c..db7775b210 100644
--- a/csit.infra.dash/app/cdash/data/data.yaml
+++ b/csit.infra.dash/app/cdash/data/data.yaml
@@ -26,9 +26,11 @@
     - result_receive_rate_rate_avg
     - result_receive_rate_rate_stdev
     - result_receive_rate_rate_unit
+    - result_receive_rate_rate_values
     - result_receive_rate_bandwidth_avg
     - result_receive_rate_bandwidth_stdev
     - result_receive_rate_bandwidth_unit
+    - result_receive_rate_bandwidth_values
     - telemetry
 - data_type: trending
   partition: test_type
@@ -257,11 +259,9 @@
     - result_receive_rate_rate_avg
     - result_receive_rate_rate_stdev
     - result_receive_rate_rate_unit
-    - result_receive_rate_rate_values
     - result_receive_rate_bandwidth_avg
     - result_receive_rate_bandwidth_stdev
     - result_receive_rate_bandwidth_unit
-    - result_receive_rate_bandwidth_values
 - data_type: iterative
   partition: test_type
   partition_name: ndrpdr
@@ -394,11 +394,9 @@
     - result_receive_rate_rate_avg
     - result_receive_rate_rate_stdev
     - result_receive_rate_rate_unit
-    - result_receive_rate_rate_values
     - result_receive_rate_bandwidth_avg
     - result_receive_rate_bandwidth_stdev
     - result_receive_rate_bandwidth_unit
-    - result_receive_rate_bandwidth_values
 - data_type: iterative
   partition: test_type
   partition_name: ndrpdr
diff --git a/csit.infra.dash/app/cdash/report/layout.py b/csit.infra.dash/app/cdash/report/layout.py
index f485894d78..2d325b0447 100644
--- a/csit.infra.dash/app/cdash/report/layout.py
+++ b/csit.infra.dash/app/cdash/report/layout.py
@@ -348,7 +348,7 @@ class Layout:
         :returns: Control panel.
         :rtype: list
         """
-        return [
+        test_selection = [
             dbc.Row(
                 class_name="g-0 p-1",
                 children=[
@@ -604,52 +604,36 @@ class Layout:
             dbc.Row(
                 class_name="g-0 p-1",
                 children=[
-                    dbc.InputGroup(
-                        [
-                            dbc.InputGroupText(
-                                children=show_tooltip(
-                                    self._tooltips,
-                                    "help-normalize",
-                                    "Normalization"
-                                )
-                            ),
-                            dbc.Col(
-                                children=[
-                                    dbc.Checklist(
-                                        id="normalize",
-                                        options=[{
-                                            "value": "normalize",
-                                            "label": (
-                                                "Normalize to CPU frequency "
-                                                "2GHz"
-                                            )
-                                        }],
-                                        value=[],
-                                        inline=True,
-                                        class_name="ms-2"
-                                    )
-                                ]
-                            )
-                        ],
-                        style={"align-items": "center"},
-                        size="sm"
+                    dbc.Button(
+                        id={"type": "ctrl-btn", "index": "add-test"},
+                        children="Add Selected",
+                        color="info",
+                        class_name="p-1"
                     )
                 ]
-            ),
+            )
+        ]
+        processing = [
             dbc.Row(
                 class_name="g-0 p-1",
                 children=[
-                    dbc.Button(
-                        id={"type": "ctrl-btn", "index": "add-test"},
-                        children="Add Selected",
-                        color="info"
+                    dbc.Checklist(
+                        id="normalize",
+                        options=[{
+                            "value": "normalize",
+                            "label": "Normalize to 2GHz CPU frequency"
+                        }],
+                        value=[],
+                        inline=True,
+                        class_name="ms-2"
                     )
                 ]
-            ),
+            )
+        ]
+        test_list = [
             dbc.Row(
                 id="row-card-sel-tests",
                 class_name="g-0 p-1",
-                style=C.STYLE_DISABLED,
                 children=[
                     dbc.ListGroup(
                         class_name="overflow-auto p-0",
@@ -663,21 +647,20 @@ class Layout:
             dbc.Stack(
                 id="row-btns-sel-tests",
                 class_name="g-0 p-1",
-                style=C.STYLE_DISABLED,
                 gap=2,
                 children=[
                     dbc.ButtonGroup(children=[
                         dbc.Button(
                             id={"type": "ctrl-btn", "index": "rm-test"},
                             children="Remove Selected",
-                            class_name="w-100",
+                            class_name="w-100 p-1",
                             color="info",
                             disabled=False
                         ),
                         dbc.Button(
                             id={"type": "ctrl-btn", "index": "rm-test-all"},
                             children="Remove All",
-                            class_name="w-100",
+                            class_name="w-100 p-1",
                             color="info",
                             disabled=False
                         )
@@ -686,14 +669,14 @@ class Layout:
                         dbc.Button(
                             id="plot-btn-url",
                             children="Show URL",
-                            class_name="w-100",
+                            class_name="w-100 p-1",
                             color="info",
                             disabled=False
                         ),
                         dbc.Button(
                             id="plot-btn-download",
                             children="Download Data",
-                            class_name="w-100",
+                            class_name="w-100 p-1",
                             color="info",
                             disabled=False
                         )
@@ -702,6 +685,59 @@ class Layout:
             )
         ]
 
+        return [
+            dbc.Row(
+                dbc.Card(
+                    [
+                        dbc.CardHeader(
+                            html.H5("Test Selection")
+                        ),
+                        dbc.CardBody(
+                            children=test_selection,
+                            class_name="g-0 p-0"
+                        )
+                    ],
+                    color="secondary",
+                    outline=True
+                ),
+                class_name="g-0 p-1"
+            ),
+            dbc.Row(
+                dbc.Card(
+                    [
+                        dbc.CardHeader(
+                            html.H5("Data Manipulations")
+                        ),
+                        dbc.CardBody(
+                            children=processing,
+                            class_name="g-0 p-0"
+                        )
+                    ],
+                    color="secondary",
+                    outline=True
+                ),
+                class_name="g-0 p-1"
+            ),
+            dbc.Row(
+                dbc.Card(
+                    [
+                        dbc.CardHeader(
+                            html.H5("Selected Tests")
+                        ),
+                        dbc.CardBody(
+                            children=test_list,
+                            class_name="g-0 p-0"
+                        )
+                    ],
+                    color="secondary",
+                    outline=True
+                ),
+                id = "row-selected-tests",
+                class_name="g-0 p-1",
+                style=C.STYLE_DISABLED,
+            )
+        ]
+
     def _get_plotting_area(
             self,
             tests: list,
@@ -798,8 +834,7 @@ class Layout:
                 Output("store-control-panel", "data"),
                 Output("store-selected-tests", "data"),
                 Output("plotting-area", "children"),
-                Output("row-card-sel-tests", "style"),
-                Output("row-btns-sel-tests", "style"),
+                Output("row-selected-tests", "style"),
                 Output("lg-selected", "children"),
 
                 Output({"type": "ctrl-dd", "index": "rls"}, "value"),
@@ -869,8 +904,7 @@ class Layout:
                 url_params = None
 
             plotting_area = no_update
-            row_card_sel_tests = no_update
-            row_btns_sel_tests = no_update
+            row_sel_tests = no_update
             lg_selected = no_update
 
             trigger = Trigger(callback_context.triggered)
@@ -882,8 +916,7 @@ class Layout:
                 except (KeyError, IndexError, AttributeError):
                     pass
                 if store_sel:
-                    row_card_sel_tests = C.STYLE_ENABLED
-                    row_btns_sel_tests = C.STYLE_ENABLED
+                    row_sel_tests = C.STYLE_ENABLED
                     last_test = store_sel[-1]
                     test = self._spec_tbs[last_test["rls"]][last_test["dut"]]\
                         [last_test["dutver"]][last_test["area"]]\
@@ -1235,20 +1268,17 @@ class Layout:
                             {"store_sel": store_sel, "norm": normalize}
                         )
                     )
-                    row_card_sel_tests = C.STYLE_ENABLED
-                    row_btns_sel_tests = C.STYLE_ENABLED
+                    row_sel_tests = C.STYLE_ENABLED
                 else:
                     plotting_area = C.PLACEHOLDER
-                    row_card_sel_tests = C.STYLE_DISABLED
-                    row_btns_sel_tests = C.STYLE_DISABLED
+                    row_sel_tests = C.STYLE_DISABLED
                     store_sel = list()
 
             ret_val = [
                 ctrl_panel.panel,
                 store_sel,
                 plotting_area,
-                row_card_sel_tests,
-                row_btns_sel_tests,
+                row_sel_tests,
                 lg_selected
             ]
             ret_val.extend(ctrl_panel.values)
diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index 1a507dfeea..b55b18a444 100644
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -18,6 +18,8 @@ import logging
 import plotly.graph_objects as go
 import pandas as pd
 
+from numpy import nan
+
 from ..utils.constants import Constants as C
 from ..utils.utils import get_color, get_hdrh_latencies
 from ..utils.anomalies import classify_anomalies
@@ -73,7 +75,8 @@ def graph_trending(
         data: pd.DataFrame,
         sel: dict,
         layout: dict,
-        normalize: bool=False
+        normalize: bool=False,
+        trials: bool=False
     ) -> tuple:
     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
     (result_latency_forward_pdr_50_avg).
@@ -83,10 +86,12 @@ def graph_trending(
     :param layout: Layout of plot.ly graph.
     :param normalize: If True, the data is normalized to CPU frquency
         Constants.NORM_FREQUENCY.
+    :param trials: If True, MRR trials are displayed in the trending graph.
     :type data: pandas.DataFrame
     :type sel: dict
     :type layout: dict
     :type normalize: bool
+    :type: trials: bool
     :returns: Trending graph(s)
     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
@@ -279,7 +284,7 @@ def graph_trending(
                 marker={
                     "size": 5,
                     "color": color,
-                    "symbol": "circle",
+                    "symbol": "circle"
                 },
                 text=hover,
                 hoverinfo="text",
@@ -369,6 +374,56 @@ def graph_trending(
         return traces, units
 
 
+    def _add_mrr_trials_traces(
+            ttype: str,
+            name: str,
+            df: pd.DataFrame,
+            color: str,
+            nf: float
+        ) -> list:
+        """Add the traces with mrr trials.
+
+        :param ttype: Test type (mrr, mrr-bandwidth).
+        :param name: The test name to be displayed in hover.
+        :param df: Data frame with test data.
+        :param color: The color of the trace.
+        :param nf: The factor used for normalization of the results to
+            CPU frequency set to Constants.NORM_FREQUENCY.
+        :type ttype: str
+        :type name: str
+        :type df: pandas.DataFrame
+        :type color: str
+        :type nf: float
+        :returns: list of Traces
+        :rtype: list
+        """
+        traces = list()
+        x_axis = df["start_time"].tolist()
+        y_data = df[C.VALUE[ttype].replace("avg", "values")].tolist()
+
+        for idx_trial in range(10):
+            y_axis = list()
+            for idx_run in range(len(x_axis)):
+                try:
+                    y_axis.append(y_data[idx_run][idx_trial] * nf)
+                except IndexError:
+                    y_axis.append(nan)
+            traces.append(go.Scatter(
+                x=x_axis,
+                y=y_axis,
+                name=name,
+                mode="markers",
+                marker={
+                    "size": 2,
+                    "color": color,
+                    "symbol": "circle"
+                },
+                showlegend=True,
+                legendgroup=name
+            ))
+        return traces
+
+
     fig_tput = None
     fig_lat = None
     fig_band = None
@@ -401,6 +456,14 @@ def graph_trending(
         if traces:
             if not fig_tput:
                 fig_tput = go.Figure()
+            if trials and "mrr" in ttype:
+                traces.extend(_add_mrr_trials_traces(
+                    ttype,
+                    itm["id"],
+                    df,
+                    get_color(idx),
+                    norm_factor
+                ))
             fig_tput.add_traces(traces)
 
         if ttype in C.TESTS_WITH_BANDWIDTH:
@@ -414,6 +477,14 @@ def graph_trending(
             if traces:
                 if not fig_band:
                     fig_band = go.Figure()
+                if trials and "mrr" in ttype:
+                    traces.extend(_add_mrr_trials_traces(
+                        f"{ttype}-bandwidth",
+                        itm["id"],
+                        df,
+                        get_color(idx),
+                        norm_factor
+                    ))
                 fig_band.add_traces(traces)
 
         if ttype in C.TESTS_WITH_LATENCY:
diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index da90ae26f9..b4487cf127 100644
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -65,7 +65,8 @@ CP_PARAMS = {
     "cl-tsttype-all-val": list(),
     "cl-tsttype-all-opt": C.CL_ALL_DISABLED,
     "btn-add-dis": True,
-    "cl-normalize-val": list()
+    "cl-normalize-val": list(),
+    "cl-show-trials": list()
 }
 
 
@@ -303,7 +304,7 @@ class Layout:
         :returns: Control panel.
         :rtype: list
         """
-        return [
+        test_selection = [
             dbc.Row(
                 dbc.InputGroup(
                     [
@@ -459,37 +460,43 @@ class Layout:
                 class_name="g-0 p-1"
             ),
             dbc.Row(
-                dbc.InputGroup(
-                    [
-                        dbc.InputGroupText(show_tooltip(
-                            self._tooltips,
-                            "help-normalize",
-                            "Normalization"
-                        )),
-                        dbc.Col(dbc.Checklist(
-                            id="normalize",
-                            options=[{
-                                "value": "normalize",
-                                "label": "Normalize to CPU frequency 2GHz"
-                            }],
-                            value=[],
-                            inline=True,
-                            class_name="ms-2"
-                        ))
-                    ],
-                    style={"align-items": "center"},
-                    size="sm"
-                ),
-                class_name="g-0 p-1"
-            ),
-            dbc.Row(
                 dbc.Button(
                     id={"type": "ctrl-btn", "index": "add-test"},
                     children="Add Selected",
-                    color="info"
+                    color="info",
+                    class_name="p-1"
                 ),
                 class_name="g-0 p-1"
-            ),
+            )
+        ]
+        processing = [
+            dbc.Row(
+                class_name="g-0 p-1",
+                children=[
+                    dbc.Checklist(
+                        id="normalize",
+                        options=[{
+                            "value": "normalize",
+                            "label": "Normalize to 2GHz CPU frequency"
+                        }],
+                        value=[],
+                        inline=True,
+                        class_name="ms-2"
+                    ),
+                    dbc.Checklist(
+                        id="show-trials",
+                        options=[{
+                            "value": "trials",
+                            "label": "Show MRR Trials"
+                        }],
+                        value=[],
+                        inline=True,
+                        class_name="ms-2"
+                    )
+                ]
+            )
+        ]
+        test_list = [
             dbc.Row(
                 dbc.ListGroup(
                     class_name="overflow-auto p-0",
@@ -498,39 +505,43 @@ class Layout:
                     style={"max-height": "20em"},
                     flush=True
                 ),
-                id="row-card-sel-tests",
-                class_name="g-0 p-1",
-                style=C.STYLE_DISABLED,
+                class_name="g-0 p-1"
             ),
             dbc.Row(
-                dbc.ButtonGroup([
-                    dbc.Button(
-                        "Remove Selected",
-                        id={"type": "ctrl-btn", "index": "rm-test"},
-                        class_name="w-100",
-                        color="info",
-                        disabled=False
-                    ),
-                    dbc.Button(
-                        "Remove All",
-                        id={"type": "ctrl-btn", "index": "rm-test-all"},
-                        class_name="w-100",
-                        color="info",
-                        disabled=False
-                    )
-                ]),
-                id="row-btns-sel-tests",
-                class_name="g-0 p-1",
-                style=C.STYLE_DISABLED,
+                dbc.ButtonGroup(
+                    children=[
+                        dbc.Button(
+                            "Remove Selected",
+                            id={"type": "ctrl-btn", "index": "rm-test"},
+                            class_name="w-100 p-1",
+                            color="info",
+                            disabled=False
+                        ),
+                        dbc.Button(
+                            "Remove All",
+                            id={"type": "ctrl-btn", "index": "rm-test-all"},
+                            class_name="w-100 p-1",
+                            color="info",
+                            disabled=False
+                        )
+                    ]
+                ),
+                class_name="g-0 p-1"
             ),
             dbc.Stack(
                 [
                     dbc.Button(
                         "Add Telemetry Panel",
                         id={"type": "telemetry-btn", "index": "open"},
-                        color="info"
+                        color="info",
+                        class_name="p-1"
+                    ),
+                    dbc.Button(
+                        "Show URL",
+                        id="plot-btn-url",
+                        color="info",
+                        class_name="p-1"
                     ),
-                    dbc.Button("Show URL", id="plot-btn-url", color="info"),
                     dbc.Modal(
                         [
                             dbc.ModalHeader(dbc.ModalTitle("URL")),
@@ -542,13 +553,65 @@ class Layout:
                         scrollable=True
                     )
                 ],
-                id="row-btns-add-tm",
                 class_name="g-0 p-1",
-                style=C.STYLE_DISABLED,
                 gap=2
             )
         ]
 
+        return [
+            dbc.Row(
+                dbc.Card(
+                    [
+                        dbc.CardHeader(
+                            html.H5("Test Selection")
+                        ),
+                        dbc.CardBody(
+                            children=test_selection,
+                            class_name="g-0 p-0"
+                        )
+                    ],
+                    color="secondary",
+                    outline=True
+                ),
+                class_name="g-0 p-1"
+            ),
+            dbc.Row(
+                dbc.Card(
+                    [
+                        dbc.CardHeader(
+                            html.H5("Data Manipulations")
+                        ),
+                        dbc.CardBody(
+                            children=processing,
+                            class_name="g-0 p-0"
+                        )
+                    ],
+                    color="secondary",
+                    outline=True
+                ),
+                class_name="g-0 p-1"
+            ),
+            dbc.Row(
+                dbc.Card(
+                    [
+                        dbc.CardHeader(
+                            html.H5("Selected Tests")
+                        ),
+                        dbc.CardBody(
+                            children=test_list,
+                            class_name="g-0 p-0"
+                        )
+                    ],
+                    color="secondary",
+                    outline=True
+                ),
+                id = "row-selected-tests",
+                class_name="g-0 p-1",
+                style=C.STYLE_DISABLED,
+            )
+        ]
+
+
     def _add_plotting_col(self) -> dbc.Col:
         """Add column with plots. It is placed on the right side.
 
@@ -919,9 +982,7 @@ class Layout:
             Output("plotting-area-trending", "children"),
             Output("plotting-area-telemetry", "children"),
             Output("col-plotting-area", "style"),
-            Output("row-card-sel-tests", "style"),
-            Output("row-btns-sel-tests", "style"),
-            Output("row-btns-add-tm", "style"),
+            Output("row-selected-tests", "style"),
             Output("lg-selected", "children"),
             Output({"type": "telemetry-search-out", "index": ALL}, "children"),
             Output({"type": "plot-mod-telemetry", "index": ALL}, "is_open"),
@@ -952,6 +1013,7 @@ class Layout:
             Output({"type": "ctrl-cl", "index": "tsttype-all"}, "options"),
             Output({"type": "ctrl-btn", "index": "add-test"}, "disabled"),
             Output("normalize", "value"),
+            Output("show-trials", "value"),
 
             State("store", "data"),
             State({"type": "sel-cl", "index": ALL}, "value"),
@@ -968,6 +1030,7 @@ class Layout:
             Input({"type": "tm-dd", "index": ALL}, "value"),
 
             Input("normalize", "value"),
+            Input("show-trials", "value"),
             Input({"type": "telemetry-search-in", "index": ALL}, "value"),
             Input({"type": "telemetry-btn", "index": ALL}, "n_clicks"),
             Input({"type": "tm-btn-remove", "index": ALL}, "n_clicks"),
@@ -1119,6 +1182,10 @@ class Layout:
                 ctrl_panel.set({"cl-normalize-val": trigger.value})
                 store["trending-graphs"] = None
                 on_draw[0] = True
+            elif trigger.type == "show-trials":
+                ctrl_panel.set({"cl-show-trials": trigger.value})
+                store["trending-graphs"] = None
+                on_draw[0] = True
             elif trigger.type == "ctrl-dd":
                 if trigger.idx == "dut":
                     try:
@@ -1431,7 +1498,8 @@ class Layout:
                             self._data,
                             store_sel,
                             self._graph_layout,
-                            bool(ctrl_panel.get("cl-normalize-val"))
+                            bool(ctrl_panel.get("cl-normalize-val")),
+                            bool(ctrl_panel.get("cl-show-trials"))
                         )
                         if graphs and graphs[0]:
                             store["trending-graphs"] = graphs
@@ -1462,16 +1530,12 @@ class Layout:
                                 store["telemetry-graphs"]
                             )
                     col_plotting_area = C.STYLE_ENABLED
-                    row_card_sel_tests = C.STYLE_ENABLED
-                    row_btns_sel_tests = C.STYLE_ENABLED
-                    row_btns_add_tm = C.STYLE_ENABLED
+                    row_selected_tests = C.STYLE_ENABLED
                 else:
                     plotting_area_trending = no_update
                     plotting_area_telemetry = C.PLACEHOLDER
                     col_plotting_area = C.STYLE_DISABLED
-                    row_card_sel_tests = C.STYLE_DISABLED
-                    row_btns_sel_tests = C.STYLE_DISABLED
-                    row_btns_add_tm = C.STYLE_DISABLED
+                    row_selected_tests = C.STYLE_DISABLED
                     lg_selected = no_update
                     store_sel = list()
                     tm_panels = list()
@@ -1481,9 +1545,7 @@ class Layout:
             else:
                 plotting_area_trending = no_update
                 col_plotting_area = no_update
-                row_card_sel_tests = no_update
-                row_btns_sel_tests = no_update
-                row_btns_add_tm = no_update
+                row_selected_tests = no_update
                 lg_selected = no_update
 
             store["url"] = gen_new_url(parsed_url, new_url_params)
@@ -1499,9 +1561,7 @@ class Layout:
                 plotting_area_trending,
                 plotting_area_telemetry,
                 col_plotting_area,
-                row_card_sel_tests,
-                row_btns_sel_tests,
-                row_btns_add_tm,
+                row_selected_tests,
                 lg_selected,
                 search_out,
                 is_open,