C-Dash: Add bandwidth to trending graphs

- NDRPDR tests only
- MRR to be done when data is in parquets

Change-Id: If2b1f76d5e2dc6f3a480eeb0950e7ab9550f8901
Signed-off-by: Tibor Frank <tifrank@cisco.com>

12 files changed, 122 insertions, 5 deletions

diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_device b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_device
new file mode 100644
index 0000000000..f619ce8a8e
--- /dev/null
+++ b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_device
diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdr
new file mode 100644
index 0000000000..06bc618bea
--- /dev/null
+++ b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdr
diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststack b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststack
new file mode 100644
index 0000000000..993d16c18c
--- /dev/null
+++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststack
diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_mrr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_mrr
index 96832850b1..96832850b1 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_mrr
+++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_mrr
diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_ndrpdr
index 2291bb8349..2291bb8349 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_ndrpdr
+++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_ndrpdr
diff --git a/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr
index 17dc30c3be..3f8b85c66e 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr
+++ b/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr
diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py
index 2bf3649778..783ebe25ff 100644
--- a/csit.infra.dash/app/cdash/data/data.py
+++ b/csit.infra.dash/app/cdash/data/data.py
@@ -30,6 +30,12 @@ from pyarrow.lib import ArrowInvalid, ArrowNotImplementedError
 from ..utils.constants import Constants as C
 
 
+# If True, pyarrow.Schema is generated. See also condition in the method
+# _write_parquet_schema.
+# To generate schema, select only one data set in data.yaml file.
+GENERATE_SCHEMA = False
+
+
 class Data:
     """Gets the data from parquets and stores it for further use by dash
     applications.
@@ -212,7 +218,10 @@ class Data:
         for itm in df:
             try:
                 # Specify the condition or remove it:
-                if pd.api.types.is_string_dtype(itm["result_rate_unit"]):
+                if all((
+                        pd.api.types.is_string_dtype(itm["<column_name>"]),
+                        pd.api.types.is_string_dtype(itm["telemetry"][0])
+                    )):
                     print(pa.Schema.from_pandas(itm))
                     pa.parquet.write_metadata(
                         pa.Schema.from_pandas(itm),
@@ -357,6 +366,18 @@ class Data:
                 time_period = days
             else:
                 time_period = None
+
+            if GENERATE_SCHEMA:
+                # Generate schema:
+                Data._write_parquet_schema(
+                    path=data_set["path"],
+                    partition_filter=partition_filter,
+                    columns=data_set.get("columns", None),
+                    days=time_period
+                )
+                return
+
+            #  Read data:
             data = Data._create_dataframe_from_parquet(
                 path=data_set["path"],
                 partition_filter=partition_filter,
diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml
index 427821d33c..f8c00731cf 100644
--- a/csit.infra.dash/app/cdash/data/data.yaml
+++ b/csit.infra.dash/app/cdash/data/data.yaml
@@ -26,6 +26,9 @@
     - result_receive_rate_rate_avg
     - result_receive_rate_rate_stdev
     - result_receive_rate_rate_unit
+    # - result_receive_rate_bandwidth_avg
+    # - result_receive_rate_bandwidth_stdev
+    # - result_receive_rate_bandwidth_unit
     - telemetry
 - data_type: trending
   partition: test_type
@@ -46,6 +49,10 @@
     - result_pdr_lower_rate_value
     - result_ndr_lower_rate_unit
     - result_ndr_lower_rate_value
+    - result_pdr_lower_bandwidth_unit
+    - result_pdr_lower_bandwidth_value
+    - result_ndr_lower_bandwidth_unit
+    - result_ndr_lower_bandwidth_value
     - result_latency_reverse_pdr_90_hdrh
     - result_latency_reverse_pdr_50_hdrh
     - result_latency_reverse_pdr_10_hdrh
diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index a1d61a07fc..fcfee26e3f 100644
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -162,6 +162,12 @@ def graph_trending(
                     f"latency [{row[C.UNIT['hoststack-latency']]}]: "
                     f"{row[C.VALUE['hoststack-latency']]:,.0f}<br>"
                 )
+            elif ttype in ("ndr", "pdr"):  # Add mrr
+                test_type = f"{ttype}-bandwidth"
+                add_info = (
+                    f"bandwidth [{row[C.UNIT[test_type]]}]: "
+                    f"{row[C.VALUE[test_type]]:,.0f}<br>"
+                )
             else:
                 add_info = str()
             hover_itm = hover_itm.replace(
@@ -295,6 +301,7 @@ def graph_trending(
 
     fig_tput = None
     fig_lat = None
+    fig_band = None
     y_units = set()
     for idx, itm in enumerate(sel):
         df = select_trending_data(data, itm)
@@ -326,6 +333,19 @@ def graph_trending(
                 fig_tput = go.Figure()
             fig_tput.add_traces(traces)
 
+        if ttype in ("ndr", "pdr"):  # Add mrr
+            traces, _ = _generate_trending_traces(
+                f"{ttype}-bandwidth",
+                itm["id"],
+                df,
+                get_color(idx),
+                norm_factor
+            )
+            if traces:
+                if not fig_band:
+                    fig_band = go.Figure()
+                fig_band.add_traces(traces)
+
         if itm["testtype"] == "pdr":
             traces, _ = _generate_trending_traces(
                 "latency",
@@ -346,10 +366,12 @@ def graph_trending(
         fig_layout["yaxis"]["title"] = \
             f"Throughput [{'|'.join(sorted(y_units))}]"
         fig_tput.update_layout(fig_layout)
+    if fig_band:
+        fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
     if fig_lat:
         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 
-    return fig_tput, fig_lat
+    return fig_tput, fig_band, fig_lat
 
 
 def graph_tm_trending(
diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index 84a68f5050..d08f911036 100644
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -265,7 +265,7 @@ class Layout:
                         dbc.Offcanvas(
                             class_name="w-50",
                             id="offcanvas-metadata",
-                            title="Throughput And Latency",
+                            title="Detailed Information",
                             placement="end",
                             is_open=False,
                             children=[
@@ -646,9 +646,21 @@ class Layout:
             tab_items.append(
                 dbc.Tab(
                     children=dcc.Graph(
-                        id={"type": "graph", "index": "lat"},
+                        id={"type": "graph", "index": "bandwidth"},
                         figure=graphs[1]
                     ),
+                    label="Bandwidth",
+                    tab_id="tab-bandwidth"
+                )
+            )
+
+        if graphs[2]:
+            tab_items.append(
+                dbc.Tab(
+                    children=dcc.Graph(
+                        id={"type": "graph", "index": "lat"},
+                        figure=graphs[2]
+                    ),
                     label="Latency",
                     tab_id="tab-lat"
                 )
@@ -1690,7 +1702,14 @@ class Layout:
             trigger = Trigger(callback_context.triggered)
 
             try:
-                idx = 0 if trigger.idx == "tput" else 1
+                if trigger.idx == "tput":
+                    idx = 0
+                elif trigger.idx == "bandwidth":
+                    idx = 1
+                elif trigger.idx == "lat":
+                    idx = 2
+                else:
+                    raise PreventUpdate
                 graph_data = graph_data[idx]["points"][0]
             except (IndexError, KeyError, ValueError, TypeError):
                 raise PreventUpdate
@@ -1721,6 +1740,8 @@ class Layout:
 
             if trigger.idx == "tput":
                 title = "Throughput"
+            elif trigger.idx == "bandwidth":
+                title = "Bandwidth"
             elif trigger.idx == "lat":
                 title = "Latency"
                 hdrh_data = graph_data.get("customdata", None)
diff --git a/csit.infra.dash/app/cdash/trending/layout.yaml b/csit.infra.dash/app/cdash/trending/layout.yaml
index 5e41b03118..2fcb206153 100644
--- a/csit.infra.dash/app/cdash/trending/layout.yaml
+++ b/csit.infra.dash/app/cdash/trending/layout.yaml
@@ -38,6 +38,46 @@ plot-trending-tput:
   hoverlabel:
     namelength: -1
 
+plot-trending-bandwidth:
+  autosize: True
+  showlegend: False
+  yaxis:
+    showticklabels: True
+    tickformat: ".3s"
+    title: "Bandwidth [bps]"
+    hoverformat: ".5s"
+    gridcolor: "rgb(238, 238, 238)"
+    linecolor: "rgb(238, 238, 238)"
+    showline: True
+    zeroline: False
+    tickcolor: "rgb(238, 238, 238)"
+    linewidth: 1
+    showgrid: True
+  xaxis:
+    title: 'Date [MMDD]'
+    type: "date"
+    autorange: True
+    fixedrange: False
+    showgrid: True
+    gridcolor: "rgb(238, 238, 238)"
+    showline: True
+    linecolor: "rgb(238, 238, 238)"
+    zeroline: False
+    linewidth: 1
+    showticklabels: True
+    tickcolor: "rgb(238, 238, 238)"
+    tickmode: "auto"
+    tickformat: "%m%d"
+  margin:
+    r: 20
+    b: 0
+    t: 5
+    l: 70
+  paper_bgcolor: "#fff"
+  plot_bgcolor: "#fff"
+  hoverlabel:
+    namelength: -1
+
 plot-trending-lat:
   autosize: True
   showlegend: False
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 34892e79bc..fb887fe79e 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -229,6 +229,9 @@ class Constants:
         "mrr": "result_receive_rate_rate_avg",
         "ndr": "result_ndr_lower_rate_value",
         "pdr": "result_pdr_lower_rate_value",
+        "mrr-bandwidth": "result_receive_rate_bandwidth_avg",
+        "ndr-bandwidth": "result_ndr_lower_bandwidth_value",
+        "pdr-bandwidth": "result_pdr_lower_bandwidth_value",
         "latency": "result_latency_forward_pdr_50_avg",
         "hoststack-cps": "result_rate_value",
         "hoststack-rps": "result_rate_value",
@@ -251,6 +254,9 @@ class Constants:
         "mrr": "result_receive_rate_rate_unit",
         "ndr": "result_ndr_lower_rate_unit",
         "pdr": "result_pdr_lower_rate_unit",
+        "mrr-bandwidth": "result_receive_rate_bandwidth_unit",
+        "ndr-bandwidth": "result_ndr_lower_bandwidth_unit",
+        "pdr-bandwidth": "result_pdr_lower_bandwidth_unit",
         "latency": "result_latency_forward_pdr_50_unit",
         "hoststack-cps": "result_rate_unit",
         "hoststack-rps": "result_rate_unit",