C-Dash: Hover info and y-axis in trending and iterative graphs

Signed-off-by: Tibor Frank <tifrank@cisco.com>
Change-Id: I49a7230b4e44f9520b347ec40de0d67581ad51ab

4 files changed, 75 insertions, 45 deletions

diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml
index 9cc366e62f..7a03e13d08 100644
--- a/csit.infra.dash/app/cdash/data/data.yaml
+++ b/csit.infra.dash/app/cdash/data/data.yaml
@@ -80,22 +80,12 @@
     - dut_version
     - hosts
     - tg_type
-    # - tg_version
     - result_bandwidth_unit
     - result_bandwidth_value
     - result_rate_unit
     - result_rate_value
     - result_latency_unit
     - result_latency_value
-    # - result_failed_requests_unit
-    # - result_failed_requests_value
-    # - result_completed_requests_unit
-    # - result_completed_requests_value
-    # - result_retransmits_unit
-    # - result_retransmits_value
-    # - result_duration_unit
-    # - result_duration_value
-    # - result_type
     - start_time
     - passed
     - telemetry
@@ -277,21 +267,12 @@
     - dut_version
     - hosts
     - tg_type
-    # - tg_version
     - result_bandwidth_unit
     - result_bandwidth_value
     - result_rate_unit
     - result_rate_value
-    - result_latency_unit
-    - result_latency_value
-    # - result_failed_requests_unit
-    # - result_failed_requests_value
-    # - result_completed_requests_unit
-    # - result_completed_requests_value
-    # - result_retransmits_unit
-    # - result_retransmits_value
-    # - result_duration_unit
-    # - result_duration_value
+    # - result_latency_unit
+    # - result_latency_value
     - start_time
     - passed
     - telemetry
diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py
index 411a599b1c..2d1f4b1873 100644
--- a/csit.infra.dash/app/cdash/report/graphs.py
+++ b/csit.infra.dash/app/cdash/report/graphs.py
@@ -98,28 +98,40 @@ def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
     lat_traces = list()
     y_lat_max = 0
     x_lat = list()
+    y_units = set()
     show_latency = False
     show_tput = False
     for idx, itm in enumerate(sel):
+
         itm_data = select_iterative_data(data, itm)
         if itm_data.empty:
             continue
+
         phy = itm["phy"].split("-")
         topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
         norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
             if normalize else 1.0
+
+        if itm["area"] == "hoststack":
+            ttype = f"hoststack-{itm['testtype']}"
+        else:
+            ttype = itm["testtype"]
+
+        y_units.update(itm_data[C.UNIT[ttype]].unique().tolist())
+
         if itm["testtype"] == "mrr":
-            y_data_raw = itm_data[C.VALUE_ITER[itm["testtype"]]].to_list()[0]
+            y_data_raw = itm_data[C.VALUE_ITER[ttype]].to_list()[0]
             y_data = [(y * norm_factor) for y in y_data_raw]
             if len(y_data) > 0:
                 y_tput_max = \
                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
         else:
-            y_data_raw = itm_data[C.VALUE_ITER[itm["testtype"]]].to_list()
+            y_data_raw = itm_data[C.VALUE_ITER[ttype]].to_list()
             y_data = [(y * norm_factor) for y in y_data_raw]
             if y_data:
                 y_tput_max = \
                     max(y_data) if max(y_data) > y_tput_max else y_tput_max
+
         nr_of_samples = len(y_data)
         tput_kwargs = dict(
             y=y_data,
@@ -137,7 +149,7 @@ def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
         tput_traces.append(go.Box(**tput_kwargs))
         show_tput = True
 
-        if itm["testtype"] == "pdr":
+        if ttype == "pdr":
             y_lat_row = itm_data[C.VALUE_ITER["pdr-lat"]].to_list()
             y_lat = [(y / norm_factor) for y in y_lat_row]
             if y_lat:
@@ -166,8 +178,9 @@ def graph_iterative(data: pd.DataFrame, sel:dict, layout: dict,
         pl_tput = deepcopy(layout["plot-throughput"])
         pl_tput["xaxis"]["tickvals"] = [i for i in range(len(sel))]
         pl_tput["xaxis"]["ticktext"] = [str(i + 1) for i in range(len(sel))]
+        pl_tput["yaxis"]["title"] = f"Throughput [{'|'.join(sorted(y_units))}]"
         if y_tput_max:
-            pl_tput["yaxis"]["range"] = [0, (int(y_tput_max / 1e6) + 1) * 1e6]
+            pl_tput["yaxis"]["range"] = [0, (int(y_tput_max / 1e6) + 2) * 1e6]
         fig_tput = go.Figure(data=tput_traces, layout=pl_tput)
 
     if show_latency:
diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index b6581e67f4..695bb0e287 100644
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -141,13 +141,14 @@ def graph_trending(
 
         df = df.dropna(subset=[C.VALUE[ttype], ])
         if df.empty:
-            return list()
+            return list(), list()
 
         x_axis = df["start_time"].tolist()
         if ttype == "pdr-lat":
             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
         else:
             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
+        units = df[C.UNIT[ttype]].unique().tolist()
 
         anomalies, trend_avg, trend_stdev = classify_anomalies(
             {k: v for k, v in zip(x_axis, y_data)}
@@ -162,6 +163,7 @@ def graph_trending(
                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
                 f"<prop> [{row[C.UNIT[ttype]]}]: {y_data[idx]:,.0f}<br>"
                 f"<stdev>"
+                f"<additional-info>"
                 f"{d_type}-ref: {row['dut_version']}<br>"
                 f"csit-ref: {row['job']}/{row['build']}<br>"
                 f"hosts: {', '.join(row['hosts'])}"
@@ -172,10 +174,19 @@ def graph_trending(
                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
                 )
             else:
-                stdev = ""
+                stdev = str()
+            if ttype in ("hoststack-cps", "hoststack-rps"):
+                add_info = (
+                    f"bandwidth [{row[C.UNIT['hoststack-bps']]}]: "
+                    f"{row[C.VALUE['hoststack-bps']]:,.0f}<br>"
+                    f"latency [{row[C.UNIT['hoststack-lat']]}]: "
+                    f"{row[C.VALUE['hoststack-lat']]:,.0f}<br>"
+                )
+            else:
+                add_info = str()
             hover_itm = hover_itm.replace(
                 "<prop>", "latency" if ttype == "pdr-lat" else "average"
-            ).replace("<stdev>", stdev)
+            ).replace("<stdev>", stdev).replace("<additional-info>", add_info)
             hover.append(hover_itm)
             if ttype == "pdr-lat":
                 customdata_samples.append(_get_hdrh_latencies(row, name))
@@ -191,8 +202,8 @@ def graph_trending(
             d_type = "trex" if row["dut_type"] == "none" else row["dut_type"]
             hover_itm = (
                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
-                f"trend [pps]: {avg:,.0f}<br>"
-                f"stdev [pps]: {stdev:,.0f}<br>"
+                f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
+                f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
                 f"{d_type}-ref: {row['dut_version']}<br>"
                 f"csit-ref: {row['job']}/{row['build']}<br>"
                 f"hosts: {', '.join(row['hosts'])}"
@@ -294,11 +305,12 @@ def graph_trending(
                 )
             )
 
-        return traces
+        return traces, units
 
 
     fig_tput = None
     fig_lat = None
+    y_units = set()
     for idx, itm in enumerate(sel):
         df = select_trending_data(data, itm)
         if df is None or df.empty:
@@ -311,23 +323,44 @@ def graph_trending(
                 if topo_arch else 1.0
         else:
             norm_factor = 1.0
-        traces = _generate_trending_traces(itm["testtype"], itm["id"], df,
-            get_color(idx), norm_factor)
+
+        if itm["area"] == "hoststack":
+            ttype = f"hoststack-{itm['testtype']}"
+        else:
+            ttype = itm["testtype"]
+
+        traces, units = _generate_trending_traces(
+            ttype,
+            itm["id"],
+            df,
+            get_color(idx),
+            norm_factor
+        )
         if traces:
             if not fig_tput:
                 fig_tput = go.Figure()
             fig_tput.add_traces(traces)
 
         if itm["testtype"] == "pdr":
-            traces = _generate_trending_traces("pdr-lat", itm["id"], df,
-                get_color(idx), norm_factor)
+            traces, _ = _generate_trending_traces(
+                "pdr-lat",
+                itm["id"],
+                df,
+                get_color(idx),
+                norm_factor
+            )
             if traces:
                 if not fig_lat:
                     fig_lat = go.Figure()
                 fig_lat.add_traces(traces)
 
+        y_units.update(units)
+
     if fig_tput:
-        fig_tput.update_layout(layout.get("plot-trending-tput", dict()))
+        fig_layout = layout.get("plot-trending-tput", dict())
+        fig_layout["yaxis"]["title"] = \
+            f"Throughput [{'|'.join(sorted(y_units))}]"
+        fig_tput.update_layout(fig_layout)
     if fig_lat:
         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 4eb52d2117..12d7ee5702 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -203,9 +203,10 @@ class Constants:
         "ndr": "result_ndr_lower_rate_value",
         "pdr": "result_pdr_lower_rate_value",
         "pdr-lat": "result_latency_forward_pdr_50_avg",
-        "cps": "result_bandwidth_value",
-        "rps": "result_bandwidth_value",
-        "bps": "result_bandwidth_value"
+        "hoststack-cps": "result_rate_value",
+        "hoststack-rps": "result_rate_value",
+        "hoststack-bps": "result_bandwidth_value",
+        "hoststack-lat": "result_latency_value"
     }
 
     VALUE_ITER = {
@@ -213,9 +214,10 @@ class Constants:
         "ndr": "result_ndr_lower_rate_value",
         "pdr": "result_pdr_lower_rate_value",
         "pdr-lat": "result_latency_forward_pdr_50_avg",
-        "cps": "result_bandwidth_value",
-        "rps": "result_bandwidth_value",
-        "bps": "result_bandwidth_value"
+        "hoststack-cps": "result_rate_value",
+        "hoststack-rps": "result_rate_value",
+        "hoststack-bps": "result_bandwidth_value",
+        "hoststack-lat": "result_latency_value"
     }
 
     UNIT = {
@@ -223,9 +225,10 @@ class Constants:
         "ndr": "result_ndr_lower_rate_unit",
         "pdr": "result_pdr_lower_rate_unit",
         "pdr-lat": "result_latency_forward_pdr_50_unit",
-        "cps": "result_bandwidth_unit",
-        "rps": "result_bandwidth_unit",
-        "bps": "result_bandwidth_unit"
+        "hoststack-cps": "result_rate_unit",
+        "hoststack-rps": "result_rate_unit",
+        "hoststack-bps": "result_bandwidth_unit",
+        "hoststack-lat": "result_latency_unit"
     }
 
     # Latencies.