aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2023-10-10 08:14:18 +0000
committerTibor Frank <tifrank@cisco.com>2023-10-11 07:38:18 +0000
commit86ec9713ecde3d92ec4585d487ddfac5077428c8 (patch)
tree2918a347f80ccf9d628f9132d7d1a454043fb9ae
parentc3ce59c652caaee06b3090f387f8826c15102703 (diff)
C-Dash: Add bandwidth to trending graphs
- NDRPDR tests only - MRR to be done when data is in parquets Change-Id: If2b1f76d5e2dc6f3a480eeb0950e7ab9550f8901 Signed-off-by: Tibor Frank <tifrank@cisco.com>
-rw-r--r--csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_devicebin0 -> 5373 bytes
-rw-r--r--csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdrbin0 -> 11868 bytes
-rw-r--r--csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststackbin0 -> 7882 bytes
-rw-r--r--csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_mrr (renamed from csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_mrr)bin7919 -> 7919 bytes
-rw-r--r--csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_ndrpdr (renamed from csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_ndrpdr)bin13081 -> 13081 bytes
-rw-r--r--csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdrbin13999 -> 16091 bytes
-rw-r--r--csit.infra.dash/app/cdash/data/data.py23
-rw-r--r--csit.infra.dash/app/cdash/data/data.yaml7
-rw-r--r--csit.infra.dash/app/cdash/trending/graphs.py24
-rw-r--r--csit.infra.dash/app/cdash/trending/layout.py27
-rw-r--r--csit.infra.dash/app/cdash/trending/layout.yaml40
-rw-r--r--csit.infra.dash/app/cdash/utils/constants.py6
12 files changed, 122 insertions, 5 deletions
diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_device b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_device
new file mode 100644
index 0000000000..f619ce8a8e
--- /dev/null
+++ b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_device
Binary files differ
diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdr
new file mode 100644
index 0000000000..06bc618bea
--- /dev/null
+++ b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2302_ndrpdr
Binary files differ
diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststack b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststack
new file mode 100644
index 0000000000..993d16c18c
--- /dev/null
+++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_hoststack
Binary files differ
diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_mrr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_mrr
index 96832850b1..96832850b1 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_mrr
+++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_mrr
Binary files differ
diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_ndrpdr
index 2291bb8349..2291bb8349 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2210_ndrpdr
+++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2302_ndrpdr
Binary files differ
diff --git a/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr
index 17dc30c3be..3f8b85c66e 100644
--- a/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr
+++ b/csit.infra.dash/app/cdash/data/_metadata/trending_ndrpdr
Binary files differ
diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py
index 2bf3649778..783ebe25ff 100644
--- a/csit.infra.dash/app/cdash/data/data.py
+++ b/csit.infra.dash/app/cdash/data/data.py
@@ -30,6 +30,12 @@ from pyarrow.lib import ArrowInvalid, ArrowNotImplementedError
from ..utils.constants import Constants as C
+# If True, pyarrow.Schema is generated. See also condition in the method
+# _write_parquet_schema.
+# To generate schema, select only one data set in data.yaml file.
+GENERATE_SCHEMA = False
+
+
class Data:
"""Gets the data from parquets and stores it for further use by dash
applications.
@@ -212,7 +218,10 @@ class Data:
for itm in df:
try:
# Specify the condition or remove it:
- if pd.api.types.is_string_dtype(itm["result_rate_unit"]):
+ if all((
+ pd.api.types.is_string_dtype(itm["<column_name>"]),
+ pd.api.types.is_string_dtype(itm["telemetry"][0])
+ )):
print(pa.Schema.from_pandas(itm))
pa.parquet.write_metadata(
pa.Schema.from_pandas(itm),
@@ -357,6 +366,18 @@ class Data:
time_period = days
else:
time_period = None
+
+ if GENERATE_SCHEMA:
+ # Generate schema:
+ Data._write_parquet_schema(
+ path=data_set["path"],
+ partition_filter=partition_filter,
+ columns=data_set.get("columns", None),
+ days=time_period
+ )
+ return
+
+ # Read data:
data = Data._create_dataframe_from_parquet(
path=data_set["path"],
partition_filter=partition_filter,
diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml
index 427821d33c..f8c00731cf 100644
--- a/csit.infra.dash/app/cdash/data/data.yaml
+++ b/csit.infra.dash/app/cdash/data/data.yaml
@@ -26,6 +26,9 @@
- result_receive_rate_rate_avg
- result_receive_rate_rate_stdev
- result_receive_rate_rate_unit
+ # - result_receive_rate_bandwidth_avg
+ # - result_receive_rate_bandwidth_stdev
+ # - result_receive_rate_bandwidth_unit
- telemetry
- data_type: trending
partition: test_type
@@ -46,6 +49,10 @@
- result_pdr_lower_rate_value
- result_ndr_lower_rate_unit
- result_ndr_lower_rate_value
+ - result_pdr_lower_bandwidth_unit
+ - result_pdr_lower_bandwidth_value
+ - result_ndr_lower_bandwidth_unit
+ - result_ndr_lower_bandwidth_value
- result_latency_reverse_pdr_90_hdrh
- result_latency_reverse_pdr_50_hdrh
- result_latency_reverse_pdr_10_hdrh
diff --git a/csit.infra.dash/app/cdash/trending/graphs.py b/csit.infra.dash/app/cdash/trending/graphs.py
index a1d61a07fc..fcfee26e3f 100644
--- a/csit.infra.dash/app/cdash/trending/graphs.py
+++ b/csit.infra.dash/app/cdash/trending/graphs.py
@@ -162,6 +162,12 @@ def graph_trending(
f"latency [{row[C.UNIT['hoststack-latency']]}]: "
f"{row[C.VALUE['hoststack-latency']]:,.0f}<br>"
)
+ elif ttype in ("ndr", "pdr"): # Add mrr
+ test_type = f"{ttype}-bandwidth"
+ add_info = (
+ f"bandwidth [{row[C.UNIT[test_type]]}]: "
+ f"{row[C.VALUE[test_type]]:,.0f}<br>"
+ )
else:
add_info = str()
hover_itm = hover_itm.replace(
@@ -295,6 +301,7 @@ def graph_trending(
fig_tput = None
fig_lat = None
+ fig_band = None
y_units = set()
for idx, itm in enumerate(sel):
df = select_trending_data(data, itm)
@@ -326,6 +333,19 @@ def graph_trending(
fig_tput = go.Figure()
fig_tput.add_traces(traces)
+ if ttype in ("ndr", "pdr"): # Add mrr
+ traces, _ = _generate_trending_traces(
+ f"{ttype}-bandwidth",
+ itm["id"],
+ df,
+ get_color(idx),
+ norm_factor
+ )
+ if traces:
+ if not fig_band:
+ fig_band = go.Figure()
+ fig_band.add_traces(traces)
+
if itm["testtype"] == "pdr":
traces, _ = _generate_trending_traces(
"latency",
@@ -346,10 +366,12 @@ def graph_trending(
fig_layout["yaxis"]["title"] = \
f"Throughput [{'|'.join(sorted(y_units))}]"
fig_tput.update_layout(fig_layout)
+ if fig_band:
+ fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
if fig_lat:
fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
- return fig_tput, fig_lat
+ return fig_tput, fig_band, fig_lat
def graph_tm_trending(
diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index 84a68f5050..d08f911036 100644
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -265,7 +265,7 @@ class Layout:
dbc.Offcanvas(
class_name="w-50",
id="offcanvas-metadata",
- title="Throughput And Latency",
+ title="Detailed Information",
placement="end",
is_open=False,
children=[
@@ -646,9 +646,21 @@ class Layout:
tab_items.append(
dbc.Tab(
children=dcc.Graph(
- id={"type": "graph", "index": "lat"},
+ id={"type": "graph", "index": "bandwidth"},
figure=graphs[1]
),
+ label="Bandwidth",
+ tab_id="tab-bandwidth"
+ )
+ )
+
+ if graphs[2]:
+ tab_items.append(
+ dbc.Tab(
+ children=dcc.Graph(
+ id={"type": "graph", "index": "lat"},
+ figure=graphs[2]
+ ),
label="Latency",
tab_id="tab-lat"
)
@@ -1690,7 +1702,14 @@ class Layout:
trigger = Trigger(callback_context.triggered)
try:
- idx = 0 if trigger.idx == "tput" else 1
+ if trigger.idx == "tput":
+ idx = 0
+ elif trigger.idx == "bandwidth":
+ idx = 1
+ elif trigger.idx == "lat":
+ idx = 2
+ else:
+ raise PreventUpdate
graph_data = graph_data[idx]["points"][0]
except (IndexError, KeyError, ValueError, TypeError):
raise PreventUpdate
@@ -1721,6 +1740,8 @@ class Layout:
if trigger.idx == "tput":
title = "Throughput"
+ elif trigger.idx == "bandwidth":
+ title = "Bandwidth"
elif trigger.idx == "lat":
title = "Latency"
hdrh_data = graph_data.get("customdata", None)
diff --git a/csit.infra.dash/app/cdash/trending/layout.yaml b/csit.infra.dash/app/cdash/trending/layout.yaml
index 5e41b03118..2fcb206153 100644
--- a/csit.infra.dash/app/cdash/trending/layout.yaml
+++ b/csit.infra.dash/app/cdash/trending/layout.yaml
@@ -38,6 +38,46 @@ plot-trending-tput:
hoverlabel:
namelength: -1
+plot-trending-bandwidth:
+ autosize: True
+ showlegend: False
+ yaxis:
+ showticklabels: True
+ tickformat: ".3s"
+ title: "Bandwidth [bps]"
+ hoverformat: ".5s"
+ gridcolor: "rgb(238, 238, 238)"
+ linecolor: "rgb(238, 238, 238)"
+ showline: True
+ zeroline: False
+ tickcolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ showgrid: True
+ xaxis:
+ title: 'Date [MMDD]'
+ type: "date"
+ autorange: True
+ fixedrange: False
+ showgrid: True
+ gridcolor: "rgb(238, 238, 238)"
+ showline: True
+ linecolor: "rgb(238, 238, 238)"
+ zeroline: False
+ linewidth: 1
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ tickmode: "auto"
+ tickformat: "%m%d"
+ margin:
+ r: 20
+ b: 0
+ t: 5
+ l: 70
+ paper_bgcolor: "#fff"
+ plot_bgcolor: "#fff"
+ hoverlabel:
+ namelength: -1
+
plot-trending-lat:
autosize: True
showlegend: False
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 34892e79bc..fb887fe79e 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -229,6 +229,9 @@ class Constants:
"mrr": "result_receive_rate_rate_avg",
"ndr": "result_ndr_lower_rate_value",
"pdr": "result_pdr_lower_rate_value",
+ "mrr-bandwidth": "result_receive_rate_bandwidth_avg",
+ "ndr-bandwidth": "result_ndr_lower_bandwidth_value",
+ "pdr-bandwidth": "result_pdr_lower_bandwidth_value",
"latency": "result_latency_forward_pdr_50_avg",
"hoststack-cps": "result_rate_value",
"hoststack-rps": "result_rate_value",
@@ -251,6 +254,9 @@ class Constants:
"mrr": "result_receive_rate_rate_unit",
"ndr": "result_ndr_lower_rate_unit",
"pdr": "result_pdr_lower_rate_unit",
+ "mrr-bandwidth": "result_receive_rate_bandwidth_unit",
+ "ndr-bandwidth": "result_ndr_lower_bandwidth_unit",
+ "pdr-bandwidth": "result_pdr_lower_bandwidth_unit",
"latency": "result_latency_forward_pdr_50_unit",
"hoststack-cps": "result_rate_unit",
"hoststack-rps": "result_rate_unit",