aboutsummaryrefslogtreecommitdiffstats
path: root/csit.infra.dash
diff options
context:
space:
mode:
Diffstat (limited to 'csit.infra.dash')
-rw-r--r--csit.infra.dash/app/cdash/trending/layout.py568
-rw-r--r--csit.infra.dash/app/cdash/utils/telemetry_data.py75
2 files changed, 290 insertions, 353 deletions
diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py
index 38381459b8..74c39eb8c0 100644
--- a/csit.infra.dash/app/cdash/trending/layout.py
+++ b/csit.infra.dash/app/cdash/trending/layout.py
@@ -279,15 +279,8 @@ class Layout:
)
else:
return html.Div(
- id="div-main-error",
- children=[
- dbc.Alert(
- [
- "An Error Occured"
- ],
- color="danger"
- )
- ]
+ dbc.Alert("An Error Occured", color="danger"),
+ id="div-main-error"
)
def _add_navbar(self):
@@ -297,17 +290,15 @@ class Layout:
:rtype: dbc.NavbarSimple
"""
return dbc.NavbarSimple(
- id="navbarsimple-main",
- children=[
- dbc.NavItem(
- dbc.NavLink(
- C.TREND_TITLE,
- disabled=True,
- external_link=True,
- href="#"
- )
+ dbc.NavItem(
+ dbc.NavLink(
+ C.TREND_TITLE,
+ disabled=True,
+ external_link=True,
+ href="#"
)
- ],
+ ),
+ id="navbarsimple-main",
brand=C.BRAND,
brand_href="/",
brand_external_link=True,
@@ -321,12 +312,7 @@ class Layout:
:returns: Column with the control panel.
:rtype: dbc.Col
"""
- return dbc.Col([
- html.Div(
- children=self._add_ctrl_panel(),
- className="sticky-top"
- )
- ])
+ return dbc.Col(html.Div(self._add_ctrl_panel(), className="sticky-top"))
def _add_ctrl_panel(self) -> list:
"""Add control panel.
@@ -336,316 +322,226 @@ class Layout:
"""
return [
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-dut",
- "DUT"
- )
- ),
- dbc.Select(
- id={"type": "ctrl-dd", "index": "dut"},
- placeholder="Select a Device under Test...",
- options=sorted(
- [
- {"label": k, "value": k} \
- for k in self._spec_tbs.keys()
- ],
- key=lambda d: d["label"]
- )
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(
+ show_tooltip(self._tooltips, "help-dut", "DUT")
+ ),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "dut"},
+ placeholder="Select a Device under Test...",
+ options=sorted(
+ [
+ {"label": k, "value": k} \
+ for k in self._spec_tbs.keys()
+ ],
+ key=lambda d: d["label"]
)
- ],
- size="sm"
- )
- ]
+ )
+ ],
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-infra",
- "Infra"
- )
- ),
- dbc.Select(
- id={"type": "ctrl-dd", "index": "phy"},
- placeholder=\
- "Select a Physical Test Bed Topology..."
- )
- ],
- size="sm"
- )
- ]
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(
+ show_tooltip(self._tooltips, "help-infra", "Infra")
+ ),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "phy"},
+ placeholder="Select a Physical Test Bed Topology..."
+ )
+ ],
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-area",
- "Area"
- )
- ),
- dbc.Select(
- id={"type": "ctrl-dd", "index": "area"},
- placeholder="Select an Area..."
- )
- ],
- size="sm"
- )
- ]
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(
+ show_tooltip(self._tooltips, "help-area", "Area")
+ ),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "area"},
+ placeholder="Select an Area..."
+ )
+ ],
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-test",
- "Test"
- )
- ),
- dbc.Select(
- id={"type": "ctrl-dd", "index": "test"},
- placeholder="Select a Test..."
- )
- ],
- size="sm"
- )
- ]
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(
+ show_tooltip(self._tooltips, "help-test", "Test")
+ ),
+ dbc.Select(
+ id={"type": "ctrl-dd", "index": "test"},
+ placeholder="Select a Test..."
+ )
+ ],
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-framesize",
- "Frame Size"
- )
- ),
- dbc.Col(
- children=[
- dbc.Checklist(
- id={
- "type": "ctrl-cl",
- "index": "frmsize-all"
- },
- options=C.CL_ALL_DISABLED,
- inline=True,
- class_name="ms-2"
- )
- ],
- width=2
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(show_tooltip(
+ self._tooltips,
+ "help-framesize",
+ "Frame Size"
+ )),
+ dbc.Col(
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "frmsize-all"},
+ options=C.CL_ALL_DISABLED,
+ inline=True,
+ class_name="ms-2"
),
- dbc.Col(
- children=[
- dbc.Checklist(
- id={
- "type": "ctrl-cl",
- "index": "frmsize"
- },
- inline=True
- )
- ]
+ width=2
+ ),
+ dbc.Col(
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "frmsize"},
+ inline=True
)
- ],
- style={"align-items": "center"},
- size="sm"
- )
- ]
+ )
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-cores",
- "Number of Cores"
- )
- ),
- dbc.Col(
- children=[
- dbc.Checklist(
- id={
- "type": "ctrl-cl",
- "index": "core-all"
- },
- options=C.CL_ALL_DISABLED,
- inline=True,
- class_name="ms-2"
- )
- ],
- width=2
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(show_tooltip(
+ self._tooltips,
+ "help-cores",
+ "Number of Cores"
+ )),
+ dbc.Col(
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "core-all"},
+ options=C.CL_ALL_DISABLED,
+ inline=True,
+ class_name="ms-2"
),
- dbc.Col(
- children=[
- dbc.Checklist(
- id={
- "type": "ctrl-cl",
- "index": "core"
- },
- inline=True
- )
- ]
+ width=2
+ ),
+ dbc.Col(
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "core"},
+ inline=True
)
- ],
- style={"align-items": "center"},
- size="sm"
- )
- ]
+ )
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-ttype",
- "Test Type"
- )
- ),
- dbc.Col(
- children=[
- dbc.Checklist(
- id={
- "type": "ctrl-cl",
- "index": "tsttype-all"
- },
- options=C.CL_ALL_DISABLED,
- inline=True,
- class_name="ms-2"
- )
- ],
- width=2
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(show_tooltip(
+ self._tooltips,
+ "help-ttype",
+ "Test Type"
+ )),
+ dbc.Col(
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "tsttype-all"},
+ options=C.CL_ALL_DISABLED,
+ inline=True,
+ class_name="ms-2"
),
- dbc.Col(
- children=[
- dbc.Checklist(
- id={
- "type": "ctrl-cl",
- "index": "tsttype"
- },
- inline=True
- )
- ]
+ width=2
+ ),
+ dbc.Col(
+ dbc.Checklist(
+ id={"type": "ctrl-cl", "index": "tsttype"},
+ inline=True
)
- ],
- style={"align-items": "center"},
- size="sm"
- )
- ]
+ )
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.InputGroup(
- [
- dbc.InputGroupText(
- children=show_tooltip(
- self._tooltips,
- "help-normalize",
- "Normalization"
- )
- ),
- dbc.Col(
- children=[
- dbc.Checklist(
- id="normalize",
- options=[{
- "value": "normalize",
- "label": (
- "Normalize to CPU frequency "
- "2GHz"
- )
- }],
- value=[],
- inline=True,
- class_name="ms-2"
- )
- ]
- )
- ],
- style={"align-items": "center"},
- size="sm"
- )
- ]
+ dbc.InputGroup(
+ [
+ dbc.InputGroupText(show_tooltip(
+ self._tooltips,
+ "help-normalize",
+ "Normalization"
+ )),
+ dbc.Col(dbc.Checklist(
+ id="normalize",
+ options=[{
+ "value": "normalize",
+ "label": "Normalize to CPU frequency 2GHz"
+ }],
+ value=[],
+ inline=True,
+ class_name="ms-2"
+ ))
+ ],
+ style={"align-items": "center"},
+ size="sm"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.Button(
- id={"type": "ctrl-btn", "index": "add-test"},
- children="Add Selected",
- color="info"
- )
- ]
+ dbc.Button(
+ id={"type": "ctrl-btn", "index": "add-test"},
+ children="Add Selected",
+ color="info"
+ ),
+ class_name="g-0 p-1"
),
dbc.Row(
+ dbc.ListGroup(
+ class_name="overflow-auto p-0",
+ id="lg-selected",
+ children=[],
+ style={"max-height": "20em"},
+ flush=True
+ ),
id="row-card-sel-tests",
class_name="g-0 p-1",
style=C.STYLE_DISABLED,
- children=[
- dbc.ListGroup(
- class_name="overflow-auto p-0",
- id="lg-selected",
- children=[],
- style={"max-height": "20em"},
- flush=True
- )
- ]
),
dbc.Row(
+ dbc.ButtonGroup([
+ dbc.Button(
+ "Remove Selected",
+ id={"type": "ctrl-btn", "index": "rm-test"},
+ class_name="w-100",
+ color="info",
+ disabled=False
+ ),
+ dbc.Button(
+ "Remove All",
+ id={"type": "ctrl-btn", "index": "rm-test-all"},
+ class_name="w-100",
+ color="info",
+ disabled=False
+ )
+ ]),
id="row-btns-sel-tests",
class_name="g-0 p-1",
style=C.STYLE_DISABLED,
- children=[
- dbc.ButtonGroup(
- children=[
- dbc.Button(
- id={"type": "ctrl-btn", "index": "rm-test"},
- children="Remove Selected",
- class_name="w-100",
- color="info",
- disabled=False
- ),
- dbc.Button(
- id={"type": "ctrl-btn", "index": "rm-test-all"},
- children="Remove All",
- class_name="w-100",
- color="info",
- disabled=False
- )
- ]
- )
- ]
),
dbc.Stack(
- id="row-btns-add-tm",
- class_name="g-0 p-1",
- style=C.STYLE_DISABLED,
- gap=2,
- children=[
+ [
dbc.Button(
"Add Telemetry Panel",
id={"type": "telemetry-btn", "index": "open"},
@@ -662,7 +558,11 @@ class Layout:
is_open=False,
scrollable=True
)
- ]
+ ],
+ id="row-btns-add-tm",
+ class_name="g-0 p-1",
+ style=C.STYLE_DISABLED,
+ gap=2
)
]
@@ -980,30 +880,38 @@ class Layout:
"""
return [
dbc.Row(
+ "Add content here.",
id={"type": "tm-container", "index": 0},
- class_name="g-0 p-1",
- children=["Add content here."]
+ class_name="g-0 p-1"
),
dbc.Row(
- class_name="g-0 p-2",
- children=[
- dbc.Checkbox(
- id={"type": "cb-all-in-one", "index": 0},
- label="All Metrics in one Graph"
+ [
+ dbc.Col(
+ dbc.Checkbox(
+ id={"type": "cb-all-in-one", "index": 0},
+ label="All Metrics in one Graph"
+ ),
+ width=6
),
- ]
+ dbc.Col(
+ dbc.Checkbox(
+ id={"type": "cb-ignore-host", "index": 0},
+ label="Ignore Host"
+ ),
+ width=6
+ )
+ ],
+ class_name="g-0 p-2"
),
dbc.Row(
- class_name="g-0 p-1",
- children=[
- dbc.Textarea(
- id={"type": "tm-list-metrics", "index": 0},
- rows=20,
- size="sm",
- wrap="off",
- readonly=True
- )
- ]
+ dbc.Textarea(
+ id={"type": "tm-list-metrics", "index": 0},
+ rows=20,
+ size="sm",
+ wrap="off",
+ readonly=True
+ ),
+ class_name="g-0 p-1"
)
]
@@ -1056,6 +964,7 @@ class Layout:
State("store", "data"),
State({"type": "sel-cl", "index": ALL}, "value"),
State({"type": "cb-all-in-one", "index": ALL}, "value"),
+ State({"type": "cb-ignore-host", "index": ALL}, "value"),
State({"type": "telemetry-search-out", "index": ALL}, "children"),
State({"type": "plot-mod-telemetry", "index": ALL}, "is_open"),
State({"type": "telemetry-btn", "index": ALL}, "disabled"),
@@ -1080,6 +989,7 @@ class Layout:
store: dict,
lst_sel: list,
all_in_one: list,
+ ignore_host: list,
search_out: list,
is_open: list,
tm_btns_disabled: list,
@@ -1102,6 +1012,7 @@ class Layout:
"selected-metrics": dict(),
"telemetry-panels": list(),
"telemetry-all-in-one": list(),
+ "telemetry-ignore-host": list(),
"telemetry-graphs": list(),
"url": str()
}
@@ -1115,6 +1026,7 @@ class Layout:
tm_user = store["selected-metrics"]
tm_panels = store["telemetry-panels"]
tm_all_in_one = store["telemetry-all-in-one"]
+ tm_ignore_host = store["telemetry-ignore-host"]
plotting_area_telemetry = no_update
on_draw = [False, False] # 0 --> trending, 1 --> telemetry
@@ -1154,10 +1066,12 @@ class Layout:
store_sel = literal_eval(url_params["store_sel"][0])
normalize = literal_eval(url_params["norm"][0])
telemetry = literal_eval(url_params["telemetry"][0])
- tm_all_in_one = literal_eval(url_params["all-in-one"][0])
+ url_p = url_params.get("all-in-one", ["[[None]]"])
+ tm_all_in_one = literal_eval(url_p[0])
+ url_p = url_params.get("ignore-host", ["[[None]]"])
+ tm_ignore_host = literal_eval(url_p[0])
if not isinstance(telemetry, list):
telemetry = [telemetry, ]
- tm_all_in_one = [tm_all_in_one, ]
except (KeyError, IndexError, AttributeError, ValueError):
pass
if store_sel:
@@ -1359,6 +1273,7 @@ class Layout:
elif trigger.type == "ctrl-btn":
tm_panels = list()
tm_all_in_one = list()
+ tm_ignore_host = list()
store["trending-graphs"] = None
store["telemetry-graphs"] = list()
# on_draw[0] = True
@@ -1446,6 +1361,7 @@ class Layout:
tm.from_json(tm_data)
tm_panels.append(tm_user["selected_metrics_with_labels"])
tm_all_in_one.append(all_in_one)
+ tm_ignore_host.append(ignore_host)
is_open = (False, False)
tm_btns_disabled[1], tm_btns_disabled[5] = True, True
on_draw = [True, True]
@@ -1455,6 +1371,7 @@ class Layout:
elif trigger.idx == "rm-all":
tm_panels = list()
tm_all_in_one = list()
+ tm_ignore_host = list()
tm_user = None
is_open = (False, False)
tm_btns_disabled[1], tm_btns_disabled[5] = True, True
@@ -1501,6 +1418,7 @@ class Layout:
elif trigger.type == "tm-btn-remove":
del tm_panels[trigger.idx]
del tm_all_in_one[trigger.idx]
+ del tm_ignore_host[trigger.idx]
del store["telemetry-graphs"][trigger.idx]
tm.from_json(tm_data)
on_draw = [True, True]
@@ -1512,6 +1430,7 @@ class Layout:
if tm_panels:
new_url_params["telemetry"] = tm_panels
new_url_params["all-in-one"] = tm_all_in_one
+ new_url_params["ignore-host"] = tm_ignore_host
if on_draw[0]: # Trending
if store_sel:
@@ -1538,7 +1457,10 @@ class Layout:
elif on_draw[1] and (end_idx >= start_idx):
for idx in range(start_idx, end_idx):
store["telemetry-graphs"].append(graph_tm_trending(
- tm.select_tm_trending_data(tm_panels[idx]),
+ tm.select_tm_trending_data(
+ tm_panels[idx],
+ ignore_host=bool(tm_ignore_host[idx][0])
+ ),
self._graph_layout,
bool(tm_all_in_one[idx][0])
))
@@ -1561,6 +1483,7 @@ class Layout:
store_sel = list()
tm_panels = list()
tm_all_in_one = list()
+ tm_ignore_host = list()
tm_user = None
else:
plotting_area_trending = no_update
@@ -1577,6 +1500,7 @@ class Layout:
store["selected-metrics"] = tm_user
store["telemetry-panels"] = tm_panels
store["telemetry-all-in-one"] = tm_all_in_one
+ store["telemetry-ignore-host"] = tm_ignore_host
ret_val = [
store,
plotting_area_trending,
diff --git a/csit.infra.dash/app/cdash/utils/telemetry_data.py b/csit.infra.dash/app/cdash/utils/telemetry_data.py
index 9c2e45f9a1..80187967fa 100644
--- a/csit.infra.dash/app/cdash/utils/telemetry_data.py
+++ b/csit.infra.dash/app/cdash/utils/telemetry_data.py
@@ -52,15 +52,17 @@ class TelemetryData:
if in_data.empty:
return
- df = pd.DataFrame()
metrics = set() # A set of unique metrics
# Create a dataframe with metrics for selected tests:
+ lst_items = list()
for itm in self._tests:
sel_data = select_trending_data(in_data, itm)
if sel_data is not None:
sel_data["test_name"] = itm["id"]
- df = pd.concat([df, sel_data], ignore_index=True, copy=False)
+ lst_items.append(sel_data)
+ df = pd.concat(lst_items, ignore_index=True, copy=False)
+
# Use only neccessary data:
df = df[[
"job",
@@ -182,23 +184,20 @@ class TelemetryData:
:rtype: dict
"""
- df_labels = pd.DataFrame()
+ lst_labels = list()
tmp_labels = dict()
for _, row in self._data.iterrows():
telemetry = row["telemetry"]
for itm in metrics:
df = telemetry.loc[(telemetry["metric"] == itm)]
- df_labels = pd.concat(
- [df_labels, df],
- ignore_index=True,
- copy=False
- )
+ lst_labels.append(df)
for _, tm in df.iterrows():
for label in tm["labels"]:
if label[0] not in tmp_labels:
tmp_labels[label[0]] = set()
tmp_labels[label[0]].add(label[1])
+ df_labels = pd.concat(lst_labels, ignore_index=True, copy=False)
selected_labels = dict()
for key in sorted(tmp_labels):
selected_labels[key] = sorted(tmp_labels[key])
@@ -279,17 +278,19 @@ class TelemetryData:
return bool(passed and all(passed))
self._selected_metrics_labels = pd.DataFrame()
+ lst_items = list()
for _, row in self._unique_metrics_labels.iterrows():
if _is_selected(row["labels"], selection):
- self._selected_metrics_labels = pd.concat(
- [self._selected_metrics_labels, row.to_frame().T],
- ignore_index=True,
- axis=0,
- copy=False
- )
+ lst_items.append(row.to_frame().T)
+ self._selected_metrics_labels = \
+ pd.concat(lst_items, ignore_index=True, axis=0, copy=False)
return self._selected_metrics_labels
- def select_tm_trending_data(self, selection: dict) -> pd.DataFrame:
+ def select_tm_trending_data(
+ self,
+ selection: dict,
+ ignore_host: bool = False
+ ) -> pd.DataFrame:
"""Select telemetry data for trending based on user's 'selection'.
The output dataframe includes these columns:
@@ -313,37 +314,49 @@ class TelemetryData:
- "tm_value".
:param selection: User's selection (metrics and labels).
+ :param ignore_host: Ignore 'hostname' and 'hook' labels in metrics.
:type selection: dict
+ :type ignore_host: bool
:returns: Dataframe with selected data.
:rtype: pandas.DataFrame
"""
- df = pd.DataFrame()
-
if self._data is None:
- return df
+ return pd.DataFrame()
if self._data.empty:
- return df
+ return pd.DataFrame()
if not selection:
- return df
+ return pd.DataFrame()
df_sel = pd.DataFrame.from_dict(selection)
+ lst_rows = list()
for _, row in self._data.iterrows():
tm_row = row["telemetry"]
for _, tm_sel in df_sel.iterrows():
df_tmp = tm_row.loc[tm_row["metric"] == tm_sel["metric"]]
for _, tm in df_tmp.iterrows():
- if tm["labels"] == tm_sel["labels"]:
- labels = ','.join(
- [f"{itm[0]}='{itm[1]}'" for itm in tm["labels"]]
- )
+ do_it = False
+ if ignore_host:
+ if tm["labels"][2:] == tm_sel["labels"][2:]:
+ labels = ','.join(
+ [f"{i[0]}='{i[1]}'" for i in tm["labels"][2:]]
+ )
+ do_it = True
+ else:
+ if tm["labels"] == tm_sel["labels"]:
+ labels = ','.join(
+ [f"{i[0]}='{i[1]}'" for i in tm["labels"]]
+ )
+ do_it = True
+ if do_it:
row["tm_metric"] = f"{tm['metric']}{{{labels}}}"
row["tm_value"] = tm["value"]
- new_row = row.drop(labels=["telemetry", ])
- df = pd.concat(
- [df, new_row.to_frame().T],
- ignore_index=True,
- axis=0,
- copy=False
+ lst_rows.append(
+ row.drop(labels=["telemetry", ]).to_frame().T
)
- return df
+ if lst_rows:
+ return pd.concat(
+ lst_rows, ignore_index=True, axis=0, copy=False
+ ).drop_duplicates()
+ else:
+ return pd.DataFrame()