aboutsummaryrefslogtreecommitdiffstats
path: root/csit.infra.dash/app/cdash/utils
diff options
context:
space:
mode:
Diffstat (limited to 'csit.infra.dash/app/cdash/utils')
-rw-r--r--csit.infra.dash/app/cdash/utils/constants.py5
-rw-r--r--csit.infra.dash/app/cdash/utils/control_panel.py4
-rw-r--r--csit.infra.dash/app/cdash/utils/telemetry_data.py330
-rw-r--r--csit.infra.dash/app/cdash/utils/trigger.py6
-rw-r--r--csit.infra.dash/app/cdash/utils/url_processing.py4
-rw-r--r--csit.infra.dash/app/cdash/utils/utils.py25
6 files changed, 357 insertions, 17 deletions
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 135f06f4d4..95acc07c47 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -63,7 +63,7 @@ class Constants:
# Maximal value of TIME_PERIOD for data read from the parquets in days.
# Do not change without a good reason.
- MAX_TIME_PERIOD = 180
+ MAX_TIME_PERIOD = 150 # 180
# It defines the time period for data read from the parquets in days from
# now back to the past.
@@ -79,6 +79,9 @@ class Constants:
############################################################################
# General, application wide, layout affecting constants.
+ # Add a time delay (in ms) to the spinner being shown
+ SPINNER_DELAY = 500
+
# If True, clear all inputs in control panel when button "ADD SELECTED" is
# pressed.
CLEAR_ALL_INPUTS = False
diff --git a/csit.infra.dash/app/cdash/utils/control_panel.py b/csit.infra.dash/app/cdash/utils/control_panel.py
index 723f404313..a81495e30c 100644
--- a/csit.infra.dash/app/cdash/utils/control_panel.py
+++ b/csit.infra.dash/app/cdash/utils/control_panel.py
@@ -15,7 +15,7 @@
"""
from copy import deepcopy
-
+from typing import Any
class ControlPanel:
"""A class representing the control panel.
@@ -74,7 +74,7 @@ class ControlPanel:
else:
raise KeyError(f"The key {key} is not defined.")
- def get(self, key: str) -> any:
+ def get(self, key: str) -> Any:
"""Returns the value of a key from the Control panel.
:param key: The key which value should be returned.
diff --git a/csit.infra.dash/app/cdash/utils/telemetry_data.py b/csit.infra.dash/app/cdash/utils/telemetry_data.py
new file mode 100644
index 0000000000..e88b8eed06
--- /dev/null
+++ b/csit.infra.dash/app/cdash/utils/telemetry_data.py
@@ -0,0 +1,330 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A module implementing the parsing of OpenMetrics data and elementary
+operations with it.
+"""
+
+
+import pandas as pd
+
+from ..trending.graphs import select_trending_data
+
+
+class TelemetryData:
+ """A class to store and manipulate the telemetry data.
+ """
+
+ def __init__(self, tests: list=list()) -> None:
+ """Initialize the object.
+
+ :param in_data: Input data.
+ :param tests: List of selected tests.
+ :type in_data: pandas.DataFrame
+ :type tests: list
+ """
+
+ self._tests = tests
+ self._data = None
+ self._unique_metrics = list()
+ self._unique_metrics_labels = pd.DataFrame()
+ self._selected_metrics_labels = pd.DataFrame()
+
+ def from_dataframe(self, in_data: pd.DataFrame=pd.DataFrame()) -> None:
+ """Read the input from pandas DataFrame.
+
+ This method must be call at the begining to create all data structures.
+ """
+
+ if in_data.empty:
+ return
+
+ df = pd.DataFrame()
+ metrics = set() # A set of unique metrics
+
+ # Create a dataframe with metrics for selected tests:
+ for itm in self._tests:
+ sel_data = select_trending_data(in_data, itm)
+ if sel_data is not None:
+ sel_data["test_name"] = itm["id"]
+ df = pd.concat([df, sel_data], ignore_index=True, copy=False)
+ # Use only neccessary data:
+ df = df[[
+ "job",
+ "build",
+ "dut_type",
+ "dut_version",
+ "start_time",
+ "passed",
+ "test_name",
+ "test_type",
+ "result_receive_rate_rate_avg",
+ "result_receive_rate_rate_stdev",
+ "result_receive_rate_rate_unit",
+ "result_pdr_lower_rate_value",
+ "result_pdr_lower_rate_unit",
+ "result_ndr_lower_rate_value",
+ "result_ndr_lower_rate_unit",
+ "telemetry"
+ ]]
+ # Transform metrics from strings to dataframes:
+ lst_telemetry = list()
+ for _, row in df.iterrows():
+ d_telemetry = {
+ "metric": list(),
+ "labels": list(), # list of tuple(label, value)
+ "value": list(),
+ "timestamp": list()
+ }
+ if row["telemetry"] is not None and \
+ not isinstance(row["telemetry"], float):
+ for itm in row["telemetry"]:
+ itm_lst = itm.replace("'", "").rsplit(" ", maxsplit=2)
+ metric, labels = itm_lst[0].split("{")
+ d_telemetry["metric"].append(metric)
+ d_telemetry["labels"].append(
+ [tuple(x.split("=")) for x in labels[:-1].split(",")]
+ )
+ d_telemetry["value"].append(itm_lst[1])
+ d_telemetry["timestamp"].append(itm_lst[2])
+ metrics.update(d_telemetry["metric"])
+ lst_telemetry.append(pd.DataFrame(data=d_telemetry))
+ df["telemetry"] = lst_telemetry
+
+ self._data = df
+ self._unique_metrics = sorted(metrics)
+
+ def from_json(self, in_data: dict) -> None:
+ """Read the input data from json.
+ """
+
+ df = pd.read_json(in_data)
+ lst_telemetry = list()
+ metrics = set() # A set of unique metrics
+ for _, row in df.iterrows():
+ telemetry = pd.DataFrame(row["telemetry"])
+ lst_telemetry.append(telemetry)
+ metrics.update(telemetry["metric"].to_list())
+ df["telemetry"] = lst_telemetry
+
+ self._data = df
+ self._unique_metrics = sorted(metrics)
+
+ def from_metrics(self, in_data: set) -> None:
+ """Read only the metrics.
+ """
+ self._unique_metrics = in_data
+
+ def from_metrics_with_labels(self, in_data: dict) -> None:
+ """Read only metrics with labels.
+ """
+ self._unique_metrics_labels = pd.DataFrame.from_dict(in_data)
+
+ def to_json(self) -> str:
+ """Return the data transformed from dataframe to json.
+
+ :returns: Telemetry data transformed to a json structure.
+ :rtype: dict
+ """
+ return self._data.to_json()
+
+ @property
+ def unique_metrics(self) -> list:
+ """Return a set of unique metrics.
+
+ :returns: A set of unique metrics.
+ :rtype: set
+ """
+ return self._unique_metrics
+
+ @property
+ def unique_metrics_with_labels(self) -> dict:
+ """
+ """
+ return self._unique_metrics_labels.to_dict()
+
+ def get_selected_labels(self, metrics: list) -> dict:
+ """Return a dictionary with labels (keys) and all their possible values
+ (values) for all selected 'metrics'.
+
+ :param metrics: List of metrics we are interested in.
+ :type metrics: list
+ :returns: A dictionary with labels and all their possible values.
+ :rtype: dict
+ """
+
+ df_labels = pd.DataFrame()
+ tmp_labels = dict()
+ for _, row in self._data.iterrows():
+ telemetry = row["telemetry"]
+ for itm in metrics:
+ df = telemetry.loc[(telemetry["metric"] == itm)]
+ df_labels = pd.concat(
+ [df_labels, df],
+ ignore_index=True,
+ copy=False
+ )
+ for _, tm in df.iterrows():
+ for label in tm["labels"]:
+ if label[0] not in tmp_labels:
+ tmp_labels[label[0]] = set()
+ tmp_labels[label[0]].add(label[1])
+
+ selected_labels = dict()
+ for key in sorted(tmp_labels):
+ selected_labels[key] = sorted(tmp_labels[key])
+
+ self._unique_metrics_labels = df_labels[["metric", "labels"]].\
+ loc[df_labels[["metric", "labels"]].astype(str).\
+ drop_duplicates().index]
+
+ return selected_labels
+
+ @property
+ def str_metrics(self) -> str:
+ """Returns all unique metrics as a string.
+ """
+ return TelemetryData.metrics_to_str(self._unique_metrics_labels)
+
+ @staticmethod
+ def metrics_to_str(in_data: pd.DataFrame) -> str:
+ """Convert metrics from pandas dataframe to string. Metrics in string
+ are separated by '\n'.
+
+ :param in_data: Metrics to be converted to a string.
+ :type in_data: pandas.DataFrame
+ :returns: Metrics as a string.
+ :rtype: str
+ """
+ metrics = str()
+ for _, row in in_data.iterrows():
+ labels = ','.join([f"{itm[0]}='{itm[1]}'" for itm in row["labels"]])
+ metrics += f"{row['metric']}{{{labels}}}\n"
+ return metrics[:-1]
+
+ def search_unique_metrics(self, string: str) -> list:
+ """Return a list of metrics which name includes the given string.
+
+ :param string: A string which must be in the name of metric.
+ :type string: str
+ :returns: A list of metrics which name includes the given string.
+ :rtype: list
+ """
+ return [itm for itm in self._unique_metrics if string in itm]
+
+ def filter_selected_metrics_by_labels(
+ self,
+ selection: dict
+ ) -> pd.DataFrame:
+ """Filter selected unique metrics by labels and their values.
+
+ :param selection: Labels and their values specified by the user.
+ :type selection: dict
+ :returns: Pandas dataframe with filtered metrics.
+ :rtype: pandas.DataFrame
+ """
+
+ def _is_selected(labels: list, sel: dict) -> bool:
+ """Check if the provided 'labels' are selected by the user.
+
+ :param labels: List of labels and their values from a metric. The
+ items in this lists are two-item-lists whre the first item is
+ the label and the second one is its value.
+ :param sel: User selection. The keys are the selected lables and the
+ values are lists with label values.
+ :type labels: list
+ :type sel: dict
+ :returns: True if the 'labels' are selected by the user.
+ :rtype: bool
+ """
+ passed = list()
+ labels = dict(labels)
+ for key in sel.keys():
+ if key in list(labels.keys()):
+ if sel[key]:
+ passed.append(labels[key] in sel[key])
+ else:
+ passed.append(True)
+ else:
+ passed.append(False)
+ return bool(passed and all(passed))
+
+ self._selected_metrics_labels = pd.DataFrame()
+ for _, row in self._unique_metrics_labels.iterrows():
+ if _is_selected(row["labels"], selection):
+ self._selected_metrics_labels = pd.concat(
+ [self._selected_metrics_labels, row.to_frame().T],
+ ignore_index=True,
+ axis=0,
+ copy=False
+ )
+ return self._selected_metrics_labels
+
+ def select_tm_trending_data(self, selection: dict) -> pd.DataFrame:
+ """Select telemetry data for trending based on user's 'selection'.
+
+ The output dataframe includes these columns:
+ - "job",
+ - "build",
+ - "dut_type",
+ - "dut_version",
+ - "start_time",
+ - "passed",
+ - "test_name",
+ - "test_id",
+ - "test_type",
+ - "result_receive_rate_rate_avg",
+ - "result_receive_rate_rate_stdev",
+ - "result_receive_rate_rate_unit",
+ - "result_pdr_lower_rate_value",
+ - "result_pdr_lower_rate_unit",
+ - "result_ndr_lower_rate_value",
+ - "result_ndr_lower_rate_unit",
+ - "tm_metric",
+ - "tm_value".
+
+ :param selection: User's selection (metrics and labels).
+ :type selection: dict
+ :returns: Dataframe with selected data.
+ :rtype: pandas.DataFrame
+ """
+
+ df = pd.DataFrame()
+
+ if self._data is None:
+ return df
+ if self._data.empty:
+ return df
+ if not selection:
+ return df
+
+ df_sel = pd.DataFrame.from_dict(selection)
+ for _, row in self._data.iterrows():
+ tm_row = row["telemetry"]
+ for _, tm_sel in df_sel.iterrows():
+ df_tmp = tm_row.loc[tm_row["metric"] == tm_sel["metric"]]
+ for _, tm in df_tmp.iterrows():
+ if tm["labels"] == tm_sel["labels"]:
+ labels = ','.join(
+ [f"{itm[0]}='{itm[1]}'" for itm in tm["labels"]]
+ )
+ row["tm_metric"] = f"{tm['metric']}{{{labels}}}"
+ row["tm_value"] = tm["value"]
+ new_row = row.drop(labels=["telemetry", ])
+ df = pd.concat(
+ [df, new_row.to_frame().T],
+ ignore_index=True,
+ axis=0,
+ copy=False
+ )
+ return df
diff --git a/csit.infra.dash/app/cdash/utils/trigger.py b/csit.infra.dash/app/cdash/utils/trigger.py
index 60ef9a3f91..ac303b6b0b 100644
--- a/csit.infra.dash/app/cdash/utils/trigger.py
+++ b/csit.infra.dash/app/cdash/utils/trigger.py
@@ -14,6 +14,8 @@
"""A module implementing the processing of a trigger.
"""
+from typing import Any
+
from json import loads, JSONDecodeError
@@ -51,7 +53,7 @@ class Trigger:
return self._id["type"]
@property
- def idx(self) -> any:
+ def idx(self) -> Any:
return self._id["index"]
@property
@@ -59,5 +61,5 @@ class Trigger:
return self._param
@property
- def value(self) -> any:
+ def value(self) -> Any:
return self._val
diff --git a/csit.infra.dash/app/cdash/utils/url_processing.py b/csit.infra.dash/app/cdash/utils/url_processing.py
index 7f0121ef34..c90c54c41f 100644
--- a/csit.infra.dash/app/cdash/utils/url_processing.py
+++ b/csit.infra.dash/app/cdash/utils/url_processing.py
@@ -69,7 +69,7 @@ def url_decode(url: str) -> dict:
parsed_url = urlparse(url)
except ValueError as err:
logging.warning(f"\nThe url {url} is not valid, ignoring.\n{repr(err)}")
- return None
+ return dict()
if parsed_url.fragment:
try:
@@ -85,7 +85,7 @@ def url_decode(url: str) -> dict:
f"\nEncoded parameters: '{parsed_url.fragment}'"
f"\n{repr(err)}"
)
- return None
+ return dict()
else:
params = None
diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py
index 8584dee067..63e13ce141 100644
--- a/csit.infra.dash/app/cdash/utils/utils.py
+++ b/csit.infra.dash/app/cdash/utils/utils.py
@@ -346,29 +346,34 @@ def set_job_params(df: pd.DataFrame, job: str) -> dict:
}
-def get_list_group_items(tests: list) -> list:
- """Generate list of ListGroupItems with checkboxes with selected tests.
-
- :param tests: List of tests to be displayed in the ListGroup.
- :type tests: list
- :returns: List of ListGroupItems with checkboxes with selected tests.
+def get_list_group_items(items: list, type: str, colorize: bool=True) -> list:
+ """Generate list of ListGroupItems with checkboxes with selected items.
+
+ :param items: List of items to be displayed in the ListGroup.
+ :param type: The type part of an element ID.
+ :param colorize: If True, the color of labels is set, otherwise the default
+ color is used.
+ :type items: list
+ :type type: str
+ :type colorize: bool
+ :returns: List of ListGroupItems with checkboxes with selected items.
:rtype: list
"""
return [
dbc.ListGroupItem(
children=[
dbc.Checkbox(
- id={"type": "sel-cl", "index": i},
- label=l["id"],
+ id={"type": type, "index": i},
+ label=l["id"] if isinstance(l, dict) else l,
value=False,
label_class_name="m-0 p-0",
label_style={
"font-size": ".875em",
- "color": get_color(i)
+ "color": get_color(i) if colorize else "#55595c"
},
class_name="info"
)
],
class_name="p-0"
- ) for i, l in enumerate(tests)
+ ) for i, l in enumerate(items)
]