diff options
Diffstat (limited to 'csit.infra.dash/app/cdash/utils')
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/constants.py | 5 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/control_panel.py | 4 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/telemetry_data.py | 330 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/trigger.py | 6 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/url_processing.py | 4 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/utils.py | 25 |
6 files changed, 357 insertions, 17 deletions
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py index 135f06f4d4..95acc07c47 100644 --- a/csit.infra.dash/app/cdash/utils/constants.py +++ b/csit.infra.dash/app/cdash/utils/constants.py @@ -63,7 +63,7 @@ class Constants: # Maximal value of TIME_PERIOD for data read from the parquets in days. # Do not change without a good reason. - MAX_TIME_PERIOD = 180 + MAX_TIME_PERIOD = 150 # 180 # It defines the time period for data read from the parquets in days from # now back to the past. @@ -79,6 +79,9 @@ class Constants: ############################################################################ # General, application wide, layout affecting constants. + # Add a time delay (in ms) to the spinner being shown + SPINNER_DELAY = 500 + # If True, clear all inputs in control panel when button "ADD SELECTED" is # pressed. CLEAR_ALL_INPUTS = False diff --git a/csit.infra.dash/app/cdash/utils/control_panel.py b/csit.infra.dash/app/cdash/utils/control_panel.py index 723f404313..a81495e30c 100644 --- a/csit.infra.dash/app/cdash/utils/control_panel.py +++ b/csit.infra.dash/app/cdash/utils/control_panel.py @@ -15,7 +15,7 @@ """ from copy import deepcopy - +from typing import Any class ControlPanel: """A class representing the control panel. @@ -74,7 +74,7 @@ class ControlPanel: else: raise KeyError(f"The key {key} is not defined.") - def get(self, key: str) -> any: + def get(self, key: str) -> Any: """Returns the value of a key from the Control panel. :param key: The key which value should be returned. diff --git a/csit.infra.dash/app/cdash/utils/telemetry_data.py b/csit.infra.dash/app/cdash/utils/telemetry_data.py new file mode 100644 index 0000000000..e88b8eed06 --- /dev/null +++ b/csit.infra.dash/app/cdash/utils/telemetry_data.py @@ -0,0 +1,330 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module implementing the parsing of OpenMetrics data and elementary +operations with it. +""" + + +import pandas as pd + +from ..trending.graphs import select_trending_data + + +class TelemetryData: + """A class to store and manipulate the telemetry data. + """ + + def __init__(self, tests: list=list()) -> None: + """Initialize the object. + + :param in_data: Input data. + :param tests: List of selected tests. + :type in_data: pandas.DataFrame + :type tests: list + """ + + self._tests = tests + self._data = None + self._unique_metrics = list() + self._unique_metrics_labels = pd.DataFrame() + self._selected_metrics_labels = pd.DataFrame() + + def from_dataframe(self, in_data: pd.DataFrame=pd.DataFrame()) -> None: + """Read the input from pandas DataFrame. + + This method must be call at the begining to create all data structures. + """ + + if in_data.empty: + return + + df = pd.DataFrame() + metrics = set() # A set of unique metrics + + # Create a dataframe with metrics for selected tests: + for itm in self._tests: + sel_data = select_trending_data(in_data, itm) + if sel_data is not None: + sel_data["test_name"] = itm["id"] + df = pd.concat([df, sel_data], ignore_index=True, copy=False) + # Use only neccessary data: + df = df[[ + "job", + "build", + "dut_type", + "dut_version", + "start_time", + "passed", + "test_name", + "test_type", + "result_receive_rate_rate_avg", + "result_receive_rate_rate_stdev", + "result_receive_rate_rate_unit", + "result_pdr_lower_rate_value", + "result_pdr_lower_rate_unit", + "result_ndr_lower_rate_value", + "result_ndr_lower_rate_unit", + "telemetry" + ]] + # Transform metrics from strings to dataframes: + lst_telemetry = list() + for _, row in df.iterrows(): + d_telemetry = { + "metric": list(), + "labels": list(), # list of tuple(label, value) + "value": list(), + "timestamp": list() + } + if row["telemetry"] is not None and \ + not isinstance(row["telemetry"], float): + for itm in row["telemetry"]: + itm_lst = itm.replace("'", "").rsplit(" ", maxsplit=2) + metric, labels = itm_lst[0].split("{") + d_telemetry["metric"].append(metric) + d_telemetry["labels"].append( + [tuple(x.split("=")) for x in labels[:-1].split(",")] + ) + d_telemetry["value"].append(itm_lst[1]) + d_telemetry["timestamp"].append(itm_lst[2]) + metrics.update(d_telemetry["metric"]) + lst_telemetry.append(pd.DataFrame(data=d_telemetry)) + df["telemetry"] = lst_telemetry + + self._data = df + self._unique_metrics = sorted(metrics) + + def from_json(self, in_data: dict) -> None: + """Read the input data from json. + """ + + df = pd.read_json(in_data) + lst_telemetry = list() + metrics = set() # A set of unique metrics + for _, row in df.iterrows(): + telemetry = pd.DataFrame(row["telemetry"]) + lst_telemetry.append(telemetry) + metrics.update(telemetry["metric"].to_list()) + df["telemetry"] = lst_telemetry + + self._data = df + self._unique_metrics = sorted(metrics) + + def from_metrics(self, in_data: set) -> None: + """Read only the metrics. + """ + self._unique_metrics = in_data + + def from_metrics_with_labels(self, in_data: dict) -> None: + """Read only metrics with labels. + """ + self._unique_metrics_labels = pd.DataFrame.from_dict(in_data) + + def to_json(self) -> str: + """Return the data transformed from dataframe to json. + + :returns: Telemetry data transformed to a json structure. + :rtype: dict + """ + return self._data.to_json() + + @property + def unique_metrics(self) -> list: + """Return a set of unique metrics. + + :returns: A set of unique metrics. + :rtype: set + """ + return self._unique_metrics + + @property + def unique_metrics_with_labels(self) -> dict: + """ + """ + return self._unique_metrics_labels.to_dict() + + def get_selected_labels(self, metrics: list) -> dict: + """Return a dictionary with labels (keys) and all their possible values + (values) for all selected 'metrics'. + + :param metrics: List of metrics we are interested in. + :type metrics: list + :returns: A dictionary with labels and all their possible values. + :rtype: dict + """ + + df_labels = pd.DataFrame() + tmp_labels = dict() + for _, row in self._data.iterrows(): + telemetry = row["telemetry"] + for itm in metrics: + df = telemetry.loc[(telemetry["metric"] == itm)] + df_labels = pd.concat( + [df_labels, df], + ignore_index=True, + copy=False + ) + for _, tm in df.iterrows(): + for label in tm["labels"]: + if label[0] not in tmp_labels: + tmp_labels[label[0]] = set() + tmp_labels[label[0]].add(label[1]) + + selected_labels = dict() + for key in sorted(tmp_labels): + selected_labels[key] = sorted(tmp_labels[key]) + + self._unique_metrics_labels = df_labels[["metric", "labels"]].\ + loc[df_labels[["metric", "labels"]].astype(str).\ + drop_duplicates().index] + + return selected_labels + + @property + def str_metrics(self) -> str: + """Returns all unique metrics as a string. + """ + return TelemetryData.metrics_to_str(self._unique_metrics_labels) + + @staticmethod + def metrics_to_str(in_data: pd.DataFrame) -> str: + """Convert metrics from pandas dataframe to string. Metrics in string + are separated by '\n'. + + :param in_data: Metrics to be converted to a string. + :type in_data: pandas.DataFrame + :returns: Metrics as a string. + :rtype: str + """ + metrics = str() + for _, row in in_data.iterrows(): + labels = ','.join([f"{itm[0]}='{itm[1]}'" for itm in row["labels"]]) + metrics += f"{row['metric']}{{{labels}}}\n" + return metrics[:-1] + + def search_unique_metrics(self, string: str) -> list: + """Return a list of metrics which name includes the given string. + + :param string: A string which must be in the name of metric. + :type string: str + :returns: A list of metrics which name includes the given string. + :rtype: list + """ + return [itm for itm in self._unique_metrics if string in itm] + + def filter_selected_metrics_by_labels( + self, + selection: dict + ) -> pd.DataFrame: + """Filter selected unique metrics by labels and their values. + + :param selection: Labels and their values specified by the user. + :type selection: dict + :returns: Pandas dataframe with filtered metrics. + :rtype: pandas.DataFrame + """ + + def _is_selected(labels: list, sel: dict) -> bool: + """Check if the provided 'labels' are selected by the user. + + :param labels: List of labels and their values from a metric. The + items in this lists are two-item-lists whre the first item is + the label and the second one is its value. + :param sel: User selection. The keys are the selected lables and the + values are lists with label values. + :type labels: list + :type sel: dict + :returns: True if the 'labels' are selected by the user. + :rtype: bool + """ + passed = list() + labels = dict(labels) + for key in sel.keys(): + if key in list(labels.keys()): + if sel[key]: + passed.append(labels[key] in sel[key]) + else: + passed.append(True) + else: + passed.append(False) + return bool(passed and all(passed)) + + self._selected_metrics_labels = pd.DataFrame() + for _, row in self._unique_metrics_labels.iterrows(): + if _is_selected(row["labels"], selection): + self._selected_metrics_labels = pd.concat( + [self._selected_metrics_labels, row.to_frame().T], + ignore_index=True, + axis=0, + copy=False + ) + return self._selected_metrics_labels + + def select_tm_trending_data(self, selection: dict) -> pd.DataFrame: + """Select telemetry data for trending based on user's 'selection'. + + The output dataframe includes these columns: + - "job", + - "build", + - "dut_type", + - "dut_version", + - "start_time", + - "passed", + - "test_name", + - "test_id", + - "test_type", + - "result_receive_rate_rate_avg", + - "result_receive_rate_rate_stdev", + - "result_receive_rate_rate_unit", + - "result_pdr_lower_rate_value", + - "result_pdr_lower_rate_unit", + - "result_ndr_lower_rate_value", + - "result_ndr_lower_rate_unit", + - "tm_metric", + - "tm_value". + + :param selection: User's selection (metrics and labels). + :type selection: dict + :returns: Dataframe with selected data. + :rtype: pandas.DataFrame + """ + + df = pd.DataFrame() + + if self._data is None: + return df + if self._data.empty: + return df + if not selection: + return df + + df_sel = pd.DataFrame.from_dict(selection) + for _, row in self._data.iterrows(): + tm_row = row["telemetry"] + for _, tm_sel in df_sel.iterrows(): + df_tmp = tm_row.loc[tm_row["metric"] == tm_sel["metric"]] + for _, tm in df_tmp.iterrows(): + if tm["labels"] == tm_sel["labels"]: + labels = ','.join( + [f"{itm[0]}='{itm[1]}'" for itm in tm["labels"]] + ) + row["tm_metric"] = f"{tm['metric']}{{{labels}}}" + row["tm_value"] = tm["value"] + new_row = row.drop(labels=["telemetry", ]) + df = pd.concat( + [df, new_row.to_frame().T], + ignore_index=True, + axis=0, + copy=False + ) + return df diff --git a/csit.infra.dash/app/cdash/utils/trigger.py b/csit.infra.dash/app/cdash/utils/trigger.py index 60ef9a3f91..ac303b6b0b 100644 --- a/csit.infra.dash/app/cdash/utils/trigger.py +++ b/csit.infra.dash/app/cdash/utils/trigger.py @@ -14,6 +14,8 @@ """A module implementing the processing of a trigger. """ +from typing import Any + from json import loads, JSONDecodeError @@ -51,7 +53,7 @@ class Trigger: return self._id["type"] @property - def idx(self) -> any: + def idx(self) -> Any: return self._id["index"] @property @@ -59,5 +61,5 @@ class Trigger: return self._param @property - def value(self) -> any: + def value(self) -> Any: return self._val diff --git a/csit.infra.dash/app/cdash/utils/url_processing.py b/csit.infra.dash/app/cdash/utils/url_processing.py index 7f0121ef34..c90c54c41f 100644 --- a/csit.infra.dash/app/cdash/utils/url_processing.py +++ b/csit.infra.dash/app/cdash/utils/url_processing.py @@ -69,7 +69,7 @@ def url_decode(url: str) -> dict: parsed_url = urlparse(url) except ValueError as err: logging.warning(f"\nThe url {url} is not valid, ignoring.\n{repr(err)}") - return None + return dict() if parsed_url.fragment: try: @@ -85,7 +85,7 @@ def url_decode(url: str) -> dict: f"\nEncoded parameters: '{parsed_url.fragment}'" f"\n{repr(err)}" ) - return None + return dict() else: params = None diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py index 8584dee067..63e13ce141 100644 --- a/csit.infra.dash/app/cdash/utils/utils.py +++ b/csit.infra.dash/app/cdash/utils/utils.py @@ -346,29 +346,34 @@ def set_job_params(df: pd.DataFrame, job: str) -> dict: } -def get_list_group_items(tests: list) -> list: - """Generate list of ListGroupItems with checkboxes with selected tests. - - :param tests: List of tests to be displayed in the ListGroup. - :type tests: list - :returns: List of ListGroupItems with checkboxes with selected tests. +def get_list_group_items(items: list, type: str, colorize: bool=True) -> list: + """Generate list of ListGroupItems with checkboxes with selected items. + + :param items: List of items to be displayed in the ListGroup. + :param type: The type part of an element ID. + :param colorize: If True, the color of labels is set, otherwise the default + color is used. + :type items: list + :type type: str + :type colorize: bool + :returns: List of ListGroupItems with checkboxes with selected items. :rtype: list """ return [ dbc.ListGroupItem( children=[ dbc.Checkbox( - id={"type": "sel-cl", "index": i}, - label=l["id"], + id={"type": type, "index": i}, + label=l["id"] if isinstance(l, dict) else l, value=False, label_class_name="m-0 p-0", label_style={ "font-size": ".875em", - "color": get_color(i) + "color": get_color(i) if colorize else "#55595c" }, class_name="info" ) ], class_name="p-0" - ) for i, l in enumerate(tests) + ) for i, l in enumerate(items) ] |