aboutsummaryrefslogtreecommitdiffstats
path: root/csit.infra.dash
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2023-04-19 11:15:53 +0200
committerTibor Frank <tifrank@cisco.com>2023-04-19 14:05:10 +0000
commita7ed9061afe084648969a669f0c38bf567583a08 (patch)
treee2a2423c9c847aeee1bceb68a6fdbe52d90ca341 /csit.infra.dash
parentf85a6f4c807c40dcdc958f75363af6fd36e4023b (diff)
C-Dash: Add regexp filtering to comparison tables
Signed-off-by: Tibor Frank <tifrank@cisco.com> Change-Id: Ibe2b951859c9d775dd386dadd1bb141d74f53652
Diffstat (limited to 'csit.infra.dash')
-rw-r--r--csit.infra.dash/app/cdash/comparisons/layout.py98
-rw-r--r--csit.infra.dash/app/cdash/comparisons/tables.py62
-rw-r--r--csit.infra.dash/app/cdash/coverage/tables.py2
-rw-r--r--csit.infra.dash/app/cdash/utils/constants.py14
4 files changed, 141 insertions, 35 deletions
diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
index 489e6ebc6a..452afad1af 100644
--- a/csit.infra.dash/app/cdash/comparisons/layout.py
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -29,7 +29,7 @@ from ..utils.control_panel import ControlPanel
from ..utils.trigger import Trigger
from ..utils.url_processing import url_decode
from ..utils.utils import generate_options, gen_new_url
-from .tables import comparison_table
+from .tables import comparison_table, filter_table_data
# Control panel partameters and their default values.
@@ -204,6 +204,8 @@ class Layout:
children=[
dcc.Store(id="store-control-panel"),
dcc.Store(id="store-selected"),
+ dcc.Store(id="store-table-data"),
+ dcc.Store(id="store-filtered-table-data"),
dcc.Location(id="url", refresh=False),
self._add_ctrl_col(),
self._add_plotting_col()
@@ -505,28 +507,26 @@ class Layout:
)
]
+ @staticmethod
def _get_plotting_area(
- self,
- selected: dict,
- url: str,
- normalize: bool
+ title: str,
+ table: pd.DataFrame,
+ url: str
) -> list:
"""Generate the plotting area with all its content.
- :param selected: Selected parameters of tests.
- :param normalize: If true, the values in tables are normalized.
+ :param title: The title of the comparison table..
+ :param table: Comparison table to be displayed.
:param url: URL to be displayed in the modal window.
- :type selected: dict
- :type normalize: bool
+ :type title: str
+ :type table: pandas.DataFrame
:type url: str
:returns: List of rows with elements to be displayed in the plotting
area.
:rtype: list
"""
- title, df = comparison_table(self._data, selected, normalize)
-
- if df.empty:
+ if table.empty:
return dbc.Row(
dbc.Col(
children=dbc.Alert(
@@ -539,7 +539,7 @@ class Layout:
)
cols = list()
- for idx, col in enumerate(df.columns):
+ for idx, col in enumerate(table.columns):
if idx == 0:
cols.append({
"name": ["", col],
@@ -568,11 +568,13 @@ class Layout:
children=[
dbc.Col(
children=dash_table.DataTable(
+ id={"type": "table", "index": "comparison"},
columns=cols,
- data=df.to_dict("records"),
+ data=table.to_dict("records"),
merge_duplicate_headers=True,
- editable=True,
- filter_action="native",
+ editable=False,
+ filter_action="custom",
+ filter_query="",
sort_action="native",
sort_mode="multi",
selected_columns=[],
@@ -648,7 +650,10 @@ class Layout:
[
Output("store-control-panel", "data"),
Output("store-selected", "data"),
+ Output("store-table-data", "data"),
+ Output("store-filtered-table-data", "data"),
Output("plotting-area", "children"),
+ Output({"type": "table", "index": ALL}, "data"),
Output({"type": "ctrl-dd", "index": "dut"}, "value"),
Output({"type": "ctrl-dd", "index": "dutver"}, "options"),
Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"),
@@ -672,11 +677,13 @@ class Layout:
],
[
State("store-control-panel", "data"),
- State("store-selected", "data")
+ State("store-selected", "data"),
+ State("store-table-data", "data")
],
[
Input("url", "href"),
Input("normalize", "value"),
+ Input({"type": "table", "index": ALL}, "filter_query"),
Input({"type": "ctrl-dd", "index": ALL}, "value"),
Input({"type": "ctrl-cl", "index": ALL}, "value"),
Input({"type": "ctrl-btn", "index": ALL}, "n_clicks")
@@ -685,8 +692,10 @@ class Layout:
def _update_application(
control_panel: dict,
selected: dict,
+ store_table_data: list,
href: str,
normalize: list,
+ table_filter: str,
*_
) -> tuple:
"""Update the application when the event is detected.
@@ -713,6 +722,8 @@ class Layout:
on_draw = False
plotting_area = no_update
+ table_data = list()
+ filtered_data = None
trigger = Trigger(callback_context.triggered)
if trigger.type == "url" and url_params:
@@ -934,19 +945,35 @@ class Layout:
"cmp-val-dis": True,
"cmp-val-val": str()
})
+ elif trigger.type == "table" and trigger.idx == "comparison":
+ table_data = filter_table_data(
+ store_table_data,
+ table_filter[0]
+ )
+ filtered_data = table_data
+ table_data = [table_data, ]
if all((on_draw, selected["reference"]["set"],
selected["compare"]["set"], )):
+ title, table = comparison_table(self._data, selected, normalize)
plotting_area = self._get_plotting_area(
- selected=selected,
- normalize=bool(normalize),
+ title=title,
+ table=table,
url=gen_new_url(
parsed_url,
params={"selected": selected, "norm": normalize}
)
)
-
- ret_val = [ctrl_panel.panel, selected, plotting_area]
+ store_table_data = table.to_dict("records")
+
+ ret_val = [
+ ctrl_panel.panel,
+ selected,
+ store_table_data,
+ filtered_data,
+ plotting_area,
+ table_data
+ ]
ret_val.extend(ctrl_panel.values)
return ret_val
@@ -964,28 +991,33 @@ class Layout:
@app.callback(
Output("download-iterative-data", "data"),
- State("store-selected", "data"),
- State("normalize", "value"),
+ State("store-table-data", "data"),
+ State("store-filtered-table-data", "data"),
Input("plot-btn-download", "n_clicks"),
prevent_initial_call=True
)
- def _download_trending_data(selected: dict, normalize: list, _: int):
+ def _download_comparison_data(
+ table_data: list,
+ filtered_table_data: list,
+ _: int
+ ) -> dict:
"""Download the data.
- :param selected: List of tests selected by user stored in the
- browser.
- :param normalize: If set, the data is normalized to 2GHz CPU
- frequency.
- :type selected: list
- :type normalize: list
+ :param table_data: Original unfiltered table data.
+ :param filtered_table_data: Filtered table data.
+ :type table_data: list
+ :type filtered_table_data: list
:returns: dict of data frame content (base64 encoded) and meta data
used by the Download component.
:rtype: dict
"""
- if not selected:
+ if not table_data:
raise PreventUpdate
-
- _, table = comparison_table(self._data, selected, normalize, "csv")
+
+ if filtered_table_data:
+ table = pd.DataFrame.from_records(filtered_table_data)
+ else:
+ table = pd.DataFrame.from_records(table_data)
return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
index 31e268c6f0..8c19d3c776 100644
--- a/csit.infra.dash/app/cdash/comparisons/tables.py
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -309,3 +309,65 @@ def comparison_table(
)
return (title, df_cmp)
+
+
+def filter_table_data(
+ store_table_data: list,
+ table_filter: str
+ ) -> list:
+ """Filter table data using user specified filter.
+
+ :param store_table_data: Table data represented as a list of records.
+ :param table_filter: User specified filter.
+ :type store_table_data: list
+ :type table_filter: str
+ :returns: A new table created by filtering of table data represented as
+ a list of records.
+ :rtype: list
+ """
+
+ # Checks:
+ if not any((table_filter, store_table_data, )):
+ return store_table_data
+
+ def _split_filter_part(filter_part: str) -> tuple:
+ """Split a part of filter into column name, operator and value.
+ A "part of filter" is a sting berween "&&" operator.
+
+ :param filter_part: A part of filter.
+ :type filter_part: str
+ :returns: Column name, operator, value
+ :rtype: tuple[str, str, str|float]
+ """
+ for operator_type in C.OPERATORS:
+ for operator in operator_type:
+ if operator in filter_part:
+ name_p, val_p = filter_part.split(operator, 1)
+ name = name_p[name_p.find("{") + 1 : name_p.rfind("}")]
+ val_p = val_p.strip()
+ if (val_p[0] == val_p[-1] and val_p[0] in ("'", '"', '`')):
+ value = val_p[1:-1].replace("\\" + val_p[0], val_p[0])
+ else:
+ try:
+ value = float(val_p)
+ except ValueError:
+ value = val_p
+
+ return name, operator_type[0].strip(), value
+ return (None, None, None)
+
+ df = pd.DataFrame.from_records(store_table_data)
+ for filter_part in table_filter.split(" && "):
+ col_name, operator, filter_value = _split_filter_part(filter_part)
+ if operator == "contains":
+ df = df.loc[df[col_name].str.contains(filter_value, regex=True)]
+ elif operator in ("eq", "ne", "lt", "le", "gt", "ge"):
+ # These operators match pandas series operator method names.
+ df = df.loc[getattr(df[col_name], operator)(filter_value)]
+ elif operator == "datestartswith":
+ # This is a simplification of the front-end filtering logic,
+ # only works with complete fields in standard format.
+ # Currently not used in comparison tables.
+ df = df.loc[df[col_name].str.startswith(filter_value)]
+
+ return df.to_dict("records")
diff --git a/csit.infra.dash/app/cdash/coverage/tables.py b/csit.infra.dash/app/cdash/coverage/tables.py
index 31b227e9a8..a34b80f024 100644
--- a/csit.infra.dash/app/cdash/coverage/tables.py
+++ b/csit.infra.dash/app/cdash/coverage/tables.py
@@ -296,7 +296,7 @@ def coverage_tables(data: pd.DataFrame, selected: dict) -> list:
columns=cols,
data=cov_data.to_dict("records"),
merge_duplicate_headers=True,
- editable=True,
+ editable=False,
filter_action="none",
sort_action="native",
sort_mode="multi",
diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 6ab80d0b5c..94008f9bc7 100644
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -174,7 +174,7 @@ class Constants:
}
############################################################################
- # General, plots constants.
+ # General, plots and tables constants.
PLOT_COLORS = (
"#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A",
@@ -273,6 +273,18 @@ class Constants:
"result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR."
}
+ # Operators used to filter data in comparison tables.
+ OPERATORS = (
+ ("contains ", ),
+ ("lt ", "<"),
+ ("gt ", ">"),
+ ("eq ", "="),
+ ("ge ", ">="),
+ ("le ", "<="),
+ ("ne ", "!="),
+ ("datestartswith ", )
+ )
+
############################################################################
# News.