diff options
Diffstat (limited to 'csit.infra.dash')
| -rw-r--r-- | csit.infra.dash/app/cdash/comparisons/layout.py | 98 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/comparisons/tables.py | 62 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/coverage/tables.py | 2 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/constants.py | 14 |
4 files changed, 141 insertions, 35 deletions
diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py index 489e6ebc6a..452afad1af 100644 --- a/csit.infra.dash/app/cdash/comparisons/layout.py +++ b/csit.infra.dash/app/cdash/comparisons/layout.py @@ -29,7 +29,7 @@ from ..utils.control_panel import ControlPanel from ..utils.trigger import Trigger from ..utils.url_processing import url_decode from ..utils.utils import generate_options, gen_new_url -from .tables import comparison_table +from .tables import comparison_table, filter_table_data # Control panel partameters and their default values. @@ -204,6 +204,8 @@ class Layout: children=[ dcc.Store(id="store-control-panel"), dcc.Store(id="store-selected"), + dcc.Store(id="store-table-data"), + dcc.Store(id="store-filtered-table-data"), dcc.Location(id="url", refresh=False), self._add_ctrl_col(), self._add_plotting_col() @@ -505,28 +507,26 @@ class Layout: ) ] + @staticmethod def _get_plotting_area( - self, - selected: dict, - url: str, - normalize: bool + title: str, + table: pd.DataFrame, + url: str ) -> list: """Generate the plotting area with all its content. - :param selected: Selected parameters of tests. - :param normalize: If true, the values in tables are normalized. + :param title: The title of the comparison table.. + :param table: Comparison table to be displayed. :param url: URL to be displayed in the modal window. - :type selected: dict - :type normalize: bool + :type title: str + :type table: pandas.DataFrame :type url: str :returns: List of rows with elements to be displayed in the plotting area. :rtype: list """ - title, df = comparison_table(self._data, selected, normalize) - - if df.empty: + if table.empty: return dbc.Row( dbc.Col( children=dbc.Alert( @@ -539,7 +539,7 @@ class Layout: ) cols = list() - for idx, col in enumerate(df.columns): + for idx, col in enumerate(table.columns): if idx == 0: cols.append({ "name": ["", col], @@ -568,11 +568,13 @@ class Layout: children=[ dbc.Col( children=dash_table.DataTable( + id={"type": "table", "index": "comparison"}, columns=cols, - data=df.to_dict("records"), + data=table.to_dict("records"), merge_duplicate_headers=True, - editable=True, - filter_action="native", + editable=False, + filter_action="custom", + filter_query="", sort_action="native", sort_mode="multi", selected_columns=[], @@ -648,7 +650,10 @@ class Layout: [ Output("store-control-panel", "data"), Output("store-selected", "data"), + Output("store-table-data", "data"), + Output("store-filtered-table-data", "data"), Output("plotting-area", "children"), + Output({"type": "table", "index": ALL}, "data"), Output({"type": "ctrl-dd", "index": "dut"}, "value"), Output({"type": "ctrl-dd", "index": "dutver"}, "options"), Output({"type": "ctrl-dd", "index": "dutver"}, "disabled"), @@ -672,11 +677,13 @@ class Layout: ], [ State("store-control-panel", "data"), - State("store-selected", "data") + State("store-selected", "data"), + State("store-table-data", "data") ], [ Input("url", "href"), Input("normalize", "value"), + Input({"type": "table", "index": ALL}, "filter_query"), Input({"type": "ctrl-dd", "index": ALL}, "value"), Input({"type": "ctrl-cl", "index": ALL}, "value"), Input({"type": "ctrl-btn", "index": ALL}, "n_clicks") @@ -685,8 +692,10 @@ class Layout: def _update_application( control_panel: dict, selected: dict, + store_table_data: list, href: str, normalize: list, + table_filter: str, *_ ) -> tuple: """Update the application when the event is detected. @@ -713,6 +722,8 @@ class Layout: on_draw = False plotting_area = no_update + table_data = list() + filtered_data = None trigger = Trigger(callback_context.triggered) if trigger.type == "url" and url_params: @@ -934,19 +945,35 @@ class Layout: "cmp-val-dis": True, "cmp-val-val": str() }) + elif trigger.type == "table" and trigger.idx == "comparison": + table_data = filter_table_data( + store_table_data, + table_filter[0] + ) + filtered_data = table_data + table_data = [table_data, ] if all((on_draw, selected["reference"]["set"], selected["compare"]["set"], )): + title, table = comparison_table(self._data, selected, normalize) plotting_area = self._get_plotting_area( - selected=selected, - normalize=bool(normalize), + title=title, + table=table, url=gen_new_url( parsed_url, params={"selected": selected, "norm": normalize} ) ) - - ret_val = [ctrl_panel.panel, selected, plotting_area] + store_table_data = table.to_dict("records") + + ret_val = [ + ctrl_panel.panel, + selected, + store_table_data, + filtered_data, + plotting_area, + table_data + ] ret_val.extend(ctrl_panel.values) return ret_val @@ -964,28 +991,33 @@ class Layout: @app.callback( Output("download-iterative-data", "data"), - State("store-selected", "data"), - State("normalize", "value"), + State("store-table-data", "data"), + State("store-filtered-table-data", "data"), Input("plot-btn-download", "n_clicks"), prevent_initial_call=True ) - def _download_trending_data(selected: dict, normalize: list, _: int): + def _download_comparison_data( + table_data: list, + filtered_table_data: list, + _: int + ) -> dict: """Download the data. - :param selected: List of tests selected by user stored in the - browser. - :param normalize: If set, the data is normalized to 2GHz CPU - frequency. - :type selected: list - :type normalize: list + :param table_data: Original unfiltered table data. + :param filtered_table_data: Filtered table data. + :type table_data: list + :type filtered_table_data: list :returns: dict of data frame content (base64 encoded) and meta data used by the Download component. :rtype: dict """ - if not selected: + if not table_data: raise PreventUpdate - - _, table = comparison_table(self._data, selected, normalize, "csv") + + if filtered_table_data: + table = pd.DataFrame.from_records(filtered_table_data) + else: + table = pd.DataFrame.from_records(table_data) return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME) diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py index 31e268c6f0..8c19d3c776 100644 --- a/csit.infra.dash/app/cdash/comparisons/tables.py +++ b/csit.infra.dash/app/cdash/comparisons/tables.py @@ -309,3 +309,65 @@ def comparison_table( ) return (title, df_cmp) + + +def filter_table_data( + store_table_data: list, + table_filter: str + ) -> list: + """Filter table data using user specified filter. + + :param store_table_data: Table data represented as a list of records. + :param table_filter: User specified filter. + :type store_table_data: list + :type table_filter: str + :returns: A new table created by filtering of table data represented as + a list of records. + :rtype: list + """ + + # Checks: + if not any((table_filter, store_table_data, )): + return store_table_data + + def _split_filter_part(filter_part: str) -> tuple: + """Split a part of filter into column name, operator and value. + A "part of filter" is a sting berween "&&" operator. + + :param filter_part: A part of filter. + :type filter_part: str + :returns: Column name, operator, value + :rtype: tuple[str, str, str|float] + """ + for operator_type in C.OPERATORS: + for operator in operator_type: + if operator in filter_part: + name_p, val_p = filter_part.split(operator, 1) + name = name_p[name_p.find("{") + 1 : name_p.rfind("}")] + val_p = val_p.strip() + if (val_p[0] == val_p[-1] and val_p[0] in ("'", '"', '`')): + value = val_p[1:-1].replace("\\" + val_p[0], val_p[0]) + else: + try: + value = float(val_p) + except ValueError: + value = val_p + + return name, operator_type[0].strip(), value + return (None, None, None) + + df = pd.DataFrame.from_records(store_table_data) + for filter_part in table_filter.split(" && "): + col_name, operator, filter_value = _split_filter_part(filter_part) + if operator == "contains": + df = df.loc[df[col_name].str.contains(filter_value, regex=True)] + elif operator in ("eq", "ne", "lt", "le", "gt", "ge"): + # These operators match pandas series operator method names. + df = df.loc[getattr(df[col_name], operator)(filter_value)] + elif operator == "datestartswith": + # This is a simplification of the front-end filtering logic, + # only works with complete fields in standard format. + # Currently not used in comparison tables. + df = df.loc[df[col_name].str.startswith(filter_value)] + + return df.to_dict("records") diff --git a/csit.infra.dash/app/cdash/coverage/tables.py b/csit.infra.dash/app/cdash/coverage/tables.py index 31b227e9a8..a34b80f024 100644 --- a/csit.infra.dash/app/cdash/coverage/tables.py +++ b/csit.infra.dash/app/cdash/coverage/tables.py @@ -296,7 +296,7 @@ def coverage_tables(data: pd.DataFrame, selected: dict) -> list: columns=cols, data=cov_data.to_dict("records"), merge_duplicate_headers=True, - editable=True, + editable=False, filter_action="none", sort_action="native", sort_mode="multi", diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py index 6ab80d0b5c..94008f9bc7 100644 --- a/csit.infra.dash/app/cdash/utils/constants.py +++ b/csit.infra.dash/app/cdash/utils/constants.py @@ -174,7 +174,7 @@ class Constants: } ############################################################################ - # General, plots constants. + # General, plots and tables constants. PLOT_COLORS = ( "#1A1110", "#DA2647", "#214FC6", "#01786F", "#BD8260", "#FFD12A", @@ -273,6 +273,18 @@ class Constants: "result_latency_reverse_pdr_90_hdrh": "High-load, 90% PDR." } + # Operators used to filter data in comparison tables. + OPERATORS = ( + ("contains ", ), + ("lt ", "<"), + ("gt ", ">"), + ("eq ", "="), + ("ge ", ">="), + ("le ", "<="), + ("ne ", "!="), + ("datestartswith ", ) + ) + ############################################################################ # News. |