aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--csit.infra.dash/app/cdash/comparisons/layout.py47
-rw-r--r--csit.infra.dash/app/cdash/comparisons/tables.py68
2 files changed, 87 insertions, 28 deletions
diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
index 82b5b2d544..9c89a55bcb 100644
--- a/csit.infra.dash/app/cdash/comparisons/layout.py
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -616,7 +616,7 @@ class Layout:
),
dbc.Button(
id="plot-btn-download",
- children="Download Data",
+ children="Download Table",
class_name="me-1",
color="info",
style={
@@ -624,7 +624,18 @@ class Layout:
"padding": "0rem 1rem"
}
),
- dcc.Download(id="download-iterative-data")
+ dcc.Download(id="download-iterative-data"),
+ dbc.Button(
+ id="plot-btn-download-raw",
+ children="Download Raw Data",
+ class_name="me-1",
+ color="info",
+ style={
+ "text-transform": "none",
+ "padding": "0rem 1rem"
+ }
+ ),
+ dcc.Download(id="download-raw-data")
],
className=\
"d-grid gap-0 d-md-flex justify-content-md-end"
@@ -1047,6 +1058,38 @@ class Layout:
return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
@app.callback(
+ Output("download-raw-data", "data"),
+ State("store-selected", "data"),
+ Input("plot-btn-download-raw", "n_clicks"),
+ prevent_initial_call=True
+ )
+ def _download_raw_comparison_data(selected: dict, _: int) -> dict:
+ """Download the data.
+
+ :param selected: Selected tests.
+ :type selected: dict
+ :returns: dict of data frame content (base64 encoded) and meta data
+ used by the Download component.
+ :rtype: dict
+ """
+
+ if not selected:
+ raise PreventUpdate
+
+ _, table = comparison_table(
+ data=self._data,
+ selected=selected,
+ normalize=False,
+ remove_outliers=False,
+ raw_data=True
+ )
+
+ return dcc.send_data_frame(
+ table.dropna(how="all", axis=1).to_csv,
+ f"raw_{C.COMP_DOWNLOAD_FILE_NAME}"
+ )
+
+ @app.callback(
Output("offcanvas-documentation", "is_open"),
Input("btn-documentation", "n_clicks"),
State("offcanvas-documentation", "is_open")
diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
index 0c247e87c2..18f9404f0a 100644
--- a/csit.infra.dash/app/cdash/comparisons/tables.py
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -27,7 +27,8 @@ def select_comp_data(
data: pd.DataFrame,
selected: dict,
normalize: bool=False,
- remove_outliers: bool=False
+ remove_outliers: bool=False,
+ raw_data: bool=False
) -> pd.DataFrame:
"""Select data for a comparison table.
@@ -38,10 +39,13 @@ def select_comp_data(
Constants.NORM_FREQUENCY.
:param remove_outliers: If True the outliers are removed before
generating the table.
+ :param raw_data: If True, returns data as it is in parquets without any
+ processing. It is used for "download raw data" feature.
:type data: pandas.DataFrame
:type selected: dict
:type normalize: bool
:type remove_outliers: bool
+ :type raw_data: bool
:returns: A data frame with selected data.
:rtype: pandas.DataFrame
"""
@@ -161,13 +165,14 @@ def select_comp_data(
norm_factor = C.NORM_FREQUENCY / C.FREQUENCY[itm["tbed"]]
else:
norm_factor = 1.0
- tmp_df = _calculate_statistics(
- tmp_df,
- itm["ttype"].lower(),
- itm["driver"],
- norm_factor,
- remove_outliers=remove_outliers
- )
+ if not raw_data:
+ tmp_df = _calculate_statistics(
+ tmp_df,
+ itm["ttype"].lower(),
+ itm["driver"],
+ norm_factor,
+ remove_outliers=remove_outliers
+ )
lst_df.append(tmp_df)
@@ -190,7 +195,8 @@ def comparison_table(
selected: dict,
normalize: bool,
format: str="html",
- remove_outliers: bool=False
+ remove_outliers: bool=False,
+ raw_data: bool=False
) -> tuple:
"""Generate a comparison table.
@@ -206,11 +212,14 @@ def comparison_table(
units.
:param remove_outliers: If True the outliers are removed before
generating the table.
+ :param raw_data: If True, returns data as it is in parquets without any
+ processing. It is used for "download raw data" feature.
:type data: pandas.DataFrame
:type selected: dict
:type normalize: bool
:type format: str
:type remove_outliers: bool
+ :type raw_data: bool
:returns: A tuple with the tabe title and the comparison table.
:rtype: tuple[str, pandas.DataFrame]
"""
@@ -241,9 +250,32 @@ def comparison_table(
})
return selection
+ # Select reference data
r_sel = deepcopy(selected["reference"]["selection"])
- c_params = selected["compare"]
r_selection = _create_selection(r_sel)
+ r_data = select_comp_data(
+ data, r_selection, normalize, remove_outliers, raw_data
+ )
+
+ # Select compare data
+ c_sel = deepcopy(selected["reference"]["selection"])
+ c_params = selected["compare"]
+ if c_params["parameter"] in ("core", "frmsize", "ttype"):
+ c_sel[c_params["parameter"]] = [c_params["value"], ]
+ else:
+ c_sel[c_params["parameter"]] = c_params["value"]
+ c_selection = _create_selection(c_sel)
+ c_data = select_comp_data(
+ data, c_selection, normalize, remove_outliers, raw_data
+ )
+
+ if raw_data:
+ r_data["ref/cmp"] = "reference"
+ c_data["ref/cmp"] = "compare"
+ return str(), pd.concat([r_data, c_data], ignore_index=True, copy=False)
+
+ if r_data.empty or c_data.empty:
+ return str(), pd.DataFrame()
if format == "html" and "Latency" not in r_sel["ttype"]:
unit_factor, s_unit_factor = (1e6, "M")
@@ -266,22 +298,6 @@ def comparison_table(
r_name = "|".join(r_name)
c_name = c_params["value"]
- # Select reference data
- r_data = select_comp_data(data, r_selection, normalize, remove_outliers)
-
- # Select compare data
- c_sel = deepcopy(selected["reference"]["selection"])
- if c_params["parameter"] in ("core", "frmsize", "ttype"):
- c_sel[c_params["parameter"]] = [c_params["value"], ]
- else:
- c_sel[c_params["parameter"]] = c_params["value"]
-
- c_selection = _create_selection(c_sel)
- c_data = select_comp_data(data, c_selection, normalize, remove_outliers)
-
- if r_data.empty or c_data.empty:
- return str(), pd.DataFrame()
-
l_name, l_r_mean, l_r_std, l_c_mean, l_c_std, l_rc_mean, l_rc_std, unit = \
list(), list(), list(), list(), list(), list(), list(), set()
for _, row in r_data.iterrows():