aboutsummaryrefslogtreecommitdiffstats
path: root/csit.infra.dash/app/cdash/utils/utils.py
blob: 29bee3d039b70ce92211bea5ef6a110b481fe4b1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Functions used by Dash applications.
"""

import pandas as pd
import plotly.graph_objects as go
import dash_bootstrap_components as dbc

import hdrh.histogram
import hdrh.codec

from math import sqrt
from dash import dcc
from datetime import datetime

from ..utils.constants import Constants as C
from ..utils.url_processing import url_encode


def get_color(idx: int) -> str:
    """Returns a color from the list defined in Constants.PLOT_COLORS defined by
    its index.

    :param idx: Index of the color.
    :type idx: int
    :returns: Color defined by hex code.
    :trype: str
    """
    return C.PLOT_COLORS[idx % len(C.PLOT_COLORS)]


def show_tooltip(tooltips:dict, id: str, title: str,
        clipboard_id: str=None) -> list:
    """Generate list of elements to display a text (e.g. a title) with a
    tooltip and optionaly with Copy&Paste icon and the clipboard
    functionality enabled.

    :param tooltips: Dictionary with tooltips.
    :param id: Tooltip ID.
    :param title: A text for which the tooltip will be displayed.
    :param clipboard_id: If defined, a Copy&Paste icon is displayed and the
        clipboard functionality is enabled.
    :type tooltips: dict
    :type id: str
    :type title: str
    :type clipboard_id: str
    :returns: List of elements to display a text with a tooltip and
        optionaly with Copy&Paste icon.
    :rtype: list
    """

    return [
        dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \
            if clipboard_id else str(),
        f"{title} ",
        dbc.Badge(
            id=id,
            children="?",
            pill=True,
            color="white",
            text_color="info",
            class_name="border ms-1",
        ),
        dbc.Tooltip(
            children=tooltips.get(id, str()),
            target=id,
            placement="auto"
        )
    ]


def label(key: str) -> str:
    """Returns a label for input elements (dropdowns, ...).

    If the label is not defined, the function returns the provided key.

    :param key: The key to the label defined in Constants.LABELS.
    :type key: str
    :returns: Label.
    :rtype: str
    """
    return C.LABELS.get(key, key)


def sync_checklists(options: list, sel: list, all: list, id: str) -> tuple:
    """Synchronize a checklist with defined "options" with its "All" checklist.

    :param options: List of options for the cheklist.
    :param sel: List of selected options.
    :param all: List of selected option from "All" checklist.
    :param id: ID of a checklist to be used for synchronization.
    :returns: Tuple of lists with otions for both checklists.
    :rtype: tuple of lists
    """
    opts = {v["value"] for v in options}
    if id =="all":
        sel = list(opts) if all else list()
    else:
        all = ["all", ] if set(sel) == opts else list()
    return sel, all


def list_tests(selection: dict) -> list:
    """Transform list of tests to a list of dictionaries usable by checkboxes.

    :param selection: List of tests to be displayed in "Selected tests" window.
    :type selection: list
    :returns: List of dictionaries with "label", "value" pairs for a checkbox.
    :rtype: list
    """
    if selection:
        return [{"label": v["id"], "value": v["id"]} for v in selection]
    else:
        return list()


def get_date(s_date: str) -> datetime:
    """Transform string reprezentation of date to datetime.datetime data type.

    :param s_date: String reprezentation of date.
    :type s_date: str
    :returns: Date as datetime.datetime.
    :rtype: datetime.datetime
    """
    return datetime(int(s_date[0:4]), int(s_date[5:7]), int(s_date[8:10]))


def gen_new_url(url_components: dict, params: dict) -> str:
    """Generate a new URL with encoded parameters.

    :param url_components: Dictionary with URL elements. It should contain
        "scheme", "netloc" and "path".
    :param url_components: URL parameters to be encoded to the URL.
    :type parsed_url: dict
    :type params: dict
    :returns Encoded URL with parameters.
    :rtype: str
    """

    if url_components:
        return url_encode(
            {
                "scheme": url_components.get("scheme", ""),
                "netloc": url_components.get("netloc", ""),
                "path": url_components.get("path", ""),
                "params": params
            }
        )
    else:
        return str()


def get_duts(df: pd.DataFrame) -> list:
    """Get the list of DUTs from the pre-processed information about jobs.

    :param df: DataFrame with information about jobs.
    :type df: pandas.DataFrame
    :returns: Alphabeticaly sorted list of DUTs.
    :rtype: list
    """
    return sorted(list(df["dut"].unique()))


def get_ttypes(df: pd.DataFrame, dut: str) -> list:
    """Get the list of test types from the pre-processed information about
    jobs.

    :param df: DataFrame with information about jobs.
    :param dut: The DUT for which the list of test types will be populated.
    :type df: pandas.DataFrame
    :type dut: str
    :returns: Alphabeticaly sorted list of test types.
    :rtype: list
    """
    return sorted(list(df.loc[(df["dut"] == dut)]["ttype"].unique()))


def get_cadences(df: pd.DataFrame, dut: str, ttype: str) -> list:
    """Get the list of cadences from the pre-processed information about
    jobs.

    :param df: DataFrame with information about jobs.
    :param dut: The DUT for which the list of cadences will be populated.
    :param ttype: The test type for which the list of cadences will be
        populated.
    :type df: pandas.DataFrame
    :type dut: str
    :type ttype: str
    :returns: Alphabeticaly sorted list of cadences.
    :rtype: list
    """
    return sorted(list(df.loc[(
        (df["dut"] == dut) &
        (df["ttype"] == ttype)
    )]["cadence"].unique()))


def get_test_beds(df: pd.DataFrame, dut: str, ttype: str, cadence: str) -> list:
    """Get the list of test beds from the pre-processed information about
    jobs.

    :param df: DataFrame with information about jobs.
    :param dut: The DUT for which the list of test beds will be populated.
    :param ttype: The test type for which the list of test beds will be
        populated.
    :param cadence: The cadence for which the list of test beds will be
        populated.
    :type df: pandas.DataFrame
    :type dut: str
    :type ttype: str
    :type cadence: str
    :returns: Alphabeticaly sorted list of test beds.
    :rtype: list
    """
    return sorted(list(df.loc[(
        (df["dut"] == dut) &
        (df["ttype"] == ttype) &
        (df["cadence"] == cadence)
    )]["tbed"].unique()))


def get_job(df: pd.DataFrame, dut, ttype, cadence, testbed):
    """Get the name of a job defined by dut, ttype, cadence, test bed.
    Input information comes from the control panel.

    :param df: DataFrame with information about jobs.
    :param dut: The DUT for which the job name will be created.
    :param ttype: The test type for which the job name will be created.
    :param cadence: The cadence for which the job name will be created.
    :param testbed: The test bed for which the job name will be created.
    :type df: pandas.DataFrame
    :type dut: str
    :type ttype: str
    :type cadence: str
    :type testbed: str
    :returns: Job name.
    :rtype: str
    """
    return df.loc[(
        (df["dut"] == dut) &
        (df["ttype"] == ttype) &
        (df["cadence"] == cadence) &
        (df["tbed"] == testbed)
    )]["job"].item()


def generate_options(opts: list, sort: bool=True) -> list:
    """Return list of options for radio items in control panel. The items in
    the list are dictionaries with keys "label" and "value".

    :params opts: List of options (str) to be used for the generated list.
    :type opts: list
    :returns: List of options (dict).
    :rtype: list
    """
    if sort:
        opts = sorted(opts)
    return [{"label": i, "value": i} for i in opts]


def set_job_params(df: pd.DataFrame, job: str) -> dict:
    """Create a dictionary with all options and values for (and from) the
    given job.

    :param df: DataFrame with information about jobs.
    :params job: The name of job for and from which the dictionary will be
        created.
    :type df: pandas.DataFrame
    :type job: str
    :returns: Dictionary with all options and values for (and from) the
        given job.
    :rtype: dict
    """

    l_job = job.split("-")
    return {
        "job": job,
        "dut": l_job[1],
        "ttype": l_job[3],
        "cadence": l_job[4],
        "tbed": "-".join(l_job[-2:]),
        "duts": generate_options(get_duts(df)),
        "ttypes": generate_options(get_ttypes(df, l_job[1])),
        "cadences": generate_options(get_cadences(df, l_job[1], l_job[3])),
        "tbeds": generate_options(
            get_test_beds(df, l_job[1], l_job[3], l_job[4]))
    }


def get_list_group_items(
        items: list,
        type: str,
        colorize: bool=True,
        add_index: bool=False
    ) -> list:
    """Generate list of ListGroupItems with checkboxes with selected items.

    :param items: List of items to be displayed in the ListGroup.
    :param type: The type part of an element ID.
    :param colorize: If True, the color of labels is set, otherwise the default
        color is used.
    :param add_index: Add index to the list items.
    :type items: list
    :type type: str
    :type colorize: bool
    :type add_index: bool
    :returns: List of ListGroupItems with checkboxes with selected items.
    :rtype: list
    """

    children = list()
    for i, l in enumerate(items):
        idx = f"{i + 1}. " if add_index else str()
        label = f"{idx}{l['id']}" if isinstance(l, dict) else f"{idx}{l}"
        children.append(
            dbc.ListGroupItem(
                children=[
                    dbc.Checkbox(
                        id={"type": type, "index": i},
                        label=label,
                        value=False,
                        label_class_name="m-0 p-0",
                        label_style={
                            "font-size": ".875em",
                            "color": get_color(i) if colorize else "#55595c"
                        },
                        class_name="info"
                    )
                ],
                class_name="p-0"
            )
        )

    return children


def relative_change_stdev(mean1, mean2, std1, std2):
    """Compute relative standard deviation of change of two values.

    The "1" values are the base for comparison.
    Results are returned as percentage (and percentual points for stdev).
    Linearized theory is used, so results are wrong for relatively large stdev.

    :param mean1: Mean of the first number.
    :param mean2: Mean of the second number.
    :param std1: Standard deviation estimate of the first number.
    :param std2: Standard deviation estimate of the second number.
    :type mean1: float
    :type mean2: float
    :type std1: float
    :type std2: float
    :returns: Relative change and its stdev.
    :rtype: float
    """
    mean1, mean2 = float(mean1), float(mean2)
    quotient = mean2 / mean1
    first = std1 / mean1
    second = std2 / mean2
    std = quotient * sqrt(first * first + second * second)
    return (quotient - 1) * 100, std * 100


def get_hdrh_latencies(row: pd.Series, name: str) -> dict:
    """Get the HDRH latencies from the test data.

    :param row: A row fron the data frame with test data.
    :param name: The test name to be displayed as the graph title.
    :type row: pandas.Series
    :type name: str
    :returns: Dictionary with HDRH latencies.
    :rtype: dict
    """

    latencies = {"name": name}
    for key in C.LAT_HDRH:
        try:
            latencies[key] = row[key]
        except KeyError:
            return None

    return latencies


def graph_hdrh_latency(data: dict, layout: dict) -> go.Figure:
    """Generate HDR Latency histogram graphs.

    :param data: HDRH data.
    :param layout: Layout of plot.ly graph.
    :type data: dict
    :type layout: dict
    :returns: HDR latency Histogram.
    :rtype: plotly.graph_objects.Figure
    """

    fig = None

    traces = list()
    for idx, (lat_name, lat_hdrh) in enumerate(data.items()):
        try:
            decoded = hdrh.histogram.HdrHistogram.decode(lat_hdrh)
        except (hdrh.codec.HdrLengthException, TypeError):
            continue
        previous_x = 0.0
        prev_perc = 0.0
        xaxis = list()
        yaxis = list()
        hovertext = list()
        for item in decoded.get_recorded_iterator():
            # The real value is "percentile".
            # For 100%, we cut that down to "x_perc" to avoid
            # infinity.
            percentile = item.percentile_level_iterated_to
            x_perc = min(percentile, C.PERCENTILE_MAX)
            xaxis.append(previous_x)
            yaxis.append(item.value_iterated_to)
            hovertext.append(
                f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
                f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
                f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
                f"Latency: {item.value_iterated_to}uSec"
            )
            next_x = 100.0 / (100.0 - x_perc)
            xaxis.append(next_x)
            yaxis.append(item.value_iterated_to)
            hovertext.append(
                f"<b>{C.GRAPH_LAT_HDRH_DESC[lat_name]}</b><br>"
                f"Direction: {('W-E', 'E-W')[idx % 2]}<br>"
                f"Percentile: {prev_perc:.5f}-{percentile:.5f}%<br>"
                f"Latency: {item.value_iterated_to}uSec"
            )
            previous_x = next_x
            prev_perc = percentile

        traces.append(
            go.Scatter(
                x=xaxis,
                y=yaxis,
                name=C.GRAPH_LAT_HDRH_DESC[lat_name],
                mode="lines",
                legendgroup=C.GRAPH_LAT_HDRH_DESC[lat_name],
                showlegend=bool(idx % 2),
                line=dict(
                    color=get_color(int(idx/2)),
                    dash="solid",
                    width=1 if idx % 2 else 2
                ),
                hovertext=hovertext,
                hoverinfo="text"
            )
        )
    if traces:
        fig = go.Figure()
        fig.add_traces(traces)
        layout_hdrh = layout.get("plot-hdrh-latency", None)
        if lat_hdrh:
            fig.update_layout(layout_hdrh)

    return fig