aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools/dash/app/pal/data
diff options
context:
space:
mode:
Diffstat (limited to 'resources/tools/dash/app/pal/data')
-rw-r--r--resources/tools/dash/app/pal/data/data.py8
-rw-r--r--resources/tools/dash/app/pal/data/data.yaml17
-rw-r--r--resources/tools/dash/app/pal/data/utils.py69
3 files changed, 89 insertions, 5 deletions
diff --git a/resources/tools/dash/app/pal/data/data.py b/resources/tools/dash/app/pal/data/data.py
index f2c02acc63..296db024c0 100644
--- a/resources/tools/dash/app/pal/data/data.py
+++ b/resources/tools/dash/app/pal/data/data.py
@@ -213,15 +213,15 @@ class Data:
days=days
),
self._create_dataframe_from_parquet(
- path=self._get_path("statistics-trending"),
+ path=self._get_path("statistics-trending-mrr"),
partition_filter=l_mrr,
- columns=self._get_columns("statistics-trending"),
+ columns=self._get_columns("statistics-trending-mrr"),
days=days
),
self._create_dataframe_from_parquet(
- path=self._get_path("statistics-trending"),
+ path=self._get_path("statistics-trending-ndrpdr"),
partition_filter=l_ndrpdr,
- columns=self._get_columns("statistics-trending"),
+ columns=self._get_columns("statistics-trending-ndrpdr"),
days=days
)
)
diff --git a/resources/tools/dash/app/pal/data/data.yaml b/resources/tools/dash/app/pal/data/data.yaml
index 2585ef0e84..59533f97a4 100644
--- a/resources/tools/dash/app/pal/data/data.yaml
+++ b/resources/tools/dash/app/pal/data/data.yaml
@@ -5,7 +5,7 @@ statistics:
- build
- start_time
- duration
-statistics-trending:
+statistics-trending-ndrpdr:
path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/trending
columns:
- job
@@ -13,8 +13,23 @@ statistics-trending:
- dut_type
- dut_version
- hosts
+ - start_time
+ - passed
+ - test_id
+ - result_pdr_lower_rate_value
+ - result_ndr_lower_rate_value
+statistics-trending-mrr:
+ path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/trending
+ columns:
+ - job
+ - build
+ - dut_type
+ - dut_version
+ - hosts
+ - start_time
- passed
- test_id
+ - result_receive_rate_rate_avg
trending-mrr:
path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/trending
columns:
diff --git a/resources/tools/dash/app/pal/data/utils.py b/resources/tools/dash/app/pal/data/utils.py
new file mode 100644
index 0000000000..63c9c1aaa4
--- /dev/null
+++ b/resources/tools/dash/app/pal/data/utils.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+"""
+
+from numpy import isnan
+
+from ..jumpavg import classify
+
+
+def classify_anomalies(data):
+ """Process the data and return anomalies and trending values.
+
+ Gather data into groups with average as trend value.
+ Decorate values within groups to be normal,
+ the first value of changed average as a regression, or a progression.
+
+ :param data: Full data set with unavailable samples replaced by nan.
+ :type data: OrderedDict
+ :returns: Classification and trend values
+ :rtype: 3-tuple, list of strings, list of floats and list of floats
+ """
+ # NaN means something went wrong.
+ # Use 0.0 to cause that being reported as a severe regression.
+ bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
+ # TODO: Make BitCountingGroupList a subclass of list again?
+ group_list = classify(bare_data).group_list
+ group_list.reverse() # Just to use .pop() for FIFO.
+ classification = list()
+ avgs = list()
+ stdevs = list()
+ active_group = None
+ values_left = 0
+ avg = 0.0
+ stdv = 0.0
+ for sample in data.values():
+ if isnan(sample):
+ classification.append("outlier")
+ avgs.append(sample)
+ stdevs.append(sample)
+ continue
+ if values_left < 1 or active_group is None:
+ values_left = 0
+ while values_left < 1: # Ignore empty groups (should not happen).
+ active_group = group_list.pop()
+ values_left = len(active_group.run_list)
+ avg = active_group.stats.avg
+ stdv = active_group.stats.stdev
+ classification.append(active_group.comment)
+ avgs.append(avg)
+ stdevs.append(stdv)
+ values_left -= 1
+ continue
+ classification.append("normal")
+ avgs.append(avg)
+ stdevs.append(stdv)
+ values_left -= 1
+ return classification, avgs, stdevs