aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2019-10-04 10:41:00 +0200
committerTibor Frank <tifrank@cisco.com>2019-10-07 15:04:27 +0000
commit0f496f4c337d06032618e861ed6dfb8b51fc2a9f (patch)
treefd167bf297ad0a5525137ad4909f40398fcf70b2 /resources/tools
parentf6a8c61f15deb7906d2caae10fea702235123954 (diff)
Trending: Send alerts when regression occurs
Change-Id: I45d218ea323245199ad5093bd64c6a9bd862cad9 Signed-off-by: Tibor Frank <tifrank@cisco.com>
Diffstat (limited to 'resources/tools')
-rw-r--r--resources/tools/presentation/generator_CPTA.py48
-rw-r--r--resources/tools/presentation/generator_alerts.py15
-rw-r--r--resources/tools/presentation/specification_CPTA.yaml20
3 files changed, 53 insertions, 30 deletions
diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 9780d35488..580b8f033b 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -333,7 +333,7 @@ def _generate_all_charts(spec, input_data):
job_name = graph["data"].keys()[0]
csv_tbl = list()
- res = list()
+ res = dict()
# Transform the data
logs.append(("INFO", " Creating the data set for the {0} '{1}'.".
@@ -379,22 +379,22 @@ def _generate_all_charts(spec, input_data):
for group in groups:
visible = list()
for tag in group:
- for test_name, test_data in chart_data.items():
+ for tst_name, test_data in chart_data.items():
if not test_data:
logs.append(("WARNING",
"No data for the test '{0}'".
- format(test_name)))
+ format(tst_name)))
continue
- if tag in chart_tags[test_name]:
+ if tag in chart_tags[tst_name]:
message = "index: {index}, test: {test}".format(
- index=index, test=test_name)
- test_name = test_name.split('.')[-1]
+ index=index, test=tst_name)
try:
trace, rslt = _generate_trending_traces(
test_data,
job_name=job_name,
build_info=build_info,
- name='-'.join(test_name.split('-')[2:-1]),
+ name='-'.join(tst_name.split('.')[-1].
+ split('-')[2:-1]),
color=COLORS[index])
except IndexError:
message = "Out of colors: {}".format(message)
@@ -404,25 +404,24 @@ def _generate_all_charts(spec, input_data):
continue
traces.extend(trace)
visible.extend([True for _ in range(len(trace))])
- res.append(rslt)
+ res[tst_name] = rslt
index += 1
break
visibility.append(visible)
else:
- for test_name, test_data in chart_data.items():
+ for tst_name, test_data in chart_data.items():
if not test_data:
logs.append(("WARNING", "No data for the test '{0}'".
- format(test_name)))
+ format(tst_name)))
continue
message = "index: {index}, test: {test}".format(
- index=index, test=test_name)
- test_name = test_name.split('.')[-1]
+ index=index, test=tst_name)
try:
trace, rslt = _generate_trending_traces(
test_data,
job_name=job_name,
build_info=build_info,
- name='-'.join(test_name.split('-')[2:-1]),
+ name='-'.join(tst_name.split('.')[-1].split('-')[2:-1]),
color=COLORS[index])
except IndexError:
message = "Out of colors: {}".format(message)
@@ -431,7 +430,7 @@ def _generate_all_charts(spec, input_data):
index += 1
continue
traces.extend(trace)
- res.append(rslt)
+ res[tst_name] = rslt
index += 1
if traces:
@@ -535,7 +534,7 @@ def _generate_all_charts(spec, input_data):
testbed
)
- anomaly_classifications = list()
+ anomaly_classifications = dict()
# Create the header:
csv_tables = dict()
@@ -554,9 +553,12 @@ def _generate_all_charts(spec, input_data):
for chart in spec.cpta["plots"]:
result = _generate_chart(chart)
- anomaly_classifications.extend(result["results"])
csv_tables[result["job_name"]].extend(result["csv_table"])
+ if anomaly_classifications.get(result["job_name"], None) is None:
+ anomaly_classifications[result["job_name"]] = dict()
+ anomaly_classifications[result["job_name"]].update(result["results"])
+
# Write the tables:
for job_name, csv_table in csv_tables.items():
file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
@@ -590,10 +592,16 @@ def _generate_all_charts(spec, input_data):
# Evaluate result:
if anomaly_classifications:
result = "PASS"
- for classification in anomaly_classifications:
- if classification == "regression" or classification == "outlier":
- result = "FAIL"
- break
+ for job_name, job_data in anomaly_classifications.iteritems():
+ file_name = "{0}-regressions-{1}.txt".\
+ format(spec.cpta["output-file"], job_name)
+ with open(file_name, 'w') as txt_file:
+ for test_name, classification in job_data.iteritems():
+ if classification == "regression":
+ txt_file.write(test_name + '\n')
+ if classification == "regression" or \
+ classification == "outlier":
+ result = "FAIL"
else:
result = "FAIL"
diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index ed4318b3a6..2c2b70dc06 100644
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -362,10 +362,23 @@ class Alerting(object):
cores=params["cores"] +
" " * (max_len_cores - len(params["cores"])))
+ # Add list of regressions:
+ file_name = "{0}/cpta-regressions-{1}.txt".\
+ format(config["output-dir"],
+ alert["urls"][idx].split('/')[-1])
+ try:
+ with open(file_name, 'r') as txt_file:
+ file_content = txt_file.read()
+ if file_content:
+ text += "\nRegressions (full paths):\n\n"
+ text += file_content
+ except IOError:
+ pass
+
text += "\nFor detailed information visit: {url}\n".\
format(url=alert["url-details"])
file_name = "{0}/{1}".format(config["output-dir"],
- config["output-file"])
+ config["output-file"])
logging.info("Writing the file '{0}.txt' ...".format(file_name))
try:
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index c6fc5fd9fd..7e46018dd6 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -188,7 +188,7 @@
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 763
+ start: 770
end: "lastCompletedBuild"
skip:
- 781
@@ -236,7 +236,7 @@
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 763
+ start: 770
end: "lastCompletedBuild"
skip:
- 781
@@ -287,7 +287,7 @@
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 513
+ start: 520
end: "lastCompletedBuild"
skip:
- 531
@@ -340,7 +340,7 @@
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 513
+ start: 520
end: "lastCompletedBuild"
skip:
- 531
@@ -396,7 +396,7 @@
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 514
+ start: 520
end: "lastCompletedBuild"
skip:
- 532
@@ -435,7 +435,7 @@
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 514
+ start: 520
end: "lastCompletedBuild"
skip:
- 532
@@ -666,6 +666,8 @@
traceorder: "normal" # "grouped" does not work: bug https://github.com/plotly/plotly.js/issues/1913
tracegroupgap: 20
bordercolor: "rgb(238, 238, 238)"
+ paper_bgcolor: "#fff"
+ plot_bgcolor: "#fff"
hoverlabel:
namelength: -1
@@ -694,7 +696,7 @@
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 763
+ start: 770
end: "lastCompletedBuild"
skip:
- 781
@@ -742,7 +744,7 @@
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 513
+ start: 520
end: "lastCompletedBuild"
skip:
- 531
@@ -795,7 +797,7 @@
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 514
+ start: 520
end: "lastCompletedBuild"
skip:
- 532