aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2019-04-01 16:31:38 +0200
committerTibor Frank <tifrank@cisco.com>2019-04-04 11:39:23 +0200
commitd45c3f997201a6f17b39b3e5ceb82b332852149f (patch)
tree845236245b530243b178271d039160860704e1a9
parentab635019fd3d5420a31fbd68c1549d0be2323104 (diff)
Trending: Alerts
Change-Id: I1c744f4eeefe3082be02daa681bfc032d6db4d15 Signed-off-by: Tibor Frank <tifrank@cisco.com>
Diffstat
-rw-r--r--resources/tools/presentation/generator_alerts.py226
-rw-r--r--resources/tools/presentation/generator_tables.py45
-rw-r--r--resources/tools/presentation/specification_CPTA.yaml173
-rw-r--r--resources/tools/presentation/specification_parser.py7
-rw-r--r--resources/tools/presentation_new/specification_CPTA.yaml18
5 files changed, 385 insertions, 84 deletions
diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index 83dfe2eb17..ce5d8035e9 100644
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -17,6 +17,7 @@ import logging
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from os.path import isdir
+from collections import OrderedDict
from utils import execute_command
from errors import PresentationError
@@ -151,6 +152,8 @@ class Alerting(object):
text=text,
html=html)
elif alert_data["way"] == "jenkins":
+ self._generate_email_body(alert_data)
+ # TODO: Remove when not needed
self._generate_files_for_jenkins(alert_data)
else:
raise AlertingError("Alert with way '{0}' is not implemented.".
@@ -203,82 +206,181 @@ class Alerting(object):
if smtp_server:
smtp_server.quit()
- def _create_alert_message(self, alert):
- """Create the message which is used in the generated alert.
+ def _get_compressed_failed_tests(self, alert, test_set, sort=True):
+ """Return the dictionary with compressed faild tests. The compression is
+ done by grouping the tests from the same area but with different NICs,
+ frame sizes and number of processor cores.
+
+ For example, the failed tests:
+ 10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+ 10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+ 10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+ 10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+ 10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+ 10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+
+ will be represented as:
+ ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
+ (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
+
+ Structure of returned data:
+
+ {
+ "trimmed_TC_name_1": {
+ "nics": [],
+ "framesizes": [],
+ "cores": []
+ }
+ ...
+ "trimmed_TC_name_N": {
+ "nics": [],
+ "framesizes": [],
+ "cores": []
+ }
+ }
- :param alert: Message is created for this alert.
+ :param alert: Files are created for this alert.
+ :param test_set: Specifies which set of tests will be included in the
+ result. Its name is the same as the name of file with failed tests.
+ :param sort: If True, the failed tests are sorted alphabetically.
:type alert: dict
- :returns: Message in the ASCII text and HTML format.
- :rtype: tuple(str, str)
+ :type test_set: str
+ :type sort: bool
+ :returns: CSIT build number, VPP version, Number of failed tests,
+ Compressed failed tests.
+ :rtype: tuple(str, str, int, OrderedDict)
"""
- if alert["type"] == "failed-tests":
- text = ""
- html = "<html><body>"
- for item in alert["include"]:
- file_name = "{path}/{name}".format(
- path=self._path_failed_tests, name=item)
- try:
- with open("{0}.txt".format(file_name), 'r') as txt_file:
- text += "{0}:\n\n".format(
- item.replace("failed-tests-", ""))
- text += txt_file.read() + "\n" * 2
- except IOError:
- logging.error("Not possible to read the file '{0}.txt'.".
- format(file_name))
- try:
- with open("{0}.rst".format(file_name), 'r') as rst_file:
- html += "<h2>{0}:</h2>".format(
- item.replace("failed-tests-", ""))
- html += rst_file.readlines()[2].\
- replace("../trending", alert.get("url", ""))
- html += "<br>" * 3
- except IOError:
- logging.error("Not possible to read the file '{0}.rst'.".
- format(file_name))
- html += "</body></html>"
+ directory = self.configs[alert["way"]]["output-dir"]
+ failed_tests = OrderedDict()
+ version = ""
+ try:
+ with open("{0}/{1}.txt".format(directory, test_set), 'r') as f_txt:
+ for idx, line in enumerate(f_txt):
+ if idx == 0:
+ build = line[:-1]
+ continue
+ if idx == 1:
+ version = line[:-1]
+ continue
+ try:
+ test = line[:-1].split('-')
+ nic = test[0]
+ framesize = test[1]
+ cores = test[2]
+ name = '-'.join(test[3:-1])
+ except IndexError:
+ continue
+ if failed_tests.get(name, None) is None:
+ failed_tests[name] = dict(nics=list(),
+ framesizes=list(),
+ cores=list())
+ if nic not in failed_tests[name]["nics"]:
+ failed_tests[name]["nics"].append(nic)
+ if framesize not in failed_tests[name]["framesizes"]:
+ failed_tests[name]["framesizes"].append(framesize)
+ if cores not in failed_tests[name]["cores"]:
+ failed_tests[name]["cores"].append(cores)
+ except IOError as err:
+ logging.error(repr(err))
+ return None, None, None, None
+ if sort:
+ sorted_failed_tests = OrderedDict()
+ keys = [k for k in failed_tests.keys()]
+ keys.sort()
+ for key in keys:
+ sorted_failed_tests[key] = failed_tests[key]
+ return build, version, idx-1, sorted_failed_tests
else:
+ return build, version, idx-1, failed_tests
+
+ def _generate_email_body(self, alert):
+ """Create the file which is used in the generated alert.
+
+ :param alert: Files are created for this alert.
+ :type alert: dict
+ """
+
+ if alert["type"] != "failed-tests":
raise AlertingError("Alert of type '{0}' is not implemented.".
format(alert["type"]))
- return text, html
+
+ config = self.configs[alert["way"]]
+
+ text = ""
+ for idx, test_set in enumerate(alert.get("include", [])):
+ build, version, nr, failed_tests = \
+ self._get_compressed_failed_tests(alert, test_set)
+ if build is None:
+ continue
+ text += ("\n\n{topo}-{arch}, "
+ "{nr} tests failed, "
+ "CSIT build: {link}/{build}, "
+ "VPP version: {version}\n\n".
+ format(topo=test_set.split('-')[-2],
+ arch=test_set.split('-')[-1],
+ nr=nr,
+ link=alert["urls"][idx],
+ build=build,
+ version=version))
+ max_len_name = 0
+ max_len_nics = 0
+ max_len_framesizes = 0
+ max_len_cores = 0
+ for name, params in failed_tests.items():
+ failed_tests[name]["nics"] = ",".join(sorted(params["nics"]))
+ failed_tests[name]["framesizes"] = \
+ ",".join(sorted(params["framesizes"]))
+ failed_tests[name]["cores"] = ",".join(sorted(params["cores"]))
+ if len(name) > max_len_name:
+ max_len_name = len(name)
+ if len(failed_tests[name]["nics"]) > max_len_nics:
+ max_len_nics = len(failed_tests[name]["nics"])
+ if len(failed_tests[name]["framesizes"]) > max_len_framesizes:
+ max_len_framesizes = len(failed_tests[name]["framesizes"])
+ if len(failed_tests[name]["cores"]) > max_len_cores:
+ max_len_cores = len(failed_tests[name]["cores"])
+
+ for name, params in failed_tests.items():
+ text += "{name} {nics} {frames} {cores}\n".format(
+ name=name + " " * (max_len_name - len(name)),
+ nics=params["nics"] +
+ " " * (max_len_nics - len(params["nics"])),
+ frames=params["framesizes"] + " " *
+ (max_len_framesizes - len(params["framesizes"])),
+ cores=params["cores"] +
+ " " * (max_len_cores - len(params["cores"])))
+
+ text += "\nFor detailed information visit: {url}\n".\
+ format(url=alert["url-details"])
+ file_name = "{0}/{1}".format(config["output-dir"],
+ config["output-file"])
+ logging.info("Writing the file '{0}.txt' ...".format(file_name))
+
+ try:
+ with open("{0}.txt".format(file_name), 'w') as txt_file:
+ txt_file.write(text)
+ except IOError:
+ logging.error("Not possible to write the file '{0}.txt'.".
+ format(file_name))
def _generate_files_for_jenkins(self, alert):
"""Create the file which is used in the generated alert.
+ # TODO: Remove when not needed.
+
:param alert: Files are created for this alert.
:type alert: dict
"""
config = self.configs[alert["way"]]
- if alert["type"] == "failed-tests":
- text, html = self._create_alert_message(alert)
- file_name = "{0}/{1}".format(config["output-dir"],
- config["output-file"])
- logging.info("Writing the file '{0}.txt' ...".format(file_name))
- try:
- with open("{0}.txt".format(file_name), 'w') as txt_file:
- txt_file.write(text)
- except IOError:
- logging.error("Not possible to write the file '{0}.txt'.".
- format(file_name))
- logging.info("Writing the file '{0}.html' ...".format(file_name))
- try:
- with open("{0}.html".format(file_name), 'w') as html_file:
- html_file.write(html)
- except IOError:
- logging.error("Not possible to write the file '{0}.html'.".
- format(file_name))
-
- zip_file = config.get("zip-output", None)
- if zip_file:
- logging.info("Writing the file '{0}/{1}' ...".
- format(config["output-dir"], zip_file))
- execute_command("tar czvf {dir}/{zip} --directory={dir} "
- "{input}.txt {input}.html".
- format(dir=config["output-dir"],
- zip=zip_file,
- input=config["output-file"]))
- else:
- raise AlertingError("Alert of type '{0}' is not implemented.".
- format(alert["type"]))
+ zip_file = config.get("zip-output", None)
+ if zip_file:
+ logging.info("Writing the file '{0}/{1}' ...".
+ format(config["output-dir"], zip_file))
+ execute_command("tar czvf {dir}/{zip} --directory={dir} "
+ "{input}.txt".
+ format(dir=config["output-dir"],
+ zip=zip_file,
+ input=config["output-file"]))
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index b55e4a545d..cdce5f98c0 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -858,6 +858,51 @@ def table_performance_trending_dashboard_html(table, input_data):
return
+def table_last_failed_tests(table, input_data):
+ """Generate the table(s) with algorithm: table_last_failed_tests
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ logging.info(" Generating the table {0} ...".
+ format(table.get("title", "")))
+
+ # Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
+ data = input_data.filter_data(table, continue_on_error=True)
+
+ if data is None or data.empty:
+ logging.warn(" No data for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
+ return
+
+ tbl_list = list()
+ for job, builds in table["data"].items():
+ for build in builds:
+ build = str(build)
+ tbl_list.append(build)
+ tbl_list.append(input_data.metadata(job, build).get("version", ""))
+ for tst_name, tst_data in data[job][build].iteritems():
+ if tst_data["status"] != "FAIL":
+ continue
+ groups = re.search(REGEX_NIC, tst_data["parent"])
+ if not groups:
+ continue
+ nic = groups.group(0)
+ tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
+
+ file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
+ logging.info(" Writing file: '{0}'".format(file_name))
+ with open(file_name, "w") as file_handler:
+ for test in tbl_list:
+ file_handler.write(test + '\n')
+
+
def table_failed_tests(table, input_data):
"""Generate the table(s) with algorithm: table_failed_tests
specified in the specification file.
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index fe89093369..bf1a359748 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -119,10 +119,16 @@
type: "failed-tests"
way: "jenkins"
include:
- - "failed-tests-3n-hsw"
- - "failed-tests-3n-skx"
- - "failed-tests-2n-skx"
- url: "https://docs.fd.io/csit/master/trending/trending"
+ - "last-failed-tests-3n-hsw"
+ - "last-failed-tests-3n-skx"
+ - "last-failed-tests-2n-skx"
+ - "last-failed-tests-nf-2n-skx"
+ urls:
+ - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master"
+ - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-skx"
+ - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-skx"
+ - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-weekly-master-2n-skx"
+ url-details: "https://docs.fd.io/csit/master/trending/introduction/failures.html"
configurations:
# Configuration of the email notifications.
@@ -148,19 +154,49 @@
data-sets:
+ # Compressed failed tests (last builds)
+ table-last-failed-tests-3n-hsw:
+ csit-vpp-perf-mrr-daily-master:
+ - "lastCompletedBuild"
+ table-last-failed-tests-3n-skx:
+ csit-vpp-perf-mrr-daily-master-3n-skx:
+ - "lastCompletedBuild"
+ table-last-failed-tests-2n-skx:
+ csit-vpp-perf-mrr-daily-master-2n-skx:
+ - "lastCompletedBuild"
+ table-last-failed-tests-nf-2n-skx:
+ csit-vpp-perf-mrr-weekly-master-2n-skx:
+ - "lastCompletedBuild"
+
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 482
+ start: 487
end: "lastCompletedBuild"
+ skip:
+ - 593
+ - 595
+ - 608
+ - 613
+ - 614
+ - 616
+ - 617
csit-dpdk-perf-mrr-weekly-master:
start: 55
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 482
+ start: 487
end: "lastCompletedBuild"
+ skip:
+ - 593
+ - 595
+ - 608
+ - 613
+ - 614
+ - 616
+ - 617
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
@@ -170,7 +206,7 @@
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 232
+ start: 237
end: "lastCompletedBuild"
skip:
- 356
@@ -203,7 +239,7 @@
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 232
+ start: 237
end: "lastCompletedBuild"
skip:
- 356
@@ -228,6 +264,7 @@
- 376
- 377
- 378
+
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
start: 20
@@ -238,16 +275,48 @@
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 232
+ start: 237
end: "lastCompletedBuild"
+ skip:
+ - 347
+ - 358
+ - 359
+ - 360
+ - 361
+ - 362
+ - 363
+ - 364
+ - 365
+ - 366
+ - 367
+ - 368
+ - 375
+ - 380
+ - 381
csit-dpdk-perf-mrr-weekly-master-2n-skx:
start: 20
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 232
+ start: 237
end: "lastCompletedBuild"
+ skip:
+ - 347
+ - 358
+ - 359
+ - 360
+ - 361
+ - 362
+ - 363
+ - 364
+ - 365
+ - 366
+ - 367
+ - 368
+ - 375
+ - 380
+ - 381
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
@@ -352,15 +421,23 @@
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 482
+ start: 487
end: "lastCompletedBuild"
+ skip:
+ - 593
+ - 595
+ - 608
+ - 613
+ - 614
+ - 616
+ - 617
csit-dpdk-perf-mrr-weekly-master:
start: 55
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 232
+ start: 237
end: "lastCompletedBuild"
skip:
- 356
@@ -393,8 +470,24 @@
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 232
+ start: 237
end: "lastCompletedBuild"
+ skip:
+ - 347
+ - 358
+ - 359
+ - 360
+ - 361
+ - 362
+ - 363
+ - 364
+ - 365
+ - 366
+ - 367
+ - 368
+ - 375
+ - 380
+ - 381
csit-vpp-perf-mrr-weekly-master-2n-skx:
start: 1
end: "lastCompletedBuild"
@@ -418,6 +511,60 @@
################################################################################
################################################################################
+
+# Compressed failed tests (last build)
+-
+ type: "table"
+ title: "Last failed tests (last builds) 3n-hsw"
+ algorithm: "table_last_failed_tests"
+ output-file-ext: ".txt"
+ output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-hsw"
+ data: "table-last-failed-tests-3n-hsw"
+ filter: "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "status"
+
+-
+ type: "table"
+ title: "Last failed tests (last builds) 3n-skx"
+ algorithm: "table_last_failed_tests"
+ output-file-ext: ".txt"
+ output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-skx"
+ data: "table-last-failed-tests-3n-skx"
+ filter: "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "status"
+
+-
+ type: "table"
+ title: "Last failed tests (last builds) 2n-skx"
+ algorithm: "table_last_failed_tests"
+ output-file-ext: ".txt"
+ output-file: "{DIR[STATIC,VPP]}/last-failed-tests-2n-skx"
+ data: "table-last-failed-tests-2n-skx"
+ filter: "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "status"
+
+-
+ type: "table"
+ title: "Last failed tests (last builds) NF 2n-skx"
+ algorithm: "table_last_failed_tests"
+ output-file-ext: ".txt"
+ output-file: "{DIR[STATIC,VPP]}/last-failed-tests-nf-2n-skx"
+ data: "table-last-failed-tests-nf-2n-skx"
+ filter: "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "status"
+
# 3n-hsw
-
type: "table"
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index ae566c67fa..53649609dd 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -434,6 +434,13 @@ class Specification(object):
builds = [x for x in range(builds["start"], build_end+1)
if x not in builds.get("skip", list())]
self.configuration["data-sets"][set_name][job] = builds
+ elif isinstance(builds, list):
+ for idx, item in enumerate(builds):
+ try:
+ builds[idx] = int(item)
+ except ValueError:
+ # defined as a range <build_type>
+ builds[idx] = self._get_build_number(job, item)
# Data sets: add sub-sets to sets (only one level):
for set_name, data_set in self.configuration["data-sets"].items():
diff --git a/resources/tools/presentation_new/specification_CPTA.yaml b/resources/tools/presentation_new/specification_CPTA.yaml
index e72fd9aa43..b2e47fd4c4 100644
--- a/resources/tools/presentation_new/specification_CPTA.yaml
+++ b/resources/tools/presentation_new/specification_CPTA.yaml
@@ -84,7 +84,7 @@
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 470
+ start: 500
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
start: 54
@@ -92,7 +92,7 @@
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 470
+ start: 500
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-hsw:
@@ -103,7 +103,7 @@
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 220
+ start: 250
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
start: 19
@@ -111,7 +111,7 @@
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 220
+ start: 250
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-skx:
@@ -122,7 +122,7 @@
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 220
+ start: 250
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
start: 19
@@ -130,7 +130,7 @@
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 220
+ start: 250
end: "lastCompletedBuild"
plot-performance-trending-dpdk-2n-skx:
@@ -228,7 +228,7 @@
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 470
+ start: 500
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
start: 54
@@ -236,7 +236,7 @@
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 220
+ start: 250
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
start: 19
@@ -244,7 +244,7 @@
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 220
+ start: 250
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
start: 19