Trending: Alerts

Change-Id: I1c744f4eeefe3082be02daa681bfc032d6db4d15 Signed-off-by: Tibor Frank <tifrank@cisco.com>
author: Tibor Frank <tifrank@cisco.com> 2019-04-01 16:31:38 +0200
committer: Tibor Frank <tifrank@cisco.com> 2019-04-04 11:39:23 +0200
commit: d45c3f997201a6f17b39b3e5ceb82b332852149f (patch)
tree: 845236245b530243b178271d039160860704e1a9 /resources/tools/presentation
parent: ab635019fd3d5420a31fbd68c1549d0be2323104 (diff)
4 files changed, 376 insertions, 75 deletions
diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index 83dfe2eb17..ce5d8035e9 100644
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -17,6 +17,7 @@ import logging
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 from os.path import isdir
+from collections import OrderedDict
 
 from utils import execute_command
 from errors import PresentationError
@@ -151,6 +152,8 @@ class Alerting(object):
                                  text=text,
                                  html=html)
             elif alert_data["way"] == "jenkins":
+                self._generate_email_body(alert_data)
+                # TODO: Remove when not needed
                 self._generate_files_for_jenkins(alert_data)
             else:
                 raise AlertingError("Alert with way '{0}' is not implemented.".
@@ -203,82 +206,181 @@ class Alerting(object):
             if smtp_server:
                 smtp_server.quit()
 
-    def _create_alert_message(self, alert):
-        """Create the message which is used in the generated alert.
+    def _get_compressed_failed_tests(self, alert, test_set, sort=True):
+        """Return the dictionary with compressed faild tests. The compression is
+        done by grouping the tests from the same area but with different NICs,
+        frame sizes and number of processor cores.
+
+        For example, the failed tests:
+          10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+
+        will be represented as:
+          ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
+          (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
+
+        Structure of returned data:
+
+        {
+            "trimmed_TC_name_1": {
+                "nics": [],
+                "framesizes": [],
+                "cores": []
+            }
+            ...
+            "trimmed_TC_name_N": {
+                "nics": [],
+                "framesizes": [],
+                "cores": []
+            }
+        }
 
-        :param alert: Message is created for this alert.
+        :param alert: Files are created for this alert.
+        :param test_set: Specifies which set of tests will be included in the
+            result. Its name is the same as the name of file with failed tests.
+        :param sort: If True, the failed tests are sorted alphabetically.
         :type alert: dict
-        :returns: Message in the ASCII text and HTML format.
-        :rtype: tuple(str, str)
+        :type test_set: str
+        :type sort: bool
+        :returns: CSIT build number, VPP version, Number of failed tests,
+            Compressed failed tests.
+        :rtype: tuple(str, str, int, OrderedDict)
         """
 
-        if alert["type"] == "failed-tests":
-            text = ""
-            html = "<html><body>"
-            for item in alert["include"]:
-                file_name = "{path}/{name}".format(
-                    path=self._path_failed_tests, name=item)
-                try:
-                    with open("{0}.txt".format(file_name), 'r') as txt_file:
-                        text += "{0}:\n\n".format(
-                            item.replace("failed-tests-", ""))
-                        text += txt_file.read() + "\n" * 2
-                except IOError:
-                    logging.error("Not possible to read the file '{0}.txt'.".
-                                  format(file_name))
-                try:
-                    with open("{0}.rst".format(file_name), 'r') as rst_file:
-                        html += "<h2>{0}:</h2>".format(
-                            item.replace("failed-tests-", ""))
-                        html += rst_file.readlines()[2].\
-                            replace("../trending", alert.get("url", ""))
-                        html += "<br>" * 3
-                except IOError:
-                    logging.error("Not possible to read the file '{0}.rst'.".
-                                  format(file_name))
-            html += "</body></html>"
+        directory = self.configs[alert["way"]]["output-dir"]
+        failed_tests = OrderedDict()
+        version = ""
+        try:
+            with open("{0}/{1}.txt".format(directory, test_set), 'r') as f_txt:
+                for idx, line in enumerate(f_txt):
+                    if idx == 0:
+                        build = line[:-1]
+                        continue
+                    if idx == 1:
+                        version = line[:-1]
+                        continue
+                    try:
+                        test = line[:-1].split('-')
+                        nic = test[0]
+                        framesize = test[1]
+                        cores = test[2]
+                        name = '-'.join(test[3:-1])
+                    except IndexError:
+                        continue
+                    if failed_tests.get(name, None) is None:
+                        failed_tests[name] = dict(nics=list(),
+                                                  framesizes=list(),
+                                                  cores=list())
+                    if nic not in failed_tests[name]["nics"]:
+                        failed_tests[name]["nics"].append(nic)
+                    if framesize not in failed_tests[name]["framesizes"]:
+                        failed_tests[name]["framesizes"].append(framesize)
+                    if cores not in failed_tests[name]["cores"]:
+                        failed_tests[name]["cores"].append(cores)
+        except IOError as err:
+            logging.error(repr(err))
+            return None, None, None, None
+        if sort:
+            sorted_failed_tests = OrderedDict()
+            keys = [k for k in failed_tests.keys()]
+            keys.sort()
+            for key in keys:
+                sorted_failed_tests[key] = failed_tests[key]
+            return build, version, idx-1, sorted_failed_tests
         else:
+            return build, version, idx-1, failed_tests
+
+    def _generate_email_body(self, alert):
+        """Create the file which is used in the generated alert.
+
+        :param alert: Files are created for this alert.
+        :type alert: dict
+        """
+
+        if alert["type"] != "failed-tests":
             raise AlertingError("Alert of type '{0}' is not implemented.".
                                 format(alert["type"]))
-        return text, html
+
+        config = self.configs[alert["way"]]
+
+        text = ""
+        for idx, test_set in enumerate(alert.get("include", [])):
+            build, version, nr, failed_tests = \
+                self._get_compressed_failed_tests(alert, test_set)
+            if build is None:
+                continue
+            text += ("\n\n{topo}-{arch}, "
+                     "{nr} tests failed, "
+                     "CSIT build: {link}/{build}, "
+                     "VPP version: {version}\n\n".
+                     format(topo=test_set.split('-')[-2],
+                            arch=test_set.split('-')[-1],
+                            nr=nr,
+                            link=alert["urls"][idx],
+                            build=build,
+                            version=version))
+            max_len_name = 0
+            max_len_nics = 0
+            max_len_framesizes = 0
+            max_len_cores = 0
+            for name, params in failed_tests.items():
+                failed_tests[name]["nics"] = ",".join(sorted(params["nics"]))
+                failed_tests[name]["framesizes"] = \
+                    ",".join(sorted(params["framesizes"]))
+                failed_tests[name]["cores"] = ",".join(sorted(params["cores"]))
+                if len(name) > max_len_name:
+                    max_len_name = len(name)
+                if len(failed_tests[name]["nics"]) > max_len_nics:
+                    max_len_nics = len(failed_tests[name]["nics"])
+                if len(failed_tests[name]["framesizes"]) > max_len_framesizes:
+                    max_len_framesizes = len(failed_tests[name]["framesizes"])
+                if len(failed_tests[name]["cores"]) > max_len_cores:
+                    max_len_cores = len(failed_tests[name]["cores"])
+
+            for name, params in failed_tests.items():
+                text += "{name}  {nics}  {frames}  {cores}\n".format(
+                    name=name + " " * (max_len_name - len(name)),
+                    nics=params["nics"] +
+                        " " * (max_len_nics - len(params["nics"])),
+                    frames=params["framesizes"] + " " *
+                        (max_len_framesizes - len(params["framesizes"])),
+                    cores=params["cores"] +
+                        " " * (max_len_cores - len(params["cores"])))
+
+        text += "\nFor detailed information visit: {url}\n".\
+            format(url=alert["url-details"])
+        file_name = "{0}/{1}".format(config["output-dir"],
+                                                config["output-file"])
+        logging.info("Writing the file '{0}.txt' ...".format(file_name))
+
+        try:
+            with open("{0}.txt".format(file_name), 'w') as txt_file:
+                txt_file.write(text)
+        except IOError:
+            logging.error("Not possible to write the file '{0}.txt'.".
+                          format(file_name))
 
     def _generate_files_for_jenkins(self, alert):
         """Create the file which is used in the generated alert.
 
+        # TODO: Remove when not needed.
+
         :param alert: Files are created for this alert.
         :type alert: dict
         """
 
         config = self.configs[alert["way"]]
 
-        if alert["type"] == "failed-tests":
-            text, html = self._create_alert_message(alert)
-            file_name = "{0}/{1}".format(config["output-dir"],
-                                         config["output-file"])
-            logging.info("Writing the file '{0}.txt' ...".format(file_name))
-            try:
-                with open("{0}.txt".format(file_name), 'w') as txt_file:
-                    txt_file.write(text)
-            except IOError:
-                logging.error("Not possible to write the file '{0}.txt'.".
-                              format(file_name))
-            logging.info("Writing the file '{0}.html' ...".format(file_name))
-            try:
-                with open("{0}.html".format(file_name), 'w') as html_file:
-                    html_file.write(html)
-            except IOError:
-                logging.error("Not possible to write the file '{0}.html'.".
-                              format(file_name))
-
-            zip_file = config.get("zip-output", None)
-            if zip_file:
-                logging.info("Writing the file '{0}/{1}' ...".
-                             format(config["output-dir"], zip_file))
-                execute_command("tar czvf {dir}/{zip} --directory={dir} "
-                                "{input}.txt {input}.html".
-                                format(dir=config["output-dir"],
-                                       zip=zip_file,
-                                       input=config["output-file"]))
-        else:
-            raise AlertingError("Alert of type '{0}' is not implemented.".
-                                format(alert["type"]))
+        zip_file = config.get("zip-output", None)
+        if zip_file:
+            logging.info("Writing the file '{0}/{1}' ...".
+                         format(config["output-dir"], zip_file))
+            execute_command("tar czvf {dir}/{zip} --directory={dir} "
+                            "{input}.txt".
+                            format(dir=config["output-dir"],
+                                   zip=zip_file,
+                                   input=config["output-file"]))
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index b55e4a545d..cdce5f98c0 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -858,6 +858,51 @@ def table_performance_trending_dashboard_html(table, input_data):
         return
 
 
+def table_last_failed_tests(table, input_data):
+    """Generate the table(s) with algorithm: table_last_failed_tests
+    specified in the specification file.
+
+    :param table: Table to generate.
+    :param input_data: Data to process.
+    :type table: pandas.Series
+    :type input_data: InputData
+    """
+
+    logging.info("  Generating the table {0} ...".
+                 format(table.get("title", "")))
+
+    # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
+    data = input_data.filter_data(table, continue_on_error=True)
+
+    if data is None or data.empty:
+        logging.warn("    No data for the {0} '{1}'.".
+                     format(table.get("type", ""), table.get("title", "")))
+        return
+
+    tbl_list = list()
+    for job, builds in table["data"].items():
+        for build in builds:
+            build = str(build)
+            tbl_list.append(build)
+            tbl_list.append(input_data.metadata(job, build).get("version", ""))
+            for tst_name, tst_data in data[job][build].iteritems():
+                if tst_data["status"] != "FAIL":
+                    continue
+                groups = re.search(REGEX_NIC, tst_data["parent"])
+                if not groups:
+                    continue
+                nic = groups.group(0)
+                tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
+
+    file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
+    logging.info("    Writing file: '{0}'".format(file_name))
+    with open(file_name, "w") as file_handler:
+        for test in tbl_list:
+            file_handler.write(test + '\n')
+
+
 def table_failed_tests(table, input_data):
     """Generate the table(s) with algorithm: table_failed_tests
     specified in the specification file.
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index fe89093369..bf1a359748 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -119,10 +119,16 @@
         type: "failed-tests"
         way: "jenkins"
         include:
-        - "failed-tests-3n-hsw"
-        - "failed-tests-3n-skx"
-        - "failed-tests-2n-skx"
-        url: "https://docs.fd.io/csit/master/trending/trending"
+        - "last-failed-tests-3n-hsw"
+        - "last-failed-tests-3n-skx"
+        - "last-failed-tests-2n-skx"
+        - "last-failed-tests-nf-2n-skx"
+        urls:
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-weekly-master-2n-skx"
+        url-details: "https://docs.fd.io/csit/master/trending/introduction/failures.html"
 
     configurations:
       # Configuration of the email notifications.
@@ -148,19 +154,49 @@
 
   data-sets:
 
+    # Compressed failed tests (last builds)
+    table-last-failed-tests-3n-hsw:
+      csit-vpp-perf-mrr-daily-master:
+        - "lastCompletedBuild"
+    table-last-failed-tests-3n-skx:
+      csit-vpp-perf-mrr-daily-master-3n-skx:
+        - "lastCompletedBuild"
+    table-last-failed-tests-2n-skx:
+      csit-vpp-perf-mrr-daily-master-2n-skx:
+        - "lastCompletedBuild"
+    table-last-failed-tests-nf-2n-skx:
+      csit-vpp-perf-mrr-weekly-master-2n-skx:
+        - "lastCompletedBuild"
+
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 482
+        start: 487
         end: "lastCompletedBuild"
+        skip:
+          - 593
+          - 595
+          - 608
+          - 613
+          - 614
+          - 616
+          - 617
       csit-dpdk-perf-mrr-weekly-master:
         start: 55
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 482
+        start: 487
         end: "lastCompletedBuild"
+        skip:
+          - 593
+          - 595
+          - 608
+          - 613
+          - 614
+          - 616
+          - 617
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
@@ -170,7 +206,7 @@
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 232
+        start: 237
         end: "lastCompletedBuild"
         skip:
           - 356
@@ -203,7 +239,7 @@
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 232
+        start: 237
         end: "lastCompletedBuild"
         skip:
           - 356
@@ -228,6 +264,7 @@
           - 376
           - 377
           - 378
+
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         start: 20
@@ -238,16 +275,48 @@
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 232
+        start: 237
         end: "lastCompletedBuild"
+        skip:
+          - 347
+          - 358
+          - 359
+          - 360
+          - 361
+          - 362
+          - 363
+          - 364
+          - 365
+          - 366
+          - 367
+          - 368
+          - 375
+          - 380
+          - 381
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         start: 20
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 232
+        start: 237
         end: "lastCompletedBuild"
+        skip:
+          - 347
+          - 358
+          - 359
+          - 360
+          - 361
+          - 362
+          - 363
+          - 364
+          - 365
+          - 366
+          - 367
+          - 368
+          - 375
+          - 380
+          - 381
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
@@ -352,15 +421,23 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 482
+      start: 487
       end: "lastCompletedBuild"
+      skip:
+        - 593
+        - 595
+        - 608
+        - 613
+        - 614
+        - 616
+        - 617
     csit-dpdk-perf-mrr-weekly-master:
       start: 55
       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 232
+      start: 237
       end: "lastCompletedBuild"
       skip:
         - 356
@@ -393,8 +470,24 @@
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 232
+      start: 237
       end: "lastCompletedBuild"
+      skip:
+        - 347
+        - 358
+        - 359
+        - 360
+        - 361
+        - 362
+        - 363
+        - 364
+        - 365
+        - 366
+        - 367
+        - 368
+        - 375
+        - 380
+        - 381
     csit-vpp-perf-mrr-weekly-master-2n-skx:
       start: 1
       end: "lastCompletedBuild"
@@ -418,6 +511,60 @@
 ################################################################################
 
 ################################################################################
+
+# Compressed failed tests (last build)
+-
+  type: "table"
+  title: "Last failed tests (last builds) 3n-hsw"
+  algorithm: "table_last_failed_tests"
+  output-file-ext: ".txt"
+  output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-hsw"
+  data: "table-last-failed-tests-3n-hsw"
+  filter: "'MRR'"
+  parameters:
+    - "name"
+    - "parent"
+    - "status"
+
+-
+  type: "table"
+  title: "Last failed tests (last builds) 3n-skx"
+  algorithm: "table_last_failed_tests"
+  output-file-ext: ".txt"
+  output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-skx"
+  data: "table-last-failed-tests-3n-skx"
+  filter: "'MRR'"
+  parameters:
+    - "name"
+    - "parent"
+    - "status"
+
+-
+  type: "table"
+  title: "Last failed tests (last builds) 2n-skx"
+  algorithm: "table_last_failed_tests"
+  output-file-ext: ".txt"
+  output-file: "{DIR[STATIC,VPP]}/last-failed-tests-2n-skx"
+  data: "table-last-failed-tests-2n-skx"
+  filter: "'MRR'"
+  parameters:
+    - "name"
+    - "parent"
+    - "status"
+
+-
+  type: "table"
+  title: "Last failed tests (last builds) NF 2n-skx"
+  algorithm: "table_last_failed_tests"
+  output-file-ext: ".txt"
+  output-file: "{DIR[STATIC,VPP]}/last-failed-tests-nf-2n-skx"
+  data: "table-last-failed-tests-nf-2n-skx"
+  filter: "'MRR'"
+  parameters:
+    - "name"
+    - "parent"
+    - "status"
+
 # 3n-hsw
 -
   type: "table"
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index ae566c67fa..53649609dd 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -434,6 +434,13 @@ class Specification(object):
                         builds = [x for x in range(builds["start"], build_end+1)
                                   if x not in builds.get("skip", list())]
                         self.configuration["data-sets"][set_name][job] = builds
+                    elif isinstance(builds, list):
+                        for idx, item in enumerate(builds):
+                            try:
+                                builds[idx] = int(item)
+                            except ValueError:
+                                # defined as a range <build_type>
+                                builds[idx] = self._get_build_number(job, item)
 
         # Data sets: add sub-sets to sets (only one level):
         for set_name, data_set in self.configuration["data-sets"].items():
author	Tibor Frank <tifrank@cisco.com>	2019-04-01 16:31:38 +0200
committer	Tibor Frank <tifrank@cisco.com>	2019-04-04 11:39:23 +0200
commit	d45c3f997201a6f17b39b3e5ceb82b332852149f (patch)
tree	845236245b530243b178271d039160860704e1a9 /resources/tools/presentation
parent	ab635019fd3d5420a31fbd68c1549d0be2323104 (diff)