CSIT-1131: Alerting

- CSIT-1132: Send e-mail with a list of failed tests
- CSIT-1288: Prepare data to be sent by Jenkins

Change-Id: I7ac720dca44d7c13b22218abbca7a00d36d459cb
Signed-off-by: Tibor Frank <tifrank@cisco.com>

4 files changed, 356 insertions, 6 deletions

diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
new file mode 100644
index 0000000000..71913eb0b5
--- /dev/null
+++ b/resources/tools/presentation/generator_alerts.py
@@ -0,0 +1,267 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import smtplib
+import logging
+
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from os.path import isdir
+
+from errors import PresentationError
+
+
+class AlertingError(PresentationError):
+    """Exception(s) raised by the alerting module.
+
+    When raising this exception, put this information to the message in this
+    order:
+     - short description of the encountered problem (parameter msg),
+     - relevant messages if there are any collected, e.g., from caught
+       exception (optional parameter details),
+     - relevant data if there are any collected (optional parameter details).
+    """
+
+    def __init__(self, msg, details='', level="CRITICAL"):
+        """Sets the exception message and the level.
+
+        :param msg: Short description of the encountered problem.
+        :param details: Relevant messages if there are any collected, e.g.,
+            from caught exception (optional parameter details), or relevant data
+            if there are any collected (optional parameter details).
+        :param level: Level of the error, possible choices are: "DEBUG", "INFO",
+            "WARNING", "ERROR" and "CRITICAL".
+        :type msg: str
+        :type details: str
+        :type level: str
+        """
+
+        super(AlertingError, self).__init__(
+            "Alerting: {0}".format(msg), details, level)
+
+    def __repr__(self):
+        return (
+            "AlertingError(msg={msg!r},details={dets!r},level={level!r})".
+            format(msg=self._msg, dets=self._details, level=self._level))
+
+
+class Alerting(object):
+    """Class implementing the alerting mechanism.
+    """
+
+    def __init__(self, spec):
+        """Initialization.
+
+        :param spec: The CPTA specification.
+        :type spec: Specification
+        """
+
+        # Implemented alerts:
+        self._ALERTS = ("failed-tests", )
+
+        self._spec = spec.alerting
+        self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
+
+        # Verify and validate input specification:
+        self.configs = self._spec.get("configurations", None)
+        if not self.configs:
+            raise AlertingError("No alert configuration is specified.")
+        for config_type, config_data in self.configs.iteritems():
+            if config_type == "email":
+                if not config_data.get("server", None):
+                    raise AlertingError("Parameter 'server' is missing.")
+                if not config_data.get("address-to", None):
+                    raise AlertingError("Parameter 'address-to' (recipient) is "
+                                        "missing.")
+                if not config_data.get("address-from", None):
+                    raise AlertingError("Parameter 'address-from' (sender) is "
+                                        "missing.")
+            elif config_type == "jenkins":
+                if not isdir(config_data.get("output-dir", "")):
+                    raise AlertingError("Parameter 'output-dir' is "
+                                        "missing or it is not a directory.")
+                if not config_data.get("output-file", None):
+                    raise AlertingError("Parameter 'output-file' is missing.")
+            else:
+                raise AlertingError("Alert of type '{0}' is not implemented.".
+                                    format(config_type))
+
+        self.alerts = self._spec.get("alerts", None)
+        if not self.alerts:
+            raise AlertingError("No alert is specified.")
+        for alert, alert_data in self.alerts.iteritems():
+            if not alert_data.get("title", None):
+                raise AlertingError("Parameter 'title' is missing.")
+            if not alert_data.get("type", None) in self._ALERTS:
+                raise AlertingError("Parameter 'failed-tests' is missing or "
+                                    "incorrect.")
+            if not alert_data.get("way", None) in self.configs.keys():
+                raise AlertingError("Parameter 'way' is missing or incorrect.")
+            if not alert_data.get("include", None):
+                raise AlertingError("Parameter 'include' is missing or the "
+                                    "list is empty.")
+
+    def __str__(self):
+        """Return string with human readable description of the alert.
+
+        :returns: Readable description.
+        :rtype: str
+        """
+        return "configs={configs}, alerts={alerts}".format(
+            configs=self.configs, alerts=self.alerts)
+
+    def __repr__(self):
+        """Return string executable as Python constructor call.
+
+        :returns: Executable constructor call.
+        :rtype: str
+        """
+        return "Alerting(spec={spec})".format(
+            spec=self._spec)
+
+    def generate_alerts(self):
+        """Generate alert(s) using specified way(s).
+        """
+
+        for alert, alert_data in self.alerts.iteritems():
+            if alert_data["way"] == "email":
+                text, html = self._create_alert_message(alert_data)
+                conf = self.configs["email"]
+                self._send_email(server=conf["server"],
+                                 addr_from=conf["address-from"],
+                                 addr_to=conf["address-to"],
+                                 subject=alert_data["title"],
+                                 text=text,
+                                 html=html)
+            elif alert_data["way"] == "jenkins":
+                self._generate_files_for_jenkins(alert_data)
+            else:
+                raise AlertingError("Alert with way '{0}' is not implemented.".
+                                    format(alert_data["way"]))
+
+    @staticmethod
+    def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
+        """Send an email using predefined configuration.
+
+        :param server: SMTP server used to send email.
+        :param addr_from: Sender address.
+        :param addr_to: Recipient address(es).
+        :param subject: Subject of the email.
+        :param text: Message in the ASCII text format.
+        :param html: Message in the HTML format.
+        :type server: str
+        :type addr_from: str
+        :type addr_to: list
+        :type subject: str
+        :type text: str
+        :type html: str
+        """
+
+        if not text and not html:
+            raise AlertingError("No text/data to send.")
+
+        msg = MIMEMultipart('alternative')
+        msg['Subject'] = subject
+        msg['From'] = addr_from
+        msg['To'] = ", ".join(addr_to)
+
+        if text:
+            msg.attach(MIMEText(text, 'plain'))
+        if html:
+            msg.attach(MIMEText(html, 'html'))
+
+        smtp_server = None
+        try:
+            logging.info("Trying to send alert '{0}' ...".format(subject))
+            logging.debug("SMTP Server: {0}".format(server))
+            logging.debug("From: {0}".format(addr_from))
+            logging.debug("To: {0}".format(", ".join(addr_to)))
+            logging.debug("Message: {0}".format(msg.as_string()))
+            smtp_server = smtplib.SMTP(server)
+            smtp_server.sendmail(addr_from, addr_to, msg.as_string())
+        except smtplib.SMTPException as err:
+            raise AlertingError("Not possible to send the alert via email.",
+                                str(err))
+        finally:
+            if smtp_server:
+                smtp_server.quit()
+
+    def _create_alert_message(self, alert):
+        """Create the message which is used in the generated alert.
+
+        :param alert: Message is created for this alert.
+        :type alert: dict
+        :returns: Message in the ASCII text and HTML format.
+        :rtype: tuple(str, str)
+        """
+
+        if alert["type"] == "failed-tests":
+            text = ""
+            html = "<html><body>"
+            for item in alert["include"]:
+                file_name = "{path}/{name}".format(
+                    path=self._path_failed_tests, name=item)
+                try:
+                    with open("{0}.txt".format(file_name), 'r') as txt_file:
+                        text += "{0}:\n\n".format(
+                            item.replace("failed-tests-", ""))
+                        text += txt_file.read() + "\n" * 2
+                except IOError:
+                    logging.error("Not possible to read the file '{0}.txt'.".
+                                  format(file_name))
+                try:
+                    with open("{0}.rst".format(file_name), 'r') as rst_file:
+                        html += "<h2>{0}:</h2>".format(
+                            item.replace("failed-tests-", ""))
+                        html += rst_file.readlines()[2].\
+                            replace("../trending", alert.get("url", ""))
+                        html += "<br>" * 3
+                except IOError:
+                    logging.error("Not possible to read the file '{0}.rst'.".
+                                  format(file_name))
+            html += "</body></html>"
+        else:
+            raise AlertingError("Alert of type '{0}' is not implemented.".
+                                format(alert["type"]))
+        return text, html
+
+    def _generate_files_for_jenkins(self, alert):
+        """Create the file which is used in the generated alert.
+
+        :param alert: Files are created for this alert.
+        :type alert: dict
+        """
+
+        config = self.configs[alert["way"]]
+
+        if alert["type"] == "failed-tests":
+            text, html = self._create_alert_message(alert)
+            file_name = "{0}/{1}".format(config["output-dir"],
+                                         config["output-file"])
+            logging.info("Writing the file '{0}.txt' ...".format(file_name))
+            try:
+                with open("{0}.txt".format(file_name), 'w') as txt_file:
+                    txt_file.write(text)
+            except IOError:
+                logging.error("Not possible to write the file '{0}.txt'.".
+                              format(file_name))
+            logging.info("Writing the file '{0}.html' ...".format(file_name))
+            try:
+                with open("{0}.html".format(file_name), 'w') as html_file:
+                    html_file.write(html)
+            except IOError:
+                logging.error("Not possible to write the file '{0}.html'.".
+                              format(file_name))
+        else:
+            raise AlertingError("Alert of type '{0}' is not implemented.".
+                                format(alert["type"]))
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index a6b4d58bcf..72493cb0d3 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -28,6 +28,7 @@ from generator_files import generate_files
 from static_content import prepare_static_content
 from generator_report import generate_report
 from generator_CPTA import generate_cpta
+from generator_alerts import Alerting, AlertingError
 
 
 def parse_args():
@@ -111,14 +112,22 @@ def main():
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))
+            alert = Alerting(spec)
+            alert.generate_alerts()
             logging.info("Successfully finished.")
         ret_code = 0
 
-    except (KeyError, ValueError, PresentationError) as err:
-        logging.info("Finished with an error.")
+    except AlertingError as err:
+        logging.critical("Finished with an alerting error.")
+        logging.critical(repr(err))
+    except PresentationError as err:
+        logging.critical("Finished with an PAL error.")
+        logging.critical(repr(err))
+    except (KeyError, ValueError) as err:
+        logging.critical("Finished with an error.")
         logging.critical(repr(err))
     except Exception as err:
-        logging.info("Finished with an unexpected error.")
+        logging.critical("Finished with an unexpected error.")
         logging.critical(repr(err))
     finally:
         if spec is not None:
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 4c47d1fb66..8c5217bb2f 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -68,12 +68,77 @@
 
   ignore-list: "ignored_tcs.yaml"
 
+  alerting:
+
+    alerts:
+
+# As Jenkins slave is not configured to send emails, this is now only as
+# a working example:
+#
+#      # Send the list of failed tests vie email.
+#      # Pre-requisites:
+#      # - SMTP server is installed on the Jenkins slave
+#      # - SMTP server is configured to send emails. Default configuration is
+#      #   sufficient.
+#      email-failed-tests:
+#        # Title is used in logs and also as the email subject.
+#        title: "Trending: Failed Tests"
+#        # Type of alert.
+#        type: "failed-tests"
+#        # How to send the alert. The used way must be specified in the
+#        # configuration part.
+#        way: "email"
+#        # Data to be included in the alert.
+#        # Here is used the list of tables generated by the function
+#        # "table_failed_tests_html".
+#        include:
+#        - "failed-tests-3n-hsw"
+#        - "failed-tests-3n-skx"
+#        - "failed-tests-2n-skx"
+#        # This url is used in the tables instead of the original one. The aim
+#        # is to make the links usable also from the email.
+#        url: "https://docs.fd.io/csit/master/trending/trending"
+
+      # Jenkins job sends the email with failed tests.
+      # Pre-requisites:
+      # - Jenkins job is configured to send emails in "Post-build Actions" -->
+      #   "Editable Email Notification".
+      jenkins-send-failed-tests:
+        title: "Trending: Failed Tests"
+        type: "failed-tests"
+        way: "jenkins"
+        include:
+        - "failed-tests-3n-hsw"
+        - "failed-tests-3n-skx"
+        - "failed-tests-2n-skx"
+        url: "https://docs.fd.io/csit/master/trending/trending"
+
+    configurations:
+      # Configuration of the email notifications.
+      email:
+        # SMTP server
+        server: "localhost"
+        # List of recipients.
+        address-to:
+        - "csit-report@lists.fd.io"
+        # Sender
+        address-from: "testuser@testserver.com"
+
+      # Configuration of notifications sent by Jenkins.
+      jenkins:
+        # The directory in the workspace where the generated data is stored and
+        # then read by Jenkins job.
+        output-dir: "_build/_static/vpp"
+        # The name of the output files. ASCII text and HTML formats are
+        # generated.
+        output-file: "jenkins-alert-failed-tests"
+
   data-sets:
 
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 100
+        start: 120
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
         start: 3
@@ -81,7 +146,7 @@
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 100
+        start: 120
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
@@ -212,7 +277,7 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 100
+      start: 120
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
       start: 3
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index f994a59b35..83838d8212 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -113,6 +113,15 @@ class Specification(object):
         return self._specification["configuration"]["ignore"]
 
     @property
+    def alerting(self):
+        """Getter - Alerting.
+
+        :returns: Specification of alerts.
+        :rtype: dict
+        """
+        return self._specification["configuration"]["alerting"]
+
+    @property
     def input(self):
         """Getter - specification - inputs.
         - jobs and builds.