CSIT-755: Presentation and analytics layer

- CSIT-760: Configuration - real example
- CSIT-774: Implementation - parse configuration
- CSIT-779: Implementation - set environment
- CSIT-780: Implementation - download data
- CSIT-783: Implementation - debug mode
- CSIT-761: Implementation - Data pre-processing - parse input files
- CSIT-784: Implementation - Data pre-processing - store the data, access to data
- CSIT-789: Implementation - Data pre-processing - extract Documentation of the suite
- CSIT-757: Low Level Design
- CSIT-788: Implementation - Data pre-processing - extract VAT history and show runtime
- CSIT-785: Implementation - Data filtering
- CSIT-763: Presentation - tables
- CSIT-804: Presentation - files
- CSIT-762: Presentation - plots
- LLD: API + functional diagram
- CSIT-807: Element's models
- CSIT-813: Process static content
- CSIT-812: Report generation
- CSIT-764: Integration to CSIT
- CSIT-822: Archiving
- CSIT-790: Documentation
- configuration od the Input data is the same as for 17.07 report

Change-Id: I6fd1eb1df4af99eaf91925282cdee1c892698c59
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 204 insertions, 0 deletions

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
new file mode 100644
index 0000000000..d57d1545ea
--- /dev/null
+++ b/resources/tools/presentation/input_data_files.py
@@ -0,0 +1,204 @@
+# Copyright (c) 2017 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Inputs
+Download all data.
+"""
+
+import logging
+
+from os import rename, remove
+from os.path import join, getsize
+from shutil import move
+from zipfile import ZipFile, is_zipfile, BadZipfile
+from httplib import responses
+from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
+    HTTPError, ConnectionError
+
+from errors import PresentationError
+
+
+# Chunk size used for file download
+CHUNK_SIZE = 512
+
+# Separator used in file names
+SEPARATOR = "__"
+
+
+def download_data_files(spec):
+    """Download all data specified in the specification file in the section
+    type: input --> builds.
+
+    :param spec: Specification.
+    :type spec: Specification
+    :raises: PresentationError if there is no url defined for the job.
+    """
+
+    for job, builds in spec.builds.items():
+        for build in builds:
+            if job.startswith("csit-"):
+                url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
+            elif job.startswith("hc2vpp-"):
+                url = spec.environment["urls"]["URL[JENKINS,HC]"]
+            else:
+                raise PresentationError("No url defined for the job '{}'.".
+                                        format(job))
+            file_name = spec.input["file-name"]
+            full_name = spec.input["download-path"].\
+                format(job=job, build=build["build"], filename=file_name)
+            url = "{0}/{1}".format(url, full_name)
+            new_name = join(
+                spec.environment["paths"]["DIR[WORKING,DATA]"],
+                "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR,
+                                                      build=build["build"],
+                                                      name=file_name))
+
+            logging.info("Downloading the file '{0}' to '{1}'.".
+                         format(url, new_name))
+
+            status = "failed"
+            try:
+                response = get(url, stream=True)
+                code = response.status_code
+                if code != codes["OK"]:
+                    logging.error("{0}: {1}".format(code, responses[code]))
+                    spec.set_input_state(job, build["build"], "not found")
+                    continue
+
+                file_handle = open(new_name, "wb")
+                for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+                    if chunk:
+                        file_handle.write(chunk)
+                file_handle.close()
+
+                expected_length = None
+                try:
+                    expected_length = int(response.headers["Content-Length"])
+                    logging.debug("  Expected file size: {0}B".
+                                  format(expected_length))
+                except KeyError:
+                    logging.debug("  No information about expected size.")
+
+                real_length = getsize(new_name)
+                logging.debug("  Downloaded size: {0}B".format(real_length))
+
+                if expected_length:
+                    if real_length == expected_length:
+                        status = "downloaded"
+                        logging.info("{0}: {1}".format(code, responses[code]))
+                    else:
+                        logging.error("The file size differs from the expected "
+                                      "size.")
+                else:
+                    status = "downloaded"
+                    logging.info("{0}: {1}".format(code, responses[code]))
+
+            except ConnectionError as err:
+                logging.error("Not possible to connect to '{0}'.".format(url))
+                logging.debug(err)
+            except HTTPError as err:
+                logging.error("Invalid HTTP response from '{0}'.".format(url))
+                logging.debug(err)
+            except TooManyRedirects as err:
+                logging.error("Request exceeded the configured number "
+                              "of maximum re-directions.")
+                logging.debug(err)
+            except Timeout as err:
+                logging.error("Request timed out.")
+                logging.debug(err)
+            except RequestException as err:
+                logging.error("Unexpected HTTP request exception.")
+                logging.debug(err)
+            except (IOError, ValueError, KeyError) as err:
+                logging.error("Download failed.")
+                logging.debug("Reason: {0}".format(err))
+
+            spec.set_input_state(job, build["build"], status)
+            spec.set_input_file_name(job, build["build"], new_name)
+
+            if status == "failed":
+                logging.info("Removing the file '{0}'".format(new_name))
+                try:
+                    remove(new_name)
+                except OSError as err:
+                    logging.warning(str(err))
+                spec.set_input_file_name(job, build["build"], None)
+
+    unzip_files(spec)
+
+
+def unzip_files(spec):
+    """Unzip downloaded zip files
+
+    :param spec: Specification.
+    :type spec: Specification
+    :raises: PresentationError if the zip file does not exist or it is not a
+    zip file.
+    """
+
+    if spec.is_debug:
+        data_file = spec.debug["extract"]
+    else:
+        data_file = spec.input["extract"]
+
+    for job, builds in spec.builds.items():
+        for build in builds:
+            if build["status"] == "failed" or build["status"] == "not found":
+                continue
+            try:
+                status = "failed"
+                directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
+                file_name = join(build["file-name"])
+
+                if build["status"] == "downloaded" and is_zipfile(file_name):
+                    logging.info("Unziping: '{0}' from '{1}'.".
+                                 format(data_file, file_name))
+                    new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
+                                                  SEPARATOR,
+                                                  data_file.split("/")[-1])
+                    try:
+                        with ZipFile(file_name, 'r') as zip_file:
+                            zip_file.extract(data_file, directory)
+                        logging.info("Moving {0} to {1} ...".
+                                     format(join(directory, data_file),
+                                            directory))
+                        move(join(directory, data_file), directory)
+                        logging.info("Renaming the file '{0}' to '{1}'".
+                                     format(join(directory,
+                                                 data_file.split("/")[-1]),
+                                            new_name))
+                        rename(join(directory, data_file.split("/")[-1]),
+                               new_name)
+                        status = "unzipped"
+                        spec.set_input_state(job, build["build"], status)
+                        spec.set_input_file_name(job, build["build"],
+                                                   new_name)
+                    except (BadZipfile, RuntimeError) as err:
+                        logging.error("Failed to unzip the file '{0}': {1}.".
+                                      format(file_name, str(err)))
+                    except OSError as err:
+                        logging.error("Failed to rename the file '{0}': {1}.".
+                                      format(data_file, str(err)))
+                    finally:
+                        if status == "failed":
+                            spec.set_input_file_name(job, build["build"],
+                                                       None)
+                else:
+                    raise PresentationError("The file '{0}' does not exist or "
+                                            "it is not a zip file".
+                                            format(file_name))
+
+                spec.set_input_state(job, build["build"], status)
+
+            except KeyError:
+                pass