CSIT-1078: Optimize input data files download and processing

Change-Id: Id1b0af996c4b77878621cfc8c262795f51e537fc
Signed-off-by: Tibor Frank <tifrank@cisco.com>

10 files changed, 256 insertions, 451 deletions

diff --git a/resources/tools/presentation/environment.py b/resources/tools/presentation/environment.py
index 1a1c3db032..a2fa9a0d5b 100644
--- a/resources/tools/presentation/environment.py
+++ b/resources/tools/presentation/environment.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -51,35 +51,6 @@ class Environment(object):
         """
         return self._env
 
-    def _set_environment_variables(self):
-        """Set environment variables.
-        """
-        logging.info("Setting the environment variables ...")
-        # logging.debug("Environment variables before:\n{}".format(os.environ))
-
-        count = 1
-
-        for var, value in self._env["configuration"].items():
-            logging.debug("  {:3d} Setting the variable {} = {}".
-                          format(count, var, value))
-            os.environ[var] = str(value)
-            count += 1
-
-        for var, value in self._env["paths"].items():
-            logging.debug("  {:3d} Setting the variable {} = {}".
-                          format(count, var, value))
-            os.environ[var] = str(value)
-            count += 1
-
-        for var, value in self._env["urls"].items():
-            logging.debug("  {:3d} Setting the variable {} = {}".
-                          format(count, var, value))
-            os.environ[var] = str(value)
-            count += 1
-
-        # logging.debug("Environment variables after:\n{}".format(os.environ))
-        logging.info("Done.")
-
     def _make_dirs(self):
         """Create the directories specified in the 'make-dirs' part of
         'environment' section in the specification file.
@@ -122,7 +93,6 @@ class Environment(object):
         """Set the environment.
         """
 
-        self._set_environment_variables()
         self._make_dirs()
 
 
@@ -147,10 +117,10 @@ def clean_environment(env):
         if os.path.isdir(dir_to_remove):
             try:
                 shutil.rmtree(dir_to_remove)
-            except OSError:
-                pass
-                # raise PresentationError("Cannot remove the directory '{}'".
-                #                         format(dir_to_remove))
+            except OSError as err:
+                logging.warning("Cannot remove the directory '{}'".
+                                format(dir_to_remove))
+                logging.debug(str(err))
         else:
             logging.warning("The directory '{}' does not exist.".
                             format(dir_to_remove))
diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index e27a52172b..89e9a41b48 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -32,7 +32,7 @@ from utils import split_outliers, archive_input_data, execute_command
 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
                '-b html -E ' \
                '-t html ' \
-               '-D version="Generated on {date}" ' \
+               '-D version="{date}" ' \
                '{working_dir} ' \
                '{build_dir}/'
 
@@ -64,7 +64,7 @@ def generate_cpta(spec, data):
     ret_code = _generate_all_charts(spec, data)
 
     cmd = HTML_BUILDER.format(
-        date=datetime.date.today().strftime('%d-%b-%Y'),
+        date=datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
     execute_command(cmd)
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index bbe521948e..1f8abd9716 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -822,9 +822,6 @@ def table_performance_trending_dashboard(table, input_data):
                 rel_change_long = round(
                     ((last_median_t - max_median) / max_median) * 100, 2)
 
-            logging.info("rel_change_last : {}".format(rel_change_last))
-            logging.info("rel_change_long : {}".format(rel_change_long))
-
             tbl_lst.append(
                 [name,
                  '-' if isnan(last_median_t) else
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index d81f64fbe6..24dc3a57d8 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,11 +16,10 @@ Download all data.
 """
 
 import re
-import gzip
 import logging
 
-from os import rename, remove
-from os.path import join, getsize
+from os import rename
+from os.path import join
 from shutil import move
 from zipfile import ZipFile, is_zipfile, BadZipfile
 from httplib import responses
@@ -39,216 +38,179 @@ SEPARATOR = "__"
 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
 
 
-def download_data_files(spec):
-    """Download all data specified in the specification file in the section
-    type: input --> builds.
+def _download_file(url, file_name):
+    """Download a file with input data.
 
-    :param spec: Specification.
+    :param url: URL to the file to download.
+    :param file_name: Name of file to download.
+    :type url: str
+    :type file_name: str
+    :returns: True if the download was successful, otherwise False.
+    :rtype: bool
+    """
+
+    success = False
+    try:
+        logging.info("      Connecting to '{0}' ...".format(url))
+
+        response = get(url, stream=True)
+        code = response.status_code
+
+        logging.info("      {0}: {1}".format(code, responses[code]))
+
+        if code != codes["OK"]:
+            return False
+
+        logging.info("      Downloading the file '{0}' to '{1}' ...".
+                     format(url, file_name))
+
+        file_handle = open(file_name, "wb")
+        for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+            if chunk:
+                file_handle.write(chunk)
+        file_handle.close()
+        success = True
+    except ConnectionError as err:
+        logging.error("Not possible to connect to '{0}'.".format(url))
+        logging.debug(str(err))
+    except HTTPError as err:
+        logging.error("Invalid HTTP response from '{0}'.".format(url))
+        logging.debug(str(err))
+    except TooManyRedirects as err:
+        logging.error("Request exceeded the configured number "
+                      "of maximum re-directions.")
+        logging.debug(str(err))
+    except Timeout as err:
+        logging.error("Request timed out.")
+        logging.debug(str(err))
+    except RequestException as err:
+        logging.error("Unexpected HTTP request exception.")
+        logging.debug(str(err))
+    except (IOError, ValueError, KeyError) as err:
+        logging.error("Download failed.")
+        logging.debug(str(err))
+
+    logging.info("      Download finished.")
+    return success
+
+
+def _unzip_file(spec, job, build):
+    """Unzip downloaded source file.
+
+    :param spec: Specification read form the specification file.
+    :param job: Name of the Jenkins job.
+    :param build: Information about the build.
     :type spec: Specification
-    :raises: PresentationError if there is no url defined for the job.
+    :type job: str
+    :type build: dict
+    :returns: True if the download was successful, otherwise False.
+    :rtype: bool
     """
 
-    for job, builds in spec.builds.items():
-        for build in builds:
-            if job.startswith("csit-"):
-                if spec.input["file-name"].endswith(".zip"):
-                    url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
-                elif spec.input["file-name"].endswith(".gz"):
-                    url = spec.environment["urls"]["URL[NEXUS,LOG]"]
-                else:
-                    logging.error("Not supported file format.")
-                    continue
-            elif job.startswith("hc2vpp-"):
-                url = spec.environment["urls"]["URL[JENKINS,HC]"]
-            else:
-                raise PresentationError("No url defined for the job '{}'.".
-                                        format(job))
-            file_name = spec.input["file-name"]
-            full_name = spec.input["download-path"].\
-                format(job=job, build=build["build"], filename=file_name)
-            url = "{0}/{1}".format(url, full_name)
-            new_name = join(
-                spec.environment["paths"]["DIR[WORKING,DATA]"],
-                "{job}{sep}{build}{sep}{name}".format(job=job,
-                                                      sep=SEPARATOR,
-                                                      build=build["build"],
-                                                      name=file_name))
-            logging.info(
-                "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
-
-            status = "failed"
-            try:
-                response = get(url, stream=True)
-                code = response.status_code
-
-                if code != codes["OK"]:
-                    logging.warning(
-                        "Jenkins: {0}: {1}.".format(code, responses[code]))
-                    logging.info("Trying to download from Nexus:")
-                    spec.set_input_state(job, build["build"], "not found")
-                    if code == codes["not_found"]:
-                        release = re.search(REGEX_RELEASE, job).group(2)
-                        nexus_file_name = "{job}{sep}{build}{sep}{name}".\
-                            format(job=job, sep=SEPARATOR, build=build["build"],
-                                   name=file_name)
-                        try:
-                            release = "rls{0}".format(int(release))
-                        except ValueError:
-                            pass
-                        url = "{url}/{release}/{dir}/{file}".\
-                            format(url=spec.environment["urls"]["URL[NEXUS]"],
-                                   release=release,
-                                   dir=spec.environment["urls"]["DIR[NEXUS]"],
-                                   file=nexus_file_name)
-                        logging.info("Downloading the file '{0}' to '{1}' ...".
-                                     format(url, new_name))
-                        response = get(url, stream=True)
-                        code = response.status_code
-                        if code != codes["OK"]:
-                            logging.error(
-                                "Nexus: {0}: {1}".format(code, responses[code]))
-                            spec.set_input_state(
-                                job, build["build"], "not found")
-                            continue
-
-                file_handle = open(new_name, "wb")
-                for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
-                    if chunk:
-                        file_handle.write(chunk)
-                file_handle.close()
-
-                if spec.input["file-name"].endswith(".zip"):
-                    expected_length = None
-                    try:
-                        expected_length = int(response.
-                                              headers["Content-Length"])
-                        logging.debug("  Expected file size: {0}B".
-                                      format(expected_length))
-                    except KeyError:
-                        logging.debug("  No information about expected size.")
-
-                    real_length = getsize(new_name)
-                    logging.debug("  Downloaded size: {0}B".format(real_length))
-
-                    if expected_length:
-                        if real_length == expected_length:
-                            status = "downloaded"
-                            logging.info("{0}: {1}".format(code,
-                                                           responses[code]))
-                        else:
-                            logging.error("The file size differs from the "
-                                          "expected size.")
-                    else:
-                        status = "downloaded"
-                        logging.info("{0}: {1}".format(code, responses[code]))
-
-                elif spec.input["file-name"].endswith(".gz"):
-                    if "docs.fd.io" in url:
-                        execute_command("gzip --decompress --keep --force {0}".
-                                        format(new_name))
-                    else:
-                        rename(new_name, new_name[:-3])
-                        execute_command("gzip --keep {0}".format(new_name[:-3]))
-                    new_name = new_name[:-3]
-                    status = "downloaded"
-                    logging.info("{0}: {1}".format(code, responses[code]))
-
-            except ConnectionError as err:
-                logging.error("Not possible to connect to '{0}'.".format(url))
-                logging.debug(err)
-            except HTTPError as err:
-                logging.error("Invalid HTTP response from '{0}'.".format(url))
-                logging.debug(err)
-            except TooManyRedirects as err:
-                logging.error("Request exceeded the configured number "
-                              "of maximum re-directions.")
-                logging.debug(err)
-            except Timeout as err:
-                logging.error("Request timed out.")
-                logging.debug(err)
-            except RequestException as err:
-                logging.error("Unexpected HTTP request exception.")
-                logging.debug(err)
-            except (IOError, ValueError, KeyError) as err:
-                logging.error("Download failed.")
-                logging.debug("Reason: {0}".format(err))
-
-            spec.set_input_state(job, build["build"], status)
-            spec.set_input_file_name(job, build["build"], new_name)
-
-            if status == "failed":
-                logging.info("Removing the file '{0}'".format(new_name))
-                try:
-                    remove(new_name)
-                except OSError as err:
-                    logging.warning(str(err))
-                spec.set_input_file_name(job, build["build"], None)
-
-    unzip_files(spec)
-
-
-def unzip_files(spec):
-    """Unzip downloaded zip files
-
-    :param spec: Specification.
+    data_file = spec.input["extract"]
+    file_name = build["file-name"]
+    directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
+    new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
+                                  SEPARATOR,
+                                  data_file.split("/")[-1])
+    logging.info("      Unzipping: '{0}' from '{1}'.".
+                 format(data_file, file_name))
+    try:
+        with ZipFile(file_name, 'r') as zip_file:
+            zip_file.extract(data_file, directory)
+        logging.info("      Moving {0} to {1} ...".
+                     format(join(directory, data_file), directory))
+        move(join(directory, data_file), directory)
+        logging.info("      Renaming the file '{0}' to '{1}'".
+                     format(join(directory, data_file.split("/")[-1]),
+                            new_name))
+        rename(join(directory, data_file.split("/")[-1]),
+               new_name)
+        spec.set_input_file_name(job, build["build"],
+                                 new_name)
+        return True
+    except (BadZipfile, RuntimeError) as err:
+        logging.error("Failed to unzip the file '{0}': {1}.".
+                      format(file_name, str(err)))
+        return False
+    except OSError as err:
+        logging.error("Failed to rename the file '{0}': {1}.".
+                      format(data_file, str(err)))
+        return False
+
+
+def download_and_unzip_data_file(spec, job, build):
+    """Download and unzip a source file.
+
+    :param spec: Specification read form the specification file.
+    :param job: Name of the Jenkins job.
+    :param build: Information about the build.
     :type spec: Specification
-    :raises: PresentationError if the zip file does not exist or it is not a
-    zip file.
+    :type job: str
+    :type build: dict
+    :returns: True if the download was successful, otherwise False.
+    :rtype: bool
     """
 
-    if spec.is_debug:
-        data_file = spec.debug["extract"]
+    if job.startswith("csit-"):
+        if spec.input["file-name"].endswith(".zip"):
+            url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
+        elif spec.input["file-name"].endswith(".gz"):
+            url = spec.environment["urls"]["URL[NEXUS,LOG]"]
+        else:
+            logging.error("Not supported file format.")
+            return False
+    elif job.startswith("hc2vpp-"):
+        url = spec.environment["urls"]["URL[JENKINS,HC]"]
+    else:
+        raise PresentationError("No url defined for the job '{}'.".
+                                format(job))
+    file_name = spec.input["file-name"]
+    full_name = spec.input["download-path"]. \
+        format(job=job, build=build["build"], filename=file_name)
+    url = "{0}/{1}".format(url, full_name)
+    new_name = join(spec.environment["paths"]["DIR[WORKING,DATA]"],
+                    "{job}{sep}{build}{sep}{name}".
+                    format(job=job, sep=SEPARATOR, build=build["build"],
+                           name=file_name))
+    # Download the file from the defined source (Jenkins, logs.fd.io):
+    success = _download_file(url, new_name)
+
+    # If not successful, download from docs.fd.io:
+    if not success:
+        logging.info("      Trying to download from https://docs.fd.io:")
+        release = re.search(REGEX_RELEASE, job).group(2)
+        nexus_file_name = "{job}{sep}{build}{sep}{name}". \
+            format(job=job, sep=SEPARATOR, build=build["build"], name=file_name)
+        try:
+            release = "rls{0}".format(int(release))
+        except ValueError:
+            pass
+        url = "{url}/{release}/{dir}/{file}". \
+            format(url=spec.environment["urls"]["URL[NEXUS]"],
+                   release=release,
+                   dir=spec.environment["urls"]["DIR[NEXUS]"],
+                   file=nexus_file_name)
+        success = _download_file(url, new_name)
+
+    if success:
+        spec.set_input_file_name(job, build["build"], new_name)
+    else:
+        return False
+
+    if spec.input["file-name"].endswith(".gz"):
+        if "docs.fd.io" in url:
+            execute_command("gzip --decompress --keep --force {0}".
+                            format(new_name))
+        else:
+            rename(new_name, new_name[:-3])
+            execute_command("gzip --keep {0}".format(new_name[:-3]))
+        spec.set_input_file_name(job, build["build"], new_name[:-3])
+
+    if spec.input["file-name"].endswith(".zip"):
+        if is_zipfile(file_name):
+            return _unzip_file(spec, job, build)
+        else:
+            return False
     else:
-        data_file = spec.input["extract"]
-
-    for job, builds in spec.builds.items():
-        for build in builds:
-            if build["status"] == "failed" or build["status"] == "not found":
-                continue
-            try:
-                status = "failed"
-                directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
-                file_name = join(build["file-name"])
-
-                if build["status"] == "downloaded":
-                    logging.info("Unziping: '{0}' from '{1}'.".
-                                 format(data_file, file_name))
-                    new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
-                                                  SEPARATOR,
-                                                  data_file.split("/")[-1])
-                    try:
-                        if is_zipfile(file_name):
-                            with ZipFile(file_name, 'r') as zip_file:
-                                zip_file.extract(data_file, directory)
-                            logging.info("Moving {0} to {1} ...".
-                                         format(join(directory, data_file),
-                                                directory))
-                            move(join(directory, data_file), directory)
-                            logging.info("Renaming the file '{0}' to '{1}'".
-                                         format(join(directory,
-                                                     data_file.split("/")[-1]),
-                                                new_name))
-                            rename(join(directory, data_file.split("/")[-1]),
-                                   new_name)
-                            spec.set_input_file_name(job, build["build"],
-                                                     new_name)
-                        status = "unzipped"
-                        spec.set_input_state(job, build["build"], status)
-                    except (BadZipfile, RuntimeError) as err:
-                        logging.error("Failed to unzip the file '{0}': {1}.".
-                                      format(file_name, str(err)))
-                    except OSError as err:
-                        logging.error("Failed to rename the file '{0}': {1}.".
-                                      format(data_file, str(err)))
-                    finally:
-                        if status == "failed":
-                            spec.set_input_file_name(job, build["build"], None)
-                else:
-                    raise PresentationError("The file '{0}' does not exist or "
-                                            "it is not a zip file".
-                                            format(file_name))
-
-                spec.set_input_state(job, build["build"], status)
-
-            except KeyError:
-                pass
+        return True
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 0ad07a95db..e12e2fb8df 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -29,6 +29,8 @@ from collections import OrderedDict
 from string import replace
 from os import remove
 
+from input_data_files import download_and_unzip_data_file
+
 
 class ExecutionChecker(ResultVisitor):
     """Class to traverse through the test suite structure.
@@ -179,7 +181,7 @@ class ExecutionChecker(ResultVisitor):
     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
                            r'tx\s(\d*),\srx\s(\d*)')
 
-    def __init__(self, **metadata):
+    def __init__(self, metadata):
         """Initialisation.
 
         :param metadata: Key-value pairs to be included in "metadata" part of
@@ -251,8 +253,6 @@ class ExecutionChecker(ResultVisitor):
             self._data["metadata"]["version"] = self._version
             self._msg_type = None
 
-            logging.info("    VPP version: {0}".format(self._version))
-
     def _get_vat_history(self, msg):
         """Called when extraction of VAT command history is required.
 
@@ -748,21 +748,29 @@ class InputData(object):
         return self.data[job][build]["tests"]
 
     @staticmethod
-    def _parse_tests(job, build):
+    def _parse_tests(job, build, get_timestamp=False):
         """Process data from robot output.xml file and return JSON structured
         data.
 
         :param job: The name of job which build output data will be processed.
         :param build: The build which output data will be processed.
+        :param get_timestamp: If True, timestamp is read form the xml source
+            file.
         :type job: str
         :type build: dict
+        :type get_timestamp: bool
         :returns: JSON data structure.
         :rtype: dict
         """
 
-        tree = ET.parse(build["file-name"])
-        root = tree.getroot()
-        generated = root.attrib["generated"]
+        metadata = {
+            "job": job,
+            "build": build
+        }
+        if get_timestamp:
+            tree = ET.parse(build["file-name"])
+            root = tree.getroot()
+            metadata["generated"] = root.attrib["generated"]
 
         with open(build["file-name"], 'r') as data_file:
             try:
@@ -771,46 +779,55 @@ class InputData(object):
                 logging.error("Error occurred while parsing output.xml: {0}".
                               format(err))
                 return None
-        checker = ExecutionChecker(job=job, build=build, generated=generated)
+        checker = ExecutionChecker(metadata)
         result.visit(checker)
 
         return checker.data
 
-    def read_data(self):
-        """Parse input data from input files and store in pandas' Series.
+    def download_and_parse_data(self, get_timestamp=False):
+        """Download the input data files, parse input data from input files and
+        store in pandas' Series.
+
+        :param get_timestamp: If True, timestamp is read form the xml source
+            file.
+        :type get_timestamp: bool
         """
 
-        logging.info("Parsing input files ...")
+        logging.info("Downloading and parsing input files ...")
 
         job_data = dict()
         for job, builds in self._cfg.builds.items():
-            logging.info("  Extracting data from the job '{0}' ...'".
+            logging.info("  Processing data from the job '{0}' ...'".
                          format(job))
             builds_data = dict()
             for build in builds:
-                if build["status"] == "failed" \
-                        or build["status"] == "not found":
+                logging.info("    Processing the build '{0}'".
+                             format(build["build"]))
+                self._cfg.set_input_state(job, build["build"], "failed")
+                if not download_and_unzip_data_file(self._cfg, job, build):
+                    logging.error("It is not possible to download the input "
+                                  "data file from the job '{job}', build "
+                                  "'{build}', or it is damaged. Skipped.".
+                                  format(job=job, build=build["build"]))
                     continue
-                logging.info("    Extracting data from the build '{0}'".
+
+                logging.info("      Processing data from the build '{0}' ...".
                              format(build["build"]))
-                logging.info("    Processing the file '{0}'".
-                             format(build["file-name"]))
+                data = InputData._parse_tests(job, build,
+                                              get_timestamp=get_timestamp)
+                if data is None:
+                    logging.error("Input data file from the job '{job}', build "
+                                  "'{build}' is damaged. Skipped.".
+                                  format(job=job, build=build["build"]))
+                    continue
 
-                data = InputData._parse_tests(job, build)
+                self._cfg.set_input_state(job, build["build"], "processed")
 
-                logging.info("    Removing the file '{0}'".
-                             format(build["file-name"]))
                 try:
                     remove(build["file-name"])
-                    build["status"] = "processed"
                 except OSError as err:
-                    logging.error("   Cannot remove the file '{0}': {1}".
+                    logging.error("Cannot remove the file '{0}': {1}".
                                   format(build["file-name"], err))
-                if data is None:
-                    logging.error("Input data file from the job '{job}', build "
-                                  "'{build}' is damaged. Skipped.".
-                                  format(job=job, build=build["build"]))
-                    continue
 
                 build_data = pd.Series({
                     "metadata": pd.Series(data["metadata"].values(),
@@ -818,9 +835,9 @@ class InputData(object):
                     "suites": pd.Series(data["suites"].values(),
                                         index=data["suites"].keys()),
                     "tests": pd.Series(data["tests"].values(),
-                                       index=data["tests"].keys()),
-                    })
+                                       index=data["tests"].keys())})
                 builds_data[str(build["build"])] = build_data
+                build["status"] = "processed"
                 logging.info("    Done.")
 
             job_data[job] = pd.Series(builds_data.values(),
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 98642c898c..2268801407 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -21,7 +21,6 @@ import logging
 from errors import PresentationError
 from environment import Environment, clean_environment
 from specification_parser import Specification
-from input_data_files import download_data_files, unzip_files
 from input_data_parser import InputData
 from generator_tables import generate_tables
 from generator_plots import generate_plots
@@ -30,8 +29,6 @@ from static_content import prepare_static_content
 from generator_report import generate_report
 from generator_CPTA import generate_cpta
 
-from pprint import pprint
-
 
 def parse_args():
     """Parse arguments from cmd line.
@@ -86,21 +83,24 @@ def main():
         logging.critical("Finished with error.")
         return 1
 
-    ret_code = 0
+    if spec.output["output"] not in ("report", "CPTA"):
+        logging.critical("The output '{0}' is not supported.".
+                         format(spec.output["output"]))
+        return 1
+
+    ret_code = 1
     try:
         env = Environment(spec.environment, args.force)
         env.set_environment()
 
-        if spec.is_debug:
-            if spec.debug["input-format"] == "zip":
-                unzip_files(spec)
-        else:
-            download_data_files(spec)
-
         prepare_static_content(spec)
 
         data = InputData(spec)
-        data.read_data()
+
+        if spec.output["output"] == "report":
+            data.download_and_parse_data(get_timestamp=False)
+        elif spec.output["output"] == "CPTA":
+            data.download_and_parse_data(get_timestamp=True)
 
         generate_tables(spec, data)
         generate_plots(spec, data)
@@ -112,21 +112,16 @@ def main():
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))
             logging.info("Successfully finished.")
-        else:
-            logging.critical("The output '{0}' is not supported.".
-                             format(spec.output["output"]))
-            ret_code = 1
-
-    except (KeyError, ValueError, PresentationError) as err:
-        logging.info("Finished with an error.")
-        logging.critical(str(err))
-        ret_code = 1
-    except Exception as err:
-        logging.info("Finished with an unexpected error.")
-        logging.critical(str(err))
-        ret_code = 1
+        ret_code = 0
+
+    # except (KeyError, ValueError, PresentationError) as err:
+    #     logging.info("Finished with an error.")
+    #     logging.critical(str(err))
+    # except Exception as err:
+    #     logging.info("Finished with an unexpected error.")
+    #     logging.critical(str(err))
     finally:
-        if spec is not None and not spec.is_debug:
+        if spec is not None:
             clean_environment(spec.environment)
         return ret_code
 
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 4b055a84ce..97e616a15d 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -1,14 +1,5 @@
 -
   type: "environment"
-  configuration:
-    # Debug mode:
-    # - Skip:
-    #   - Download of input data files
-    # - Do:
-    #   - Read data from given zip / xml files
-    #   - Set the configuration as it is done in normal mode
-    # If the section "type: debug" is missing, CFG[DEBUG] is set to 0.
-    CFG[DEBUG]: 0
 
   paths:
     # Top level directories:
@@ -546,27 +537,6 @@
       height: 1000
 
 -
-  type: "debug"
-  general:
-    input-format: "xml"  # zip or xml
-    extract: "robot-plugin/output.xml"  # Only for zip
-  builds:
-    # The files must be in the directory DIR[WORKING,DATA]
-    csit-vpp-perf-1801-all:
-    -
-      build: 1
-      file: "{DIR[WORKING,DATA]}/output.xml"
-    -
-      build: 2
-      file: "{DIR[WORKING,DATA]}/output.xml"
-    -
-      build: 3
-      file: "{DIR[WORKING,DATA]}/output.xml"
-    -
-      build: 4
-      file: "{DIR[WORKING,DATA]}/output.xml"
-
--
   type: "static"
   src-path: "{DIR[RST]}"
   dst-path: "{DIR[WORKING,SRC]}"
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index a570dccbbe..9b47083b2b 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -16,15 +16,6 @@
 
 -
   type: "environment"
-  configuration:
-    # Debug mode:
-    # - Skip:
-    #   - Download of input data files
-    # - Do:
-    #   - Read data from given zip / xml files
-    #   - Set the configuration as it is done in normal mode
-    # If the section "type: debug" is missing, CFG[DEBUG] is set to 0.
-    CFG[DEBUG]: 0
 
   paths:
     # Top level directories:
@@ -102,7 +93,6 @@
         showgrid: True
       xaxis:
         showticklabels: True
-        # title: "csit-vpp-perf-mrr-daily-master/vpp-build"
         title: '<a href="https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master" target="_blank">csit-vpp-perf-mrr-daily-master-build</a>/<a href="https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/vpp/" target="_blank">vpp-build</a>'
         autorange: True
         showgrid: True
@@ -130,66 +120,22 @@
         namelength: -1
 
 -
-  type: "debug"
-  general:
-    input-format: "xml"  # zip or xml
-    extract: "robot-plugin/output.xml"  # Only for zip
-  builds:
-    # The files must be in the directory DIR[WORKING,DATA]
-    csit-vpp-perf-mrr-daily-master:
-    -
-      build: 1
-      file: "{DIR[WORKING,DATA]}/output_mrr_1.xml"
-    -
-      build: 2
-      file: "{DIR[WORKING,DATA]}/output_mrr_2.xml"
-    -
-      build: 3
-      file: "{DIR[WORKING,DATA]}/output_mrr_3.xml"
-    -
-      build: 4
-      file: "{DIR[WORKING,DATA]}/output_mrr_4.xml"
-    -
-      build: 5
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 6
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 7
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 8
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 9
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 10
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 11
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-    -
-      build: 12
-      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-
--
   type: "static"
   src-path: "{DIR[RST]}"
   dst-path: "{DIR[WORKING,SRC]}"
 
 -
   type: "input"  # Ignored in debug mode
+#  general:
+#    file-name: "robot-plugin.zip"
+#    file-format: ".zip"
+#    download-path: "{job}/{build}/robot/report/*zip*/{filename}"
+#    extract: "robot-plugin/output.xml"
   general:
     file-name: "output.xml.gz"
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"
-#    file-name: "robot-plugin.zip"
-#    file-format: ".zip"
-#    download-path: "{job}/{build}/robot/report/*zip*/{filename}"
-#    extract: "robot-plugin/output.xml"
   builds:
     csit-vpp-perf-mrr-daily-master:
       start: 15
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index af4d707613..ebd84530a3 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -49,7 +49,6 @@ class Specification(object):
 
         self._specification = {"environment": dict(),
                                "configuration": dict(),
-                               "debug": dict(),
                                "static": dict(),
                                "input": dict(),
                                "output": dict(),
@@ -95,15 +94,6 @@ class Specification(object):
         return self._specification["static"]
 
     @property
-    def debug(self):
-        """Getter - debug
-
-        :returns: Debug specification
-        :rtype: dict
-        """
-        return self._specification["debug"]
-
-    @property
     def is_debug(self):
         """Getter - debug mode
 
@@ -425,43 +415,6 @@ class Specification(object):
                         self.configuration["data-sets"][set_name][job] = builds
         logging.info("Done.")
 
-    def _parse_debug(self):
-        """Parse debug specification in the specification YAML file.
-        """
-
-        if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
-            return None
-
-        logging.info("Parsing specification file: debug ...")
-
-        idx = self._get_type_index("debug")
-        if idx is None:
-            self.environment["configuration"]["CFG[DEBUG]"] = 0
-            return None
-
-        try:
-            for key, value in self._cfg_yaml[idx]["general"].items():
-                self._specification["debug"][key] = value
-
-            self._specification["input"]["builds"] = dict()
-            for job, builds in self._cfg_yaml[idx]["builds"].items():
-                if builds:
-                    self._specification["input"]["builds"][job] = list()
-                    for build in builds:
-                        self._specification["input"]["builds"][job].\
-                            append({"build": build["build"],
-                                    "status": "downloaded",
-                                    "file-name": self._replace_tags(
-                                        build["file"],
-                                        self.environment["paths"])})
-                else:
-                    logging.warning("No build is defined for the job '{}'. "
-                                    "Trying to continue without it.".
-                                    format(job))
-
-        except KeyError:
-            raise PresentationError("No data to process.")
-
     def _parse_input(self):
         """Parse input specification in the specification YAML file.
 
@@ -664,9 +617,7 @@ class Specification(object):
 
         self._parse_env()
         self._parse_configuration()
-        self._parse_debug()
-        if not self.debug:
-            self._parse_input()
+        self._parse_input()
         self._parse_output()
         self._parse_static()
         self._parse_elements()
diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py
index 95faffdd66..59dbfa634b 100644
--- a/resources/tools/presentation/utils.py
+++ b/resources/tools/presentation/utils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -198,8 +198,8 @@ def execute_command(cmd):
 
     stdout, stderr = proc.communicate()
 
-    logging.info(stdout)
-    logging.info(stderr)
+    logging.debug(stdout)
+    logging.debug(stderr)
 
     if proc.returncode != 0:
         logging.error("    Command execution failed.")
@@ -250,10 +250,7 @@ def archive_input_data(spec):
 
     logging.info("    Archiving the input data files ...")
 
-    if spec.is_debug:
-        extension = spec.debug["input-format"]
-    else:
-        extension = spec.input["file-format"]
+    extension = spec.input["file-format"]
     data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"],
                            extension=extension)
     dst = spec.environment["paths"]["DIR[STATIC,ARCH]"]