CSIT-1078: Optimize input data files download and processing

Change-Id: I973238d98d549555c0d43c91f9fd96a9209065ac
Signed-off-by: Tibor Frank <tifrank@cisco.com>

3 files changed, 14 insertions, 28 deletions

diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py
index 58dd56163a..84890d9c46 100644
--- a/resources/tools/presentation/conf.py
+++ b/resources/tools/presentation/conf.py
@@ -73,7 +73,8 @@ rst_epilog = """
 .. _TRex intallation: https://git.fd.io/csit/tree/resources/tools/trex/trex_installer.sh?h={release}
 .. _TRex driver: https://git.fd.io/csit/tree/resources/tools/trex/trex_stateless_profile.py?h={release}
 .. _VIRL topologies directory: https://git.fd.io/csit/tree/resources/tools/virl/topologies/?h={release}
-.. _VIRL images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/ubuntu/lists/?h={release}
+.. _VIRL ubuntu images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/ubuntu/lists/?h={release}
+.. _VIRL centos images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/centos/lists/?h={release}
 .. _VIRL nested: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/nested/?h={release}
 .. _CSIT Honeycomb Functional Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.vpp.func.honeycomb.html
 .. _CSIT Honeycomb Performance Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.vpp.perf.honeycomb.html
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index e12e2fb8df..d0f9eed9a4 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -251,6 +251,7 @@ class ExecutionChecker(ResultVisitor):
             self._version = str(re.search(self.REGEX_VERSION, msg.message).
                                 group(2))
             self._data["metadata"]["version"] = self._version
+            self._data["metadata"]["generated"] = msg.timestamp
             self._msg_type = None
 
     def _get_vat_history(self, msg):
@@ -748,17 +749,14 @@ class InputData(object):
         return self.data[job][build]["tests"]
 
     @staticmethod
-    def _parse_tests(job, build, get_timestamp=False):
+    def _parse_tests(job, build):
         """Process data from robot output.xml file and return JSON structured
         data.
 
         :param job: The name of job which build output data will be processed.
         :param build: The build which output data will be processed.
-        :param get_timestamp: If True, timestamp is read form the xml source
-            file.
         :type job: str
         :type build: dict
-        :type get_timestamp: bool
         :returns: JSON data structure.
         :rtype: dict
         """
@@ -767,10 +765,6 @@ class InputData(object):
             "job": job,
             "build": build
         }
-        if get_timestamp:
-            tree = ET.parse(build["file-name"])
-            root = tree.getroot()
-            metadata["generated"] = root.attrib["generated"]
 
         with open(build["file-name"], 'r') as data_file:
             try:
@@ -784,20 +778,16 @@ class InputData(object):
 
         return checker.data
 
-    def download_and_parse_data(self, get_timestamp=False):
+    def download_and_parse_data(self):
         """Download the input data files, parse input data from input files and
         store in pandas' Series.
-
-        :param get_timestamp: If True, timestamp is read form the xml source
-            file.
-        :type get_timestamp: bool
         """
 
         logging.info("Downloading and parsing input files ...")
 
         job_data = dict()
         for job, builds in self._cfg.builds.items():
-            logging.info("  Processing data from the job '{0}' ...'".
+            logging.info("  Processing data from the job '{0}' ...".
                          format(job))
             builds_data = dict()
             for build in builds:
@@ -813,8 +803,7 @@ class InputData(object):
 
                 logging.info("      Processing data from the build '{0}' ...".
                              format(build["build"]))
-                data = InputData._parse_tests(job, build,
-                                              get_timestamp=get_timestamp)
+                data = InputData._parse_tests(job, build)
                 if data is None:
                     logging.error("Input data file from the job '{job}', build "
                                   "'{build}' is damaged. Skipped.".
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 2268801407..1ccefd3b43 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -96,11 +96,7 @@ def main():
         prepare_static_content(spec)
 
         data = InputData(spec)
-
-        if spec.output["output"] == "report":
-            data.download_and_parse_data(get_timestamp=False)
-        elif spec.output["output"] == "CPTA":
-            data.download_and_parse_data(get_timestamp=True)
+        data.download_and_parse_data()
 
         generate_tables(spec, data)
         generate_plots(spec, data)
@@ -114,12 +110,12 @@ def main():
             logging.info("Successfully finished.")
         ret_code = 0
 
-    # except (KeyError, ValueError, PresentationError) as err:
-    #     logging.info("Finished with an error.")
-    #     logging.critical(str(err))
-    # except Exception as err:
-    #     logging.info("Finished with an unexpected error.")
-    #     logging.critical(str(err))
+    except (KeyError, ValueError, PresentationError) as err:
+        logging.info("Finished with an error.")
+        logging.critical(str(err))
+    except Exception as err:
+        logging.info("Finished with an unexpected error.")
+        logging.critical(str(err))
     finally:
         if spec is not None:
             clean_environment(spec.environment)