aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools/presentation
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2018-05-14 14:35:46 +0200
committerTibor Frank <tifrank@cisco.com>2018-05-14 14:36:43 +0200
commit9964155b56020914494f7341c2d1162f2ac9d720 (patch)
treebf60e93fb4aca7cbbc81e919cf2d91e9abf2ff29 /resources/tools/presentation
parentfd6fedc983a7d1796eb2531782be8a37b6d0921d (diff)
CSIT-1078: Optimize input data files download and processing
Change-Id: I973238d98d549555c0d43c91f9fd96a9209065ac Signed-off-by: Tibor Frank <tifrank@cisco.com>
Diffstat (limited to 'resources/tools/presentation')
-rw-r--r--resources/tools/presentation/conf.py3
-rw-r--r--resources/tools/presentation/input_data_parser.py21
-rw-r--r--resources/tools/presentation/pal.py18
3 files changed, 14 insertions, 28 deletions
diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py
index 58dd56163a..84890d9c46 100644
--- a/resources/tools/presentation/conf.py
+++ b/resources/tools/presentation/conf.py
@@ -73,7 +73,8 @@ rst_epilog = """
.. _TRex intallation: https://git.fd.io/csit/tree/resources/tools/trex/trex_installer.sh?h={release}
.. _TRex driver: https://git.fd.io/csit/tree/resources/tools/trex/trex_stateless_profile.py?h={release}
.. _VIRL topologies directory: https://git.fd.io/csit/tree/resources/tools/virl/topologies/?h={release}
-.. _VIRL images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/ubuntu/lists/?h={release}
+.. _VIRL ubuntu images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/ubuntu/lists/?h={release}
+.. _VIRL centos images lists: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/centos/lists/?h={release}
.. _VIRL nested: https://git.fd.io/csit/tree/resources/tools/disk-image-builder/nested/?h={release}
.. _CSIT Honeycomb Functional Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.vpp.func.honeycomb.html
.. _CSIT Honeycomb Performance Tests Documentation: https://docs.fd.io/csit/{release}/doc/tests.vpp.perf.honeycomb.html
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index e12e2fb8df..d0f9eed9a4 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -251,6 +251,7 @@ class ExecutionChecker(ResultVisitor):
self._version = str(re.search(self.REGEX_VERSION, msg.message).
group(2))
self._data["metadata"]["version"] = self._version
+ self._data["metadata"]["generated"] = msg.timestamp
self._msg_type = None
def _get_vat_history(self, msg):
@@ -748,17 +749,14 @@ class InputData(object):
return self.data[job][build]["tests"]
@staticmethod
- def _parse_tests(job, build, get_timestamp=False):
+ def _parse_tests(job, build):
"""Process data from robot output.xml file and return JSON structured
data.
:param job: The name of job which build output data will be processed.
:param build: The build which output data will be processed.
- :param get_timestamp: If True, timestamp is read form the xml source
- file.
:type job: str
:type build: dict
- :type get_timestamp: bool
:returns: JSON data structure.
:rtype: dict
"""
@@ -767,10 +765,6 @@ class InputData(object):
"job": job,
"build": build
}
- if get_timestamp:
- tree = ET.parse(build["file-name"])
- root = tree.getroot()
- metadata["generated"] = root.attrib["generated"]
with open(build["file-name"], 'r') as data_file:
try:
@@ -784,20 +778,16 @@ class InputData(object):
return checker.data
- def download_and_parse_data(self, get_timestamp=False):
+ def download_and_parse_data(self):
"""Download the input data files, parse input data from input files and
store in pandas' Series.
-
- :param get_timestamp: If True, timestamp is read form the xml source
- file.
- :type get_timestamp: bool
"""
logging.info("Downloading and parsing input files ...")
job_data = dict()
for job, builds in self._cfg.builds.items():
- logging.info(" Processing data from the job '{0}' ...'".
+ logging.info(" Processing data from the job '{0}' ...".
format(job))
builds_data = dict()
for build in builds:
@@ -813,8 +803,7 @@ class InputData(object):
logging.info(" Processing data from the build '{0}' ...".
format(build["build"]))
- data = InputData._parse_tests(job, build,
- get_timestamp=get_timestamp)
+ data = InputData._parse_tests(job, build)
if data is None:
logging.error("Input data file from the job '{job}', build "
"'{build}' is damaged. Skipped.".
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 2268801407..1ccefd3b43 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -96,11 +96,7 @@ def main():
prepare_static_content(spec)
data = InputData(spec)
-
- if spec.output["output"] == "report":
- data.download_and_parse_data(get_timestamp=False)
- elif spec.output["output"] == "CPTA":
- data.download_and_parse_data(get_timestamp=True)
+ data.download_and_parse_data()
generate_tables(spec, data)
generate_plots(spec, data)
@@ -114,12 +110,12 @@ def main():
logging.info("Successfully finished.")
ret_code = 0
- # except (KeyError, ValueError, PresentationError) as err:
- # logging.info("Finished with an error.")
- # logging.critical(str(err))
- # except Exception as err:
- # logging.info("Finished with an unexpected error.")
- # logging.critical(str(err))
+ except (KeyError, ValueError, PresentationError) as err:
+ logging.info("Finished with an error.")
+ logging.critical(str(err))
+ except Exception as err:
+ logging.info("Finished with an unexpected error.")
+ logging.critical(str(err))
finally:
if spec is not None:
clean_environment(spec.environment)