aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2018-05-11 15:14:13 +0200
committerTibor Frank <tifrank@cisco.com>2018-05-14 08:09:34 +0200
commit5646509aea6b43ef1efb282aad908289cc005b26 (patch)
tree12084133502bee8855a5c8deedf113f3e04cb8fa /resources/tools
parent632c598ddc0e2a4586c85a3cd627f8c1eaee981a (diff)
CSIT-1078: Optimize input data files download and processing
Change-Id: Id1b0af996c4b77878621cfc8c262795f51e537fc Signed-off-by: Tibor Frank <tifrank@cisco.com>
Diffstat (limited to 'resources/tools')
-rw-r--r--resources/tools/presentation/environment.py40
-rw-r--r--resources/tools/presentation/generator_CPTA.py4
-rw-r--r--resources/tools/presentation/generator_tables.py5
-rw-r--r--resources/tools/presentation/input_data_files.py380
-rw-r--r--resources/tools/presentation/input_data_parser.py75
-rw-r--r--resources/tools/presentation/pal.py45
-rw-r--r--resources/tools/presentation/specification.yaml30
-rw-r--r--resources/tools/presentation/specification_CPTA.yaml64
-rw-r--r--resources/tools/presentation/specification_parser.py53
-rw-r--r--resources/tools/presentation/utils.py11
10 files changed, 256 insertions, 451 deletions
diff --git a/resources/tools/presentation/environment.py b/resources/tools/presentation/environment.py
index 1a1c3db032..a2fa9a0d5b 100644
--- a/resources/tools/presentation/environment.py
+++ b/resources/tools/presentation/environment.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -51,35 +51,6 @@ class Environment(object):
"""
return self._env
- def _set_environment_variables(self):
- """Set environment variables.
- """
- logging.info("Setting the environment variables ...")
- # logging.debug("Environment variables before:\n{}".format(os.environ))
-
- count = 1
-
- for var, value in self._env["configuration"].items():
- logging.debug(" {:3d} Setting the variable {} = {}".
- format(count, var, value))
- os.environ[var] = str(value)
- count += 1
-
- for var, value in self._env["paths"].items():
- logging.debug(" {:3d} Setting the variable {} = {}".
- format(count, var, value))
- os.environ[var] = str(value)
- count += 1
-
- for var, value in self._env["urls"].items():
- logging.debug(" {:3d} Setting the variable {} = {}".
- format(count, var, value))
- os.environ[var] = str(value)
- count += 1
-
- # logging.debug("Environment variables after:\n{}".format(os.environ))
- logging.info("Done.")
-
def _make_dirs(self):
"""Create the directories specified in the 'make-dirs' part of
'environment' section in the specification file.
@@ -122,7 +93,6 @@ class Environment(object):
"""Set the environment.
"""
- self._set_environment_variables()
self._make_dirs()
@@ -147,10 +117,10 @@ def clean_environment(env):
if os.path.isdir(dir_to_remove):
try:
shutil.rmtree(dir_to_remove)
- except OSError:
- pass
- # raise PresentationError("Cannot remove the directory '{}'".
- # format(dir_to_remove))
+ except OSError as err:
+ logging.warning("Cannot remove the directory '{}'".
+ format(dir_to_remove))
+ logging.debug(str(err))
else:
logging.warning("The directory '{}' does not exist.".
format(dir_to_remove))
diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index e27a52172b..89e9a41b48 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -32,7 +32,7 @@ from utils import split_outliers, archive_input_data, execute_command
HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
'-b html -E ' \
'-t html ' \
- '-D version="Generated on {date}" ' \
+ '-D version="{date}" ' \
'{working_dir} ' \
'{build_dir}/'
@@ -64,7 +64,7 @@ def generate_cpta(spec, data):
ret_code = _generate_all_charts(spec, data)
cmd = HTML_BUILDER.format(
- date=datetime.date.today().strftime('%d-%b-%Y'),
+ date=datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'),
working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
execute_command(cmd)
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 36933cc0e5..38439bac5c 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -822,9 +822,6 @@ def table_performance_trending_dashboard(table, input_data):
rel_change_long = round(
((last_median_t - max_median) / max_median) * 100, 2)
- logging.info("rel_change_last : {}".format(rel_change_last))
- logging.info("rel_change_long : {}".format(rel_change_long))
-
tbl_lst.append(
[name,
'-' if isnan(last_median_t) else
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index d81f64fbe6..24dc3a57d8 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -16,11 +16,10 @@ Download all data.
"""
import re
-import gzip
import logging
-from os import rename, remove
-from os.path import join, getsize
+from os import rename
+from os.path import join
from shutil import move
from zipfile import ZipFile, is_zipfile, BadZipfile
from httplib import responses
@@ -39,216 +38,179 @@ SEPARATOR = "__"
REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
-def download_data_files(spec):
- """Download all data specified in the specification file in the section
- type: input --> builds.
+def _download_file(url, file_name):
+ """Download a file with input data.
- :param spec: Specification.
+ :param url: URL to the file to download.
+ :param file_name: Name of file to download.
+ :type url: str
+ :type file_name: str
+ :returns: True if the download was successful, otherwise False.
+ :rtype: bool
+ """
+
+ success = False
+ try:
+ logging.info(" Connecting to '{0}' ...".format(url))
+
+ response = get(url, stream=True)
+ code = response.status_code
+
+ logging.info(" {0}: {1}".format(code, responses[code]))
+
+ if code != codes["OK"]:
+ return False
+
+ logging.info(" Downloading the file '{0}' to '{1}' ...".
+ format(url, file_name))
+
+ file_handle = open(file_name, "wb")
+ for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+ if chunk:
+ file_handle.write(chunk)
+ file_handle.close()
+ success = True
+ except ConnectionError as err:
+ logging.error("Not possible to connect to '{0}'.".format(url))
+ logging.debug(str(err))
+ except HTTPError as err:
+ logging.error("Invalid HTTP response from '{0}'.".format(url))
+ logging.debug(str(err))
+ except TooManyRedirects as err:
+ logging.error("Request exceeded the configured number "
+ "of maximum re-directions.")
+ logging.debug(str(err))
+ except Timeout as err:
+ logging.error("Request timed out.")
+ logging.debug(str(err))
+ except RequestException as err:
+ logging.error("Unexpected HTTP request exception.")
+ logging.debug(str(err))
+ except (IOError, ValueError, KeyError) as err:
+ logging.error("Download failed.")
+ logging.debug(str(err))
+
+ logging.info(" Download finished.")
+ return success
+
+
+def _unzip_file(spec, job, build):
+ """Unzip downloaded source file.
+
+ :param spec: Specification read form the specification file.
+ :param job: Name of the Jenkins job.
+ :param build: Information about the build.
:type spec: Specification
- :raises: PresentationError if there is no url defined for the job.
+ :type job: str
+ :type build: dict
+ :returns: True if the download was successful, otherwise False.
+ :rtype: bool
"""
- for job, builds in spec.builds.items():
- for build in builds:
- if job.startswith("csit-"):
- if spec.input["file-name"].endswith(".zip"):
- url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
- elif spec.input["file-name"].endswith(".gz"):
- url = spec.environment["urls"]["URL[NEXUS,LOG]"]
- else:
- logging.error("Not supported file format.")
- continue
- elif job.startswith("hc2vpp-"):
- url = spec.environment["urls"]["URL[JENKINS,HC]"]
- else:
- raise PresentationError("No url defined for the job '{}'.".
- format(job))
- file_name = spec.input["file-name"]
- full_name = spec.input["download-path"].\
- format(job=job, build=build["build"], filename=file_name)
- url = "{0}/{1}".format(url, full_name)
- new_name = join(
- spec.environment["paths"]["DIR[WORKING,DATA]"],
- "{job}{sep}{build}{sep}{name}".format(job=job,
- sep=SEPARATOR,
- build=build["build"],
- name=file_name))
- logging.info(
- "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
-
- status = "failed"
- try:
- response = get(url, stream=True)
- code = response.status_code
-
- if code != codes["OK"]:
- logging.warning(
- "Jenkins: {0}: {1}.".format(code, responses[code]))
- logging.info("Trying to download from Nexus:")
- spec.set_input_state(job, build["build"], "not found")
- if code == codes["not_found"]:
- release = re.search(REGEX_RELEASE, job).group(2)
- nexus_file_name = "{job}{sep}{build}{sep}{name}".\
- format(job=job, sep=SEPARATOR, build=build["build"],
- name=file_name)
- try:
- release = "rls{0}".format(int(release))
- except ValueError:
- pass
- url = "{url}/{release}/{dir}/{file}".\
- format(url=spec.environment["urls"]["URL[NEXUS]"],
- release=release,
- dir=spec.environment["urls"]["DIR[NEXUS]"],
- file=nexus_file_name)
- logging.info("Downloading the file '{0}' to '{1}' ...".
- format(url, new_name))
- response = get(url, stream=True)
- code = response.status_code
- if code != codes["OK"]:
- logging.error(
- "Nexus: {0}: {1}".format(code, responses[code]))
- spec.set_input_state(
- job, build["build"], "not found")
- continue
-
- file_handle = open(new_name, "wb")
- for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
- if chunk:
- file_handle.write(chunk)
- file_handle.close()
-
- if spec.input["file-name"].endswith(".zip"):
- expected_length = None
- try:
- expected_length = int(response.
- headers["Content-Length"])
- logging.debug(" Expected file size: {0}B".
- format(expected_length))
- except KeyError:
- logging.debug(" No information about expected size.")
-
- real_length = getsize(new_name)
- logging.debug(" Downloaded size: {0}B".format(real_length))
-
- if expected_length:
- if real_length == expected_length:
- status = "downloaded"
- logging.info("{0}: {1}".format(code,
- responses[code]))
- else:
- logging.error("The file size differs from the "
- "expected size.")
- else:
- status = "downloaded"
- logging.info("{0}: {1}".format(code, responses[code]))
-
- elif spec.input["file-name"].endswith(".gz"):
- if "docs.fd.io" in url:
- execute_command("gzip --decompress --keep --force {0}".
- format(new_name))
- else:
- rename(new_name, new_name[:-3])
- execute_command("gzip --keep {0}".format(new_name[:-3]))
- new_name = new_name[:-3]
- status = "downloaded"
- logging.info("{0}: {1}".format(code, responses[code]))
-
- except ConnectionError as err:
- logging.error("Not possible to connect to '{0}'.".format(url))
- logging.debug(err)
- except HTTPError as err:
- logging.error("Invalid HTTP response from '{0}'.".format(url))
- logging.debug(err)
- except TooManyRedirects as err:
- logging.error("Request exceeded the configured number "
- "of maximum re-directions.")
- logging.debug(err)
- except Timeout as err:
- logging.error("Request timed out.")
- logging.debug(err)
- except RequestException as err:
- logging.error("Unexpected HTTP request exception.")
- logging.debug(err)
- except (IOError, ValueError, KeyError) as err:
- logging.error("Download failed.")
- logging.debug("Reason: {0}".format(err))
-
- spec.set_input_state(job, build["build"], status)
- spec.set_input_file_name(job, build["build"], new_name)
-
- if status == "failed":
- logging.info("Removing the file '{0}'".format(new_name))
- try:
- remove(new_name)
- except OSError as err:
- logging.warning(str(err))
- spec.set_input_file_name(job, build["build"], None)
-
- unzip_files(spec)
-
-
-def unzip_files(spec):
- """Unzip downloaded zip files
-
- :param spec: Specification.
+ data_file = spec.input["extract"]
+ file_name = build["file-name"]
+ directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
+ new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
+ SEPARATOR,
+ data_file.split("/")[-1])
+ logging.info(" Unzipping: '{0}' from '{1}'.".
+ format(data_file, file_name))
+ try:
+ with ZipFile(file_name, 'r') as zip_file:
+ zip_file.extract(data_file, directory)
+ logging.info(" Moving {0} to {1} ...".
+ format(join(directory, data_file), directory))
+ move(join(directory, data_file), directory)
+ logging.info(" Renaming the file '{0}' to '{1}'".
+ format(join(directory, data_file.split("/")[-1]),
+ new_name))
+ rename(join(directory, data_file.split("/")[-1]),
+ new_name)
+ spec.set_input_file_name(job, build["build"],
+ new_name)
+ return True
+ except (BadZipfile, RuntimeError) as err:
+ logging.error("Failed to unzip the file '{0}': {1}.".
+ format(file_name, str(err)))
+ return False
+ except OSError as err:
+ logging.error("Failed to rename the file '{0}': {1}.".
+ format(data_file, str(err)))
+ return False
+
+
+def download_and_unzip_data_file(spec, job, build):
+ """Download and unzip a source file.
+
+ :param spec: Specification read form the specification file.
+ :param job: Name of the Jenkins job.
+ :param build: Information about the build.
:type spec: Specification
- :raises: PresentationError if the zip file does not exist or it is not a
- zip file.
+ :type job: str
+ :type build: dict
+ :returns: True if the download was successful, otherwise False.
+ :rtype: bool
"""
- if spec.is_debug:
- data_file = spec.debug["extract"]
+ if job.startswith("csit-"):
+ if spec.input["file-name"].endswith(".zip"):
+ url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
+ elif spec.input["file-name"].endswith(".gz"):
+ url = spec.environment["urls"]["URL[NEXUS,LOG]"]
+ else:
+ logging.error("Not supported file format.")
+ return False
+ elif job.startswith("hc2vpp-"):
+ url = spec.environment["urls"]["URL[JENKINS,HC]"]
+ else:
+ raise PresentationError("No url defined for the job '{}'.".
+ format(job))
+ file_name = spec.input["file-name"]
+ full_name = spec.input["download-path"]. \
+ format(job=job, build=build["build"], filename=file_name)
+ url = "{0}/{1}".format(url, full_name)
+ new_name = join(spec.environment["paths"]["DIR[WORKING,DATA]"],
+ "{job}{sep}{build}{sep}{name}".
+ format(job=job, sep=SEPARATOR, build=build["build"],
+ name=file_name))
+ # Download the file from the defined source (Jenkins, logs.fd.io):
+ success = _download_file(url, new_name)
+
+ # If not successful, download from docs.fd.io:
+ if not success:
+ logging.info(" Trying to download from https://docs.fd.io:")
+ release = re.search(REGEX_RELEASE, job).group(2)
+ nexus_file_name = "{job}{sep}{build}{sep}{name}". \
+ format(job=job, sep=SEPARATOR, build=build["build"], name=file_name)
+ try:
+ release = "rls{0}".format(int(release))
+ except ValueError:
+ pass
+ url = "{url}/{release}/{dir}/{file}". \
+ format(url=spec.environment["urls"]["URL[NEXUS]"],
+ release=release,
+ dir=spec.environment["urls"]["DIR[NEXUS]"],
+ file=nexus_file_name)
+ success = _download_file(url, new_name)
+
+ if success:
+ spec.set_input_file_name(job, build["build"], new_name)
+ else:
+ return False
+
+ if spec.input["file-name"].endswith(".gz"):
+ if "docs.fd.io" in url:
+ execute_command("gzip --decompress --keep --force {0}".
+ format(new_name))
+ else:
+ rename(new_name, new_name[:-3])
+ execute_command("gzip --keep {0}".format(new_name[:-3]))
+ spec.set_input_file_name(job, build["build"], new_name[:-3])
+
+ if spec.input["file-name"].endswith(".zip"):
+ if is_zipfile(file_name):
+ return _unzip_file(spec, job, build)
+ else:
+ return False
else:
- data_file = spec.input["extract"]
-
- for job, builds in spec.builds.items():
- for build in builds:
- if build["status"] == "failed" or build["status"] == "not found":
- continue
- try:
- status = "failed"
- directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
- file_name = join(build["file-name"])
-
- if build["status"] == "downloaded":
- logging.info("Unziping: '{0}' from '{1}'.".
- format(data_file, file_name))
- new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
- SEPARATOR,
- data_file.split("/")[-1])
- try:
- if is_zipfile(file_name):
- with ZipFile(file_name, 'r') as zip_file:
- zip_file.extract(data_file, directory)
- logging.info("Moving {0} to {1} ...".
- format(join(directory, data_file),
- directory))
- move(join(directory, data_file), directory)
- logging.info("Renaming the file '{0}' to '{1}'".
- format(join(directory,
- data_file.split("/")[-1]),
- new_name))
- rename(join(directory, data_file.split("/")[-1]),
- new_name)
- spec.set_input_file_name(job, build["build"],
- new_name)
- status = "unzipped"
- spec.set_input_state(job, build["build"], status)
- except (BadZipfile, RuntimeError) as err:
- logging.error("Failed to unzip the file '{0}': {1}.".
- format(file_name, str(err)))
- except OSError as err:
- logging.error("Failed to rename the file '{0}': {1}.".
- format(data_file, str(err)))
- finally:
- if status == "failed":
- spec.set_input_file_name(job, build["build"], None)
- else:
- raise PresentationError("The file '{0}' does not exist or "
- "it is not a zip file".
- format(file_name))
-
- spec.set_input_state(job, build["build"], status)
-
- except KeyError:
- pass
+ return True
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 0ad07a95db..e12e2fb8df 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -29,6 +29,8 @@ from collections import OrderedDict
from string import replace
from os import remove
+from input_data_files import download_and_unzip_data_file
+
class ExecutionChecker(ResultVisitor):
"""Class to traverse through the test suite structure.
@@ -179,7 +181,7 @@ class ExecutionChecker(ResultVisitor):
REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
r'tx\s(\d*),\srx\s(\d*)')
- def __init__(self, **metadata):
+ def __init__(self, metadata):
"""Initialisation.
:param metadata: Key-value pairs to be included in "metadata" part of
@@ -251,8 +253,6 @@ class ExecutionChecker(ResultVisitor):
self._data["metadata"]["version"] = self._version
self._msg_type = None
- logging.info(" VPP version: {0}".format(self._version))
-
def _get_vat_history(self, msg):
"""Called when extraction of VAT command history is required.
@@ -748,21 +748,29 @@ class InputData(object):
return self.data[job][build]["tests"]
@staticmethod
- def _parse_tests(job, build):
+ def _parse_tests(job, build, get_timestamp=False):
"""Process data from robot output.xml file and return JSON structured
data.
:param job: The name of job which build output data will be processed.
:param build: The build which output data will be processed.
+ :param get_timestamp: If True, timestamp is read form the xml source
+ file.
:type job: str
:type build: dict
+ :type get_timestamp: bool
:returns: JSON data structure.
:rtype: dict
"""
- tree = ET.parse(build["file-name"])
- root = tree.getroot()
- generated = root.attrib["generated"]
+ metadata = {
+ "job": job,
+ "build": build
+ }
+ if get_timestamp:
+ tree = ET.parse(build["file-name"])
+ root = tree.getroot()
+ metadata["generated"] = root.attrib["generated"]
with open(build["file-name"], 'r') as data_file:
try:
@@ -771,46 +779,55 @@ class InputData(object):
logging.error("Error occurred while parsing output.xml: {0}".
format(err))
return None
- checker = ExecutionChecker(job=job, build=build, generated=generated)
+ checker = ExecutionChecker(metadata)
result.visit(checker)
return checker.data
- def read_data(self):
- """Parse input data from input files and store in pandas' Series.
+ def download_and_parse_data(self, get_timestamp=False):
+ """Download the input data files, parse input data from input files and
+ store in pandas' Series.
+
+ :param get_timestamp: If True, timestamp is read form the xml source
+ file.
+ :type get_timestamp: bool
"""
- logging.info("Parsing input files ...")
+ logging.info("Downloading and parsing input files ...")
job_data = dict()
for job, builds in self._cfg.builds.items():
- logging.info(" Extracting data from the job '{0}' ...'".
+ logging.info(" Processing data from the job '{0}' ...'".
format(job))
builds_data = dict()
for build in builds:
- if build["status"] == "failed" \
- or build["status"] == "not found":
+ logging.info(" Processing the build '{0}'".
+ format(build["build"]))
+ self._cfg.set_input_state(job, build["build"], "failed")
+ if not download_and_unzip_data_file(self._cfg, job, build):
+ logging.error("It is not possible to download the input "
+ "data file from the job '{job}', build "
+ "'{build}', or it is damaged. Skipped.".
+ format(job=job, build=build["build"]))
continue
- logging.info(" Extracting data from the build '{0}'".
+
+ logging.info(" Processing data from the build '{0}' ...".
format(build["build"]))
- logging.info(" Processing the file '{0}'".
- format(build["file-name"]))
+ data = InputData._parse_tests(job, build,
+ get_timestamp=get_timestamp)
+ if data is None:
+ logging.error("Input data file from the job '{job}', build "
+ "'{build}' is damaged. Skipped.".
+ format(job=job, build=build["build"]))
+ continue
- data = InputData._parse_tests(job, build)
+ self._cfg.set_input_state(job, build["build"], "processed")
- logging.info(" Removing the file '{0}'".
- format(build["file-name"]))
try:
remove(build["file-name"])
- build["status"] = "processed"
except OSError as err:
- logging.error(" Cannot remove the file '{0}': {1}".
+ logging.error("Cannot remove the file '{0}': {1}".
format(build["file-name"], err))
- if data is None:
- logging.error("Input data file from the job '{job}', build "
- "'{build}' is damaged. Skipped.".
- format(job=job, build=build["build"]))
- continue
build_data = pd.Series({
"metadata": pd.Series(data["metadata"].values(),
@@ -818,9 +835,9 @@ class InputData(object):
"suites": pd.Series(data["suites"].values(),
index=data["suites"].keys()),
"tests": pd.Series(data["tests"].values(),
- index=data["tests"].keys()),
- })
+ index=data["tests"].keys())})
builds_data[str(build["build"])] = build_data
+ build["status"] = "processed"
logging.info(" Done.")
job_data[job] = pd.Series(builds_data.values(),
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 98642c898c..2268801407 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -21,7 +21,6 @@ import logging
from errors import PresentationError
from environment import Environment, clean_environment
from specification_parser import Specification
-from input_data_files import download_data_files, unzip_files
from input_data_parser import InputData
from generator_tables import generate_tables
from generator_plots import generate_plots
@@ -30,8 +29,6 @@ from static_content import prepare_static_content
from generator_report import generate_report
from generator_CPTA import generate_cpta
-from pprint import pprint
-
def parse_args():
"""Parse arguments from cmd line.
@@ -86,21 +83,24 @@ def main():
logging.critical("Finished with error.")
return 1
- ret_code = 0
+ if spec.output["output"] not in ("report", "CPTA"):
+ logging.critical("The output '{0}' is not supported.".
+ format(spec.output["output"]))
+ return 1
+
+ ret_code = 1
try:
env = Environment(spec.environment, args.force)
env.set_environment()
- if spec.is_debug:
- if spec.debug["input-format"] == "zip":
- unzip_files(spec)
- else:
- download_data_files(spec)
-
prepare_static_content(spec)
data = InputData(spec)
- data.read_data()
+
+ if spec.output["output"] == "report":
+ data.download_and_parse_data(get_timestamp=False)
+ elif spec.output["output"] == "CPTA":
+ data.download_and_parse_data(get_timestamp=True)
generate_tables(spec, data)
generate_plots(spec, data)
@@ -112,21 +112,16 @@ def main():
elif spec.output["output"] == "CPTA":
sys.stdout.write(generate_cpta(spec, data))
logging.info("Successfully finished.")
- else:
- logging.critical("The output '{0}' is not supported.".
- format(spec.output["output"]))
- ret_code = 1
-
- except (KeyError, ValueError, PresentationError) as err:
- logging.info("Finished with an error.")
- logging.critical(str(err))
- ret_code = 1
- except Exception as err:
- logging.info("Finished with an unexpected error.")
- logging.critical(str(err))
- ret_code = 1
+ ret_code = 0
+
+ # except (KeyError, ValueError, PresentationError) as err:
+ # logging.info("Finished with an error.")
+ # logging.critical(str(err))
+ # except Exception as err:
+ # logging.info("Finished with an unexpected error.")
+ # logging.critical(str(err))
finally:
- if spec is not None and not spec.is_debug:
+ if spec is not None:
clean_environment(spec.environment)
return ret_code
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 4b055a84ce..97e616a15d 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -1,14 +1,5 @@
-
type: "environment"
- configuration:
- # Debug mode:
- # - Skip:
- # - Download of input data files
- # - Do:
- # - Read data from given zip / xml files
- # - Set the configuration as it is done in normal mode
- # If the section "type: debug" is missing, CFG[DEBUG] is set to 0.
- CFG[DEBUG]: 0
paths:
# Top level directories:
@@ -546,27 +537,6 @@
height: 1000
-
- type: "debug"
- general:
- input-format: "xml" # zip or xml
- extract: "robot-plugin/output.xml" # Only for zip
- builds:
- # The files must be in the directory DIR[WORKING,DATA]
- csit-vpp-perf-1801-all:
- -
- build: 1
- file: "{DIR[WORKING,DATA]}/output.xml"
- -
- build: 2
- file: "{DIR[WORKING,DATA]}/output.xml"
- -
- build: 3
- file: "{DIR[WORKING,DATA]}/output.xml"
- -
- build: 4
- file: "{DIR[WORKING,DATA]}/output.xml"
-
--
type: "static"
src-path: "{DIR[RST]}"
dst-path: "{DIR[WORKING,SRC]}"
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index a570dccbbe..9b47083b2b 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -16,15 +16,6 @@
-
type: "environment"
- configuration:
- # Debug mode:
- # - Skip:
- # - Download of input data files
- # - Do:
- # - Read data from given zip / xml files
- # - Set the configuration as it is done in normal mode
- # If the section "type: debug" is missing, CFG[DEBUG] is set to 0.
- CFG[DEBUG]: 0
paths:
# Top level directories:
@@ -102,7 +93,6 @@
showgrid: True
xaxis:
showticklabels: True
- # title: "csit-vpp-perf-mrr-daily-master/vpp-build"
title: '<a href="https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master" target="_blank">csit-vpp-perf-mrr-daily-master-build</a>/<a href="https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/vpp/" target="_blank">vpp-build</a>'
autorange: True
showgrid: True
@@ -130,66 +120,22 @@
namelength: -1
-
- type: "debug"
- general:
- input-format: "xml" # zip or xml
- extract: "robot-plugin/output.xml" # Only for zip
- builds:
- # The files must be in the directory DIR[WORKING,DATA]
- csit-vpp-perf-mrr-daily-master:
- -
- build: 1
- file: "{DIR[WORKING,DATA]}/output_mrr_1.xml"
- -
- build: 2
- file: "{DIR[WORKING,DATA]}/output_mrr_2.xml"
- -
- build: 3
- file: "{DIR[WORKING,DATA]}/output_mrr_3.xml"
- -
- build: 4
- file: "{DIR[WORKING,DATA]}/output_mrr_4.xml"
- -
- build: 5
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 6
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 7
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 8
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 9
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 10
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 11
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
- -
- build: 12
- file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
-
--
type: "static"
src-path: "{DIR[RST]}"
dst-path: "{DIR[WORKING,SRC]}"
-
type: "input" # Ignored in debug mode
+# general:
+# file-name: "robot-plugin.zip"
+# file-format: ".zip"
+# download-path: "{job}/{build}/robot/report/*zip*/{filename}"
+# extract: "robot-plugin/output.xml"
general:
file-name: "output.xml.gz"
file-format: ".gz"
download-path: "{job}/{build}/archives/{filename}"
extract: "output.xml"
-# file-name: "robot-plugin.zip"
-# file-format: ".zip"
-# download-path: "{job}/{build}/robot/report/*zip*/{filename}"
-# extract: "robot-plugin/output.xml"
builds:
csit-vpp-perf-mrr-daily-master:
start: 15
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index af4d707613..ebd84530a3 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -49,7 +49,6 @@ class Specification(object):
self._specification = {"environment": dict(),
"configuration": dict(),
- "debug": dict(),
"static": dict(),
"input": dict(),
"output": dict(),
@@ -95,15 +94,6 @@ class Specification(object):
return self._specification["static"]
@property
- def debug(self):
- """Getter - debug
-
- :returns: Debug specification
- :rtype: dict
- """
- return self._specification["debug"]
-
- @property
def is_debug(self):
"""Getter - debug mode
@@ -425,43 +415,6 @@ class Specification(object):
self.configuration["data-sets"][set_name][job] = builds
logging.info("Done.")
- def _parse_debug(self):
- """Parse debug specification in the specification YAML file.
- """
-
- if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
- return None
-
- logging.info("Parsing specification file: debug ...")
-
- idx = self._get_type_index("debug")
- if idx is None:
- self.environment["configuration"]["CFG[DEBUG]"] = 0
- return None
-
- try:
- for key, value in self._cfg_yaml[idx]["general"].items():
- self._specification["debug"][key] = value
-
- self._specification["input"]["builds"] = dict()
- for job, builds in self._cfg_yaml[idx]["builds"].items():
- if builds:
- self._specification["input"]["builds"][job] = list()
- for build in builds:
- self._specification["input"]["builds"][job].\
- append({"build": build["build"],
- "status": "downloaded",
- "file-name": self._replace_tags(
- build["file"],
- self.environment["paths"])})
- else:
- logging.warning("No build is defined for the job '{}'. "
- "Trying to continue without it.".
- format(job))
-
- except KeyError:
- raise PresentationError("No data to process.")
-
def _parse_input(self):
"""Parse input specification in the specification YAML file.
@@ -664,9 +617,7 @@ class Specification(object):
self._parse_env()
self._parse_configuration()
- self._parse_debug()
- if not self.debug:
- self._parse_input()
+ self._parse_input()
self._parse_output()
self._parse_static()
self._parse_elements()
diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py
index 95faffdd66..59dbfa634b 100644
--- a/resources/tools/presentation/utils.py
+++ b/resources/tools/presentation/utils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -198,8 +198,8 @@ def execute_command(cmd):
stdout, stderr = proc.communicate()
- logging.info(stdout)
- logging.info(stderr)
+ logging.debug(stdout)
+ logging.debug(stderr)
if proc.returncode != 0:
logging.error(" Command execution failed.")
@@ -250,10 +250,7 @@ def archive_input_data(spec):
logging.info(" Archiving the input data files ...")
- if spec.is_debug:
- extension = spec.debug["input-format"]
- else:
- extension = spec.input["file-format"]
+ extension = spec.input["file-format"]
data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"],
extension=extension)
dst = spec.environment["paths"]["DIR[STATIC,ARCH]"]