diff options
Diffstat (limited to 'resources/tools')
| -rw-r--r-- | resources/tools/presentation/environment.py | 40 | ||||
| -rw-r--r-- | resources/tools/presentation/generator_CPTA.py | 4 | ||||
| -rw-r--r-- | resources/tools/presentation/generator_tables.py | 5 | ||||
| -rw-r--r-- | resources/tools/presentation/input_data_files.py | 380 | ||||
| -rw-r--r-- | resources/tools/presentation/input_data_parser.py | 75 | ||||
| -rw-r--r-- | resources/tools/presentation/pal.py | 45 | ||||
| -rw-r--r-- | resources/tools/presentation/specification.yaml | 30 | ||||
| -rw-r--r-- | resources/tools/presentation/specification_CPTA.yaml | 64 | ||||
| -rw-r--r-- | resources/tools/presentation/specification_parser.py | 53 | ||||
| -rw-r--r-- | resources/tools/presentation/utils.py | 11 |
10 files changed, 256 insertions, 451 deletions
diff --git a/resources/tools/presentation/environment.py b/resources/tools/presentation/environment.py index 1a1c3db032..a2fa9a0d5b 100644 --- a/resources/tools/presentation/environment.py +++ b/resources/tools/presentation/environment.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -51,35 +51,6 @@ class Environment(object): """ return self._env - def _set_environment_variables(self): - """Set environment variables. - """ - logging.info("Setting the environment variables ...") - # logging.debug("Environment variables before:\n{}".format(os.environ)) - - count = 1 - - for var, value in self._env["configuration"].items(): - logging.debug(" {:3d} Setting the variable {} = {}". - format(count, var, value)) - os.environ[var] = str(value) - count += 1 - - for var, value in self._env["paths"].items(): - logging.debug(" {:3d} Setting the variable {} = {}". - format(count, var, value)) - os.environ[var] = str(value) - count += 1 - - for var, value in self._env["urls"].items(): - logging.debug(" {:3d} Setting the variable {} = {}". - format(count, var, value)) - os.environ[var] = str(value) - count += 1 - - # logging.debug("Environment variables after:\n{}".format(os.environ)) - logging.info("Done.") - def _make_dirs(self): """Create the directories specified in the 'make-dirs' part of 'environment' section in the specification file. @@ -122,7 +93,6 @@ class Environment(object): """Set the environment. """ - self._set_environment_variables() self._make_dirs() @@ -147,10 +117,10 @@ def clean_environment(env): if os.path.isdir(dir_to_remove): try: shutil.rmtree(dir_to_remove) - except OSError: - pass - # raise PresentationError("Cannot remove the directory '{}'". - # format(dir_to_remove)) + except OSError as err: + logging.warning("Cannot remove the directory '{}'". + format(dir_to_remove)) + logging.debug(str(err)) else: logging.warning("The directory '{}' does not exist.". format(dir_to_remove)) diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py index e27a52172b..89e9a41b48 100644 --- a/resources/tools/presentation/generator_CPTA.py +++ b/resources/tools/presentation/generator_CPTA.py @@ -32,7 +32,7 @@ from utils import split_outliers, archive_input_data, execute_command HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \ '-b html -E ' \ '-t html ' \ - '-D version="Generated on {date}" ' \ + '-D version="{date}" ' \ '{working_dir} ' \ '{build_dir}/' @@ -64,7 +64,7 @@ def generate_cpta(spec, data): ret_code = _generate_all_charts(spec, data) cmd = HTML_BUILDER.format( - date=datetime.date.today().strftime('%d-%b-%Y'), + date=datetime.datetime.utcnow().strftime('%m/%d/%Y %H:%M UTC'), working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"], build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"]) execute_command(cmd) diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 36933cc0e5..38439bac5c 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -822,9 +822,6 @@ def table_performance_trending_dashboard(table, input_data): rel_change_long = round( ((last_median_t - max_median) / max_median) * 100, 2) - logging.info("rel_change_last : {}".format(rel_change_last)) - logging.info("rel_change_long : {}".format(rel_change_long)) - tbl_lst.append( [name, '-' if isnan(last_median_t) else diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index d81f64fbe6..24dc3a57d8 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,11 +16,10 @@ Download all data. """ import re -import gzip import logging -from os import rename, remove -from os.path import join, getsize +from os import rename +from os.path import join from shutil import move from zipfile import ZipFile, is_zipfile, BadZipfile from httplib import responses @@ -39,216 +38,179 @@ SEPARATOR = "__" REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)') -def download_data_files(spec): - """Download all data specified in the specification file in the section - type: input --> builds. +def _download_file(url, file_name): + """Download a file with input data. - :param spec: Specification. + :param url: URL to the file to download. + :param file_name: Name of file to download. + :type url: str + :type file_name: str + :returns: True if the download was successful, otherwise False. + :rtype: bool + """ + + success = False + try: + logging.info(" Connecting to '{0}' ...".format(url)) + + response = get(url, stream=True) + code = response.status_code + + logging.info(" {0}: {1}".format(code, responses[code])) + + if code != codes["OK"]: + return False + + logging.info(" Downloading the file '{0}' to '{1}' ...". + format(url, file_name)) + + file_handle = open(file_name, "wb") + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + file_handle.write(chunk) + file_handle.close() + success = True + except ConnectionError as err: + logging.error("Not possible to connect to '{0}'.".format(url)) + logging.debug(str(err)) + except HTTPError as err: + logging.error("Invalid HTTP response from '{0}'.".format(url)) + logging.debug(str(err)) + except TooManyRedirects as err: + logging.error("Request exceeded the configured number " + "of maximum re-directions.") + logging.debug(str(err)) + except Timeout as err: + logging.error("Request timed out.") + logging.debug(str(err)) + except RequestException as err: + logging.error("Unexpected HTTP request exception.") + logging.debug(str(err)) + except (IOError, ValueError, KeyError) as err: + logging.error("Download failed.") + logging.debug(str(err)) + + logging.info(" Download finished.") + return success + + +def _unzip_file(spec, job, build): + """Unzip downloaded source file. + + :param spec: Specification read form the specification file. + :param job: Name of the Jenkins job. + :param build: Information about the build. :type spec: Specification - :raises: PresentationError if there is no url defined for the job. + :type job: str + :type build: dict + :returns: True if the download was successful, otherwise False. + :rtype: bool """ - for job, builds in spec.builds.items(): - for build in builds: - if job.startswith("csit-"): - if spec.input["file-name"].endswith(".zip"): - url = spec.environment["urls"]["URL[JENKINS,CSIT]"] - elif spec.input["file-name"].endswith(".gz"): - url = spec.environment["urls"]["URL[NEXUS,LOG]"] - else: - logging.error("Not supported file format.") - continue - elif job.startswith("hc2vpp-"): - url = spec.environment["urls"]["URL[JENKINS,HC]"] - else: - raise PresentationError("No url defined for the job '{}'.". - format(job)) - file_name = spec.input["file-name"] - full_name = spec.input["download-path"].\ - format(job=job, build=build["build"], filename=file_name) - url = "{0}/{1}".format(url, full_name) - new_name = join( - spec.environment["paths"]["DIR[WORKING,DATA]"], - "{job}{sep}{build}{sep}{name}".format(job=job, - sep=SEPARATOR, - build=build["build"], - name=file_name)) - logging.info( - "Downloading the file '{0}' to '{1}' ...".format(url, new_name)) - - status = "failed" - try: - response = get(url, stream=True) - code = response.status_code - - if code != codes["OK"]: - logging.warning( - "Jenkins: {0}: {1}.".format(code, responses[code])) - logging.info("Trying to download from Nexus:") - spec.set_input_state(job, build["build"], "not found") - if code == codes["not_found"]: - release = re.search(REGEX_RELEASE, job).group(2) - nexus_file_name = "{job}{sep}{build}{sep}{name}".\ - format(job=job, sep=SEPARATOR, build=build["build"], - name=file_name) - try: - release = "rls{0}".format(int(release)) - except ValueError: - pass - url = "{url}/{release}/{dir}/{file}".\ - format(url=spec.environment["urls"]["URL[NEXUS]"], - release=release, - dir=spec.environment["urls"]["DIR[NEXUS]"], - file=nexus_file_name) - logging.info("Downloading the file '{0}' to '{1}' ...". - format(url, new_name)) - response = get(url, stream=True) - code = response.status_code - if code != codes["OK"]: - logging.error( - "Nexus: {0}: {1}".format(code, responses[code])) - spec.set_input_state( - job, build["build"], "not found") - continue - - file_handle = open(new_name, "wb") - for chunk in response.iter_content(chunk_size=CHUNK_SIZE): - if chunk: - file_handle.write(chunk) - file_handle.close() - - if spec.input["file-name"].endswith(".zip"): - expected_length = None - try: - expected_length = int(response. - headers["Content-Length"]) - logging.debug(" Expected file size: {0}B". - format(expected_length)) - except KeyError: - logging.debug(" No information about expected size.") - - real_length = getsize(new_name) - logging.debug(" Downloaded size: {0}B".format(real_length)) - - if expected_length: - if real_length == expected_length: - status = "downloaded" - logging.info("{0}: {1}".format(code, - responses[code])) - else: - logging.error("The file size differs from the " - "expected size.") - else: - status = "downloaded" - logging.info("{0}: {1}".format(code, responses[code])) - - elif spec.input["file-name"].endswith(".gz"): - if "docs.fd.io" in url: - execute_command("gzip --decompress --keep --force {0}". - format(new_name)) - else: - rename(new_name, new_name[:-3]) - execute_command("gzip --keep {0}".format(new_name[:-3])) - new_name = new_name[:-3] - status = "downloaded" - logging.info("{0}: {1}".format(code, responses[code])) - - except ConnectionError as err: - logging.error("Not possible to connect to '{0}'.".format(url)) - logging.debug(err) - except HTTPError as err: - logging.error("Invalid HTTP response from '{0}'.".format(url)) - logging.debug(err) - except TooManyRedirects as err: - logging.error("Request exceeded the configured number " - "of maximum re-directions.") - logging.debug(err) - except Timeout as err: - logging.error("Request timed out.") - logging.debug(err) - except RequestException as err: - logging.error("Unexpected HTTP request exception.") - logging.debug(err) - except (IOError, ValueError, KeyError) as err: - logging.error("Download failed.") - logging.debug("Reason: {0}".format(err)) - - spec.set_input_state(job, build["build"], status) - spec.set_input_file_name(job, build["build"], new_name) - - if status == "failed": - logging.info("Removing the file '{0}'".format(new_name)) - try: - remove(new_name) - except OSError as err: - logging.warning(str(err)) - spec.set_input_file_name(job, build["build"], None) - - unzip_files(spec) - - -def unzip_files(spec): - """Unzip downloaded zip files - - :param spec: Specification. + data_file = spec.input["extract"] + file_name = build["file-name"] + directory = spec.environment["paths"]["DIR[WORKING,DATA]"] + new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2], + SEPARATOR, + data_file.split("/")[-1]) + logging.info(" Unzipping: '{0}' from '{1}'.". + format(data_file, file_name)) + try: + with ZipFile(file_name, 'r') as zip_file: + zip_file.extract(data_file, directory) + logging.info(" Moving {0} to {1} ...". + format(join(directory, data_file), directory)) + move(join(directory, data_file), directory) + logging.info(" Renaming the file '{0}' to '{1}'". + format(join(directory, data_file.split("/")[-1]), + new_name)) + rename(join(directory, data_file.split("/")[-1]), + new_name) + spec.set_input_file_name(job, build["build"], + new_name) + return True + except (BadZipfile, RuntimeError) as err: + logging.error("Failed to unzip the file '{0}': {1}.". + format(file_name, str(err))) + return False + except OSError as err: + logging.error("Failed to rename the file '{0}': {1}.". + format(data_file, str(err))) + return False + + +def download_and_unzip_data_file(spec, job, build): + """Download and unzip a source file. + + :param spec: Specification read form the specification file. + :param job: Name of the Jenkins job. + :param build: Information about the build. :type spec: Specification - :raises: PresentationError if the zip file does not exist or it is not a - zip file. + :type job: str + :type build: dict + :returns: True if the download was successful, otherwise False. + :rtype: bool """ - if spec.is_debug: - data_file = spec.debug["extract"] + if job.startswith("csit-"): + if spec.input["file-name"].endswith(".zip"): + url = spec.environment["urls"]["URL[JENKINS,CSIT]"] + elif spec.input["file-name"].endswith(".gz"): + url = spec.environment["urls"]["URL[NEXUS,LOG]"] + else: + logging.error("Not supported file format.") + return False + elif job.startswith("hc2vpp-"): + url = spec.environment["urls"]["URL[JENKINS,HC]"] + else: + raise PresentationError("No url defined for the job '{}'.". + format(job)) + file_name = spec.input["file-name"] + full_name = spec.input["download-path"]. \ + format(job=job, build=build["build"], filename=file_name) + url = "{0}/{1}".format(url, full_name) + new_name = join(spec.environment["paths"]["DIR[WORKING,DATA]"], + "{job}{sep}{build}{sep}{name}". + format(job=job, sep=SEPARATOR, build=build["build"], + name=file_name)) + # Download the file from the defined source (Jenkins, logs.fd.io): + success = _download_file(url, new_name) + + # If not successful, download from docs.fd.io: + if not success: + logging.info(" Trying to download from https://docs.fd.io:") + release = re.search(REGEX_RELEASE, job).group(2) + nexus_file_name = "{job}{sep}{build}{sep}{name}". \ + format(job=job, sep=SEPARATOR, build=build["build"], name=file_name) + try: + release = "rls{0}".format(int(release)) + except ValueError: + pass + url = "{url}/{release}/{dir}/{file}". \ + format(url=spec.environment["urls"]["URL[NEXUS]"], + release=release, + dir=spec.environment["urls"]["DIR[NEXUS]"], + file=nexus_file_name) + success = _download_file(url, new_name) + + if success: + spec.set_input_file_name(job, build["build"], new_name) + else: + return False + + if spec.input["file-name"].endswith(".gz"): + if "docs.fd.io" in url: + execute_command("gzip --decompress --keep --force {0}". + format(new_name)) + else: + rename(new_name, new_name[:-3]) + execute_command("gzip --keep {0}".format(new_name[:-3])) + spec.set_input_file_name(job, build["build"], new_name[:-3]) + + if spec.input["file-name"].endswith(".zip"): + if is_zipfile(file_name): + return _unzip_file(spec, job, build) + else: + return False else: - data_file = spec.input["extract"] - - for job, builds in spec.builds.items(): - for build in builds: - if build["status"] == "failed" or build["status"] == "not found": - continue - try: - status = "failed" - directory = spec.environment["paths"]["DIR[WORKING,DATA]"] - file_name = join(build["file-name"]) - - if build["status"] == "downloaded": - logging.info("Unziping: '{0}' from '{1}'.". - format(data_file, file_name)) - new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2], - SEPARATOR, - data_file.split("/")[-1]) - try: - if is_zipfile(file_name): - with ZipFile(file_name, 'r') as zip_file: - zip_file.extract(data_file, directory) - logging.info("Moving {0} to {1} ...". - format(join(directory, data_file), - directory)) - move(join(directory, data_file), directory) - logging.info("Renaming the file '{0}' to '{1}'". - format(join(directory, - data_file.split("/")[-1]), - new_name)) - rename(join(directory, data_file.split("/")[-1]), - new_name) - spec.set_input_file_name(job, build["build"], - new_name) - status = "unzipped" - spec.set_input_state(job, build["build"], status) - except (BadZipfile, RuntimeError) as err: - logging.error("Failed to unzip the file '{0}': {1}.". - format(file_name, str(err))) - except OSError as err: - logging.error("Failed to rename the file '{0}': {1}.". - format(data_file, str(err))) - finally: - if status == "failed": - spec.set_input_file_name(job, build["build"], None) - else: - raise PresentationError("The file '{0}' does not exist or " - "it is not a zip file". - format(file_name)) - - spec.set_input_state(job, build["build"], status) - - except KeyError: - pass + return True diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 0ad07a95db..e12e2fb8df 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -29,6 +29,8 @@ from collections import OrderedDict from string import replace from os import remove +from input_data_files import download_and_unzip_data_file + class ExecutionChecker(ResultVisitor): """Class to traverse through the test suite structure. @@ -179,7 +181,7 @@ class ExecutionChecker(ResultVisitor): REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s' r'tx\s(\d*),\srx\s(\d*)') - def __init__(self, **metadata): + def __init__(self, metadata): """Initialisation. :param metadata: Key-value pairs to be included in "metadata" part of @@ -251,8 +253,6 @@ class ExecutionChecker(ResultVisitor): self._data["metadata"]["version"] = self._version self._msg_type = None - logging.info(" VPP version: {0}".format(self._version)) - def _get_vat_history(self, msg): """Called when extraction of VAT command history is required. @@ -748,21 +748,29 @@ class InputData(object): return self.data[job][build]["tests"] @staticmethod - def _parse_tests(job, build): + def _parse_tests(job, build, get_timestamp=False): """Process data from robot output.xml file and return JSON structured data. :param job: The name of job which build output data will be processed. :param build: The build which output data will be processed. + :param get_timestamp: If True, timestamp is read form the xml source + file. :type job: str :type build: dict + :type get_timestamp: bool :returns: JSON data structure. :rtype: dict """ - tree = ET.parse(build["file-name"]) - root = tree.getroot() - generated = root.attrib["generated"] + metadata = { + "job": job, + "build": build + } + if get_timestamp: + tree = ET.parse(build["file-name"]) + root = tree.getroot() + metadata["generated"] = root.attrib["generated"] with open(build["file-name"], 'r') as data_file: try: @@ -771,46 +779,55 @@ class InputData(object): logging.error("Error occurred while parsing output.xml: {0}". format(err)) return None - checker = ExecutionChecker(job=job, build=build, generated=generated) + checker = ExecutionChecker(metadata) result.visit(checker) return checker.data - def read_data(self): - """Parse input data from input files and store in pandas' Series. + def download_and_parse_data(self, get_timestamp=False): + """Download the input data files, parse input data from input files and + store in pandas' Series. + + :param get_timestamp: If True, timestamp is read form the xml source + file. + :type get_timestamp: bool """ - logging.info("Parsing input files ...") + logging.info("Downloading and parsing input files ...") job_data = dict() for job, builds in self._cfg.builds.items(): - logging.info(" Extracting data from the job '{0}' ...'". + logging.info(" Processing data from the job '{0}' ...'". format(job)) builds_data = dict() for build in builds: - if build["status"] == "failed" \ - or build["status"] == "not found": + logging.info(" Processing the build '{0}'". + format(build["build"])) + self._cfg.set_input_state(job, build["build"], "failed") + if not download_and_unzip_data_file(self._cfg, job, build): + logging.error("It is not possible to download the input " + "data file from the job '{job}', build " + "'{build}', or it is damaged. Skipped.". + format(job=job, build=build["build"])) continue - logging.info(" Extracting data from the build '{0}'". + + logging.info(" Processing data from the build '{0}' ...". format(build["build"])) - logging.info(" Processing the file '{0}'". - format(build["file-name"])) + data = InputData._parse_tests(job, build, + get_timestamp=get_timestamp) + if data is None: + logging.error("Input data file from the job '{job}', build " + "'{build}' is damaged. Skipped.". + format(job=job, build=build["build"])) + continue - data = InputData._parse_tests(job, build) + self._cfg.set_input_state(job, build["build"], "processed") - logging.info(" Removing the file '{0}'". - format(build["file-name"])) try: remove(build["file-name"]) - build["status"] = "processed" except OSError as err: - logging.error(" Cannot remove the file '{0}': {1}". + logging.error("Cannot remove the file '{0}': {1}". format(build["file-name"], err)) - if data is None: - logging.error("Input data file from the job '{job}', build " - "'{build}' is damaged. Skipped.". - format(job=job, build=build["build"])) - continue build_data = pd.Series({ "metadata": pd.Series(data["metadata"].values(), @@ -818,9 +835,9 @@ class InputData(object): "suites": pd.Series(data["suites"].values(), index=data["suites"].keys()), "tests": pd.Series(data["tests"].values(), - index=data["tests"].keys()), - }) + index=data["tests"].keys())}) builds_data[str(build["build"])] = build_data + build["status"] = "processed" logging.info(" Done.") job_data[job] = pd.Series(builds_data.values(), diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py index 98642c898c..2268801407 100644 --- a/resources/tools/presentation/pal.py +++ b/resources/tools/presentation/pal.py @@ -21,7 +21,6 @@ import logging from errors import PresentationError from environment import Environment, clean_environment from specification_parser import Specification -from input_data_files import download_data_files, unzip_files from input_data_parser import InputData from generator_tables import generate_tables from generator_plots import generate_plots @@ -30,8 +29,6 @@ from static_content import prepare_static_content from generator_report import generate_report from generator_CPTA import generate_cpta -from pprint import pprint - def parse_args(): """Parse arguments from cmd line. @@ -86,21 +83,24 @@ def main(): logging.critical("Finished with error.") return 1 - ret_code = 0 + if spec.output["output"] not in ("report", "CPTA"): + logging.critical("The output '{0}' is not supported.". + format(spec.output["output"])) + return 1 + + ret_code = 1 try: env = Environment(spec.environment, args.force) env.set_environment() - if spec.is_debug: - if spec.debug["input-format"] == "zip": - unzip_files(spec) - else: - download_data_files(spec) - prepare_static_content(spec) data = InputData(spec) - data.read_data() + + if spec.output["output"] == "report": + data.download_and_parse_data(get_timestamp=False) + elif spec.output["output"] == "CPTA": + data.download_and_parse_data(get_timestamp=True) generate_tables(spec, data) generate_plots(spec, data) @@ -112,21 +112,16 @@ def main(): elif spec.output["output"] == "CPTA": sys.stdout.write(generate_cpta(spec, data)) logging.info("Successfully finished.") - else: - logging.critical("The output '{0}' is not supported.". - format(spec.output["output"])) - ret_code = 1 - - except (KeyError, ValueError, PresentationError) as err: - logging.info("Finished with an error.") - logging.critical(str(err)) - ret_code = 1 - except Exception as err: - logging.info("Finished with an unexpected error.") - logging.critical(str(err)) - ret_code = 1 + ret_code = 0 + + # except (KeyError, ValueError, PresentationError) as err: + # logging.info("Finished with an error.") + # logging.critical(str(err)) + # except Exception as err: + # logging.info("Finished with an unexpected error.") + # logging.critical(str(err)) finally: - if spec is not None and not spec.is_debug: + if spec is not None: clean_environment(spec.environment) return ret_code diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 4b055a84ce..97e616a15d 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -1,14 +1,5 @@ - type: "environment" - configuration: - # Debug mode: - # - Skip: - # - Download of input data files - # - Do: - # - Read data from given zip / xml files - # - Set the configuration as it is done in normal mode - # If the section "type: debug" is missing, CFG[DEBUG] is set to 0. - CFG[DEBUG]: 0 paths: # Top level directories: @@ -546,27 +537,6 @@ height: 1000 - - type: "debug" - general: - input-format: "xml" # zip or xml - extract: "robot-plugin/output.xml" # Only for zip - builds: - # The files must be in the directory DIR[WORKING,DATA] - csit-vpp-perf-1801-all: - - - build: 1 - file: "{DIR[WORKING,DATA]}/output.xml" - - - build: 2 - file: "{DIR[WORKING,DATA]}/output.xml" - - - build: 3 - file: "{DIR[WORKING,DATA]}/output.xml" - - - build: 4 - file: "{DIR[WORKING,DATA]}/output.xml" - -- type: "static" src-path: "{DIR[RST]}" dst-path: "{DIR[WORKING,SRC]}" diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index a570dccbbe..9b47083b2b 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -16,15 +16,6 @@ - type: "environment" - configuration: - # Debug mode: - # - Skip: - # - Download of input data files - # - Do: - # - Read data from given zip / xml files - # - Set the configuration as it is done in normal mode - # If the section "type: debug" is missing, CFG[DEBUG] is set to 0. - CFG[DEBUG]: 0 paths: # Top level directories: @@ -102,7 +93,6 @@ showgrid: True xaxis: showticklabels: True - # title: "csit-vpp-perf-mrr-daily-master/vpp-build" title: '<a href="https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master" target="_blank">csit-vpp-perf-mrr-daily-master-build</a>/<a href="https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/vpp/" target="_blank">vpp-build</a>' autorange: True showgrid: True @@ -130,66 +120,22 @@ namelength: -1 - - type: "debug" - general: - input-format: "xml" # zip or xml - extract: "robot-plugin/output.xml" # Only for zip - builds: - # The files must be in the directory DIR[WORKING,DATA] - csit-vpp-perf-mrr-daily-master: - - - build: 1 - file: "{DIR[WORKING,DATA]}/output_mrr_1.xml" - - - build: 2 - file: "{DIR[WORKING,DATA]}/output_mrr_2.xml" - - - build: 3 - file: "{DIR[WORKING,DATA]}/output_mrr_3.xml" - - - build: 4 - file: "{DIR[WORKING,DATA]}/output_mrr_4.xml" - - - build: 5 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 6 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 7 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 8 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 9 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 10 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 11 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - - - build: 12 - file: "{DIR[WORKING,DATA]}/output_mrr_5.xml" - -- type: "static" src-path: "{DIR[RST]}" dst-path: "{DIR[WORKING,SRC]}" - type: "input" # Ignored in debug mode +# general: +# file-name: "robot-plugin.zip" +# file-format: ".zip" +# download-path: "{job}/{build}/robot/report/*zip*/{filename}" +# extract: "robot-plugin/output.xml" general: file-name: "output.xml.gz" file-format: ".gz" download-path: "{job}/{build}/archives/{filename}" extract: "output.xml" -# file-name: "robot-plugin.zip" -# file-format: ".zip" -# download-path: "{job}/{build}/robot/report/*zip*/{filename}" -# extract: "robot-plugin/output.xml" builds: csit-vpp-perf-mrr-daily-master: start: 15 diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index af4d707613..ebd84530a3 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -49,7 +49,6 @@ class Specification(object): self._specification = {"environment": dict(), "configuration": dict(), - "debug": dict(), "static": dict(), "input": dict(), "output": dict(), @@ -95,15 +94,6 @@ class Specification(object): return self._specification["static"] @property - def debug(self): - """Getter - debug - - :returns: Debug specification - :rtype: dict - """ - return self._specification["debug"] - - @property def is_debug(self): """Getter - debug mode @@ -425,43 +415,6 @@ class Specification(object): self.configuration["data-sets"][set_name][job] = builds logging.info("Done.") - def _parse_debug(self): - """Parse debug specification in the specification YAML file. - """ - - if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1: - return None - - logging.info("Parsing specification file: debug ...") - - idx = self._get_type_index("debug") - if idx is None: - self.environment["configuration"]["CFG[DEBUG]"] = 0 - return None - - try: - for key, value in self._cfg_yaml[idx]["general"].items(): - self._specification["debug"][key] = value - - self._specification["input"]["builds"] = dict() - for job, builds in self._cfg_yaml[idx]["builds"].items(): - if builds: - self._specification["input"]["builds"][job] = list() - for build in builds: - self._specification["input"]["builds"][job].\ - append({"build": build["build"], - "status": "downloaded", - "file-name": self._replace_tags( - build["file"], - self.environment["paths"])}) - else: - logging.warning("No build is defined for the job '{}'. " - "Trying to continue without it.". - format(job)) - - except KeyError: - raise PresentationError("No data to process.") - def _parse_input(self): """Parse input specification in the specification YAML file. @@ -664,9 +617,7 @@ class Specification(object): self._parse_env() self._parse_configuration() - self._parse_debug() - if not self.debug: - self._parse_input() + self._parse_input() self._parse_output() self._parse_static() self._parse_elements() diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py index 95faffdd66..59dbfa634b 100644 --- a/resources/tools/presentation/utils.py +++ b/resources/tools/presentation/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -198,8 +198,8 @@ def execute_command(cmd): stdout, stderr = proc.communicate() - logging.info(stdout) - logging.info(stderr) + logging.debug(stdout) + logging.debug(stderr) if proc.returncode != 0: logging.error(" Command execution failed.") @@ -250,10 +250,7 @@ def archive_input_data(spec): logging.info(" Archiving the input data files ...") - if spec.is_debug: - extension = spec.debug["input-format"] - else: - extension = spec.input["file-format"] + extension = spec.input["file-format"] data_files = get_files(spec.environment["paths"]["DIR[WORKING,DATA]"], extension=extension) dst = spec.environment["paths"]["DIR[STATIC,ARCH]"] |