diff options
Diffstat (limited to 'resources/tools/presentation/input_data_files.py')
-rw-r--r-- | resources/tools/presentation/input_data_files.py | 204 |
1 files changed, 204 insertions, 0 deletions
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py new file mode 100644 index 0000000000..d57d1545ea --- /dev/null +++ b/resources/tools/presentation/input_data_files.py @@ -0,0 +1,204 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Inputs +Download all data. +""" + +import logging + +from os import rename, remove +from os.path import join, getsize +from shutil import move +from zipfile import ZipFile, is_zipfile, BadZipfile +from httplib import responses +from requests import get, codes, RequestException, Timeout, TooManyRedirects, \ + HTTPError, ConnectionError + +from errors import PresentationError + + +# Chunk size used for file download +CHUNK_SIZE = 512 + +# Separator used in file names +SEPARATOR = "__" + + +def download_data_files(spec): + """Download all data specified in the specification file in the section + type: input --> builds. + + :param spec: Specification. + :type spec: Specification + :raises: PresentationError if there is no url defined for the job. + """ + + for job, builds in spec.builds.items(): + for build in builds: + if job.startswith("csit-"): + url = spec.environment["urls"]["URL[JENKINS,CSIT]"] + elif job.startswith("hc2vpp-"): + url = spec.environment["urls"]["URL[JENKINS,HC]"] + else: + raise PresentationError("No url defined for the job '{}'.". + format(job)) + file_name = spec.input["file-name"] + full_name = spec.input["download-path"].\ + format(job=job, build=build["build"], filename=file_name) + url = "{0}/{1}".format(url, full_name) + new_name = join( + spec.environment["paths"]["DIR[WORKING,DATA]"], + "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR, + build=build["build"], + name=file_name)) + + logging.info("Downloading the file '{0}' to '{1}'.". + format(url, new_name)) + + status = "failed" + try: + response = get(url, stream=True) + code = response.status_code + if code != codes["OK"]: + logging.error("{0}: {1}".format(code, responses[code])) + spec.set_input_state(job, build["build"], "not found") + continue + + file_handle = open(new_name, "wb") + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + file_handle.write(chunk) + file_handle.close() + + expected_length = None + try: + expected_length = int(response.headers["Content-Length"]) + logging.debug(" Expected file size: {0}B". + format(expected_length)) + except KeyError: + logging.debug(" No information about expected size.") + + real_length = getsize(new_name) + logging.debug(" Downloaded size: {0}B".format(real_length)) + + if expected_length: + if real_length == expected_length: + status = "downloaded" + logging.info("{0}: {1}".format(code, responses[code])) + else: + logging.error("The file size differs from the expected " + "size.") + else: + status = "downloaded" + logging.info("{0}: {1}".format(code, responses[code])) + + except ConnectionError as err: + logging.error("Not possible to connect to '{0}'.".format(url)) + logging.debug(err) + except HTTPError as err: + logging.error("Invalid HTTP response from '{0}'.".format(url)) + logging.debug(err) + except TooManyRedirects as err: + logging.error("Request exceeded the configured number " + "of maximum re-directions.") + logging.debug(err) + except Timeout as err: + logging.error("Request timed out.") + logging.debug(err) + except RequestException as err: + logging.error("Unexpected HTTP request exception.") + logging.debug(err) + except (IOError, ValueError, KeyError) as err: + logging.error("Download failed.") + logging.debug("Reason: {0}".format(err)) + + spec.set_input_state(job, build["build"], status) + spec.set_input_file_name(job, build["build"], new_name) + + if status == "failed": + logging.info("Removing the file '{0}'".format(new_name)) + try: + remove(new_name) + except OSError as err: + logging.warning(str(err)) + spec.set_input_file_name(job, build["build"], None) + + unzip_files(spec) + + +def unzip_files(spec): + """Unzip downloaded zip files + + :param spec: Specification. + :type spec: Specification + :raises: PresentationError if the zip file does not exist or it is not a + zip file. + """ + + if spec.is_debug: + data_file = spec.debug["extract"] + else: + data_file = spec.input["extract"] + + for job, builds in spec.builds.items(): + for build in builds: + if build["status"] == "failed" or build["status"] == "not found": + continue + try: + status = "failed" + directory = spec.environment["paths"]["DIR[WORKING,DATA]"] + file_name = join(build["file-name"]) + + if build["status"] == "downloaded" and is_zipfile(file_name): + logging.info("Unziping: '{0}' from '{1}'.". + format(data_file, file_name)) + new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2], + SEPARATOR, + data_file.split("/")[-1]) + try: + with ZipFile(file_name, 'r') as zip_file: + zip_file.extract(data_file, directory) + logging.info("Moving {0} to {1} ...". + format(join(directory, data_file), + directory)) + move(join(directory, data_file), directory) + logging.info("Renaming the file '{0}' to '{1}'". + format(join(directory, + data_file.split("/")[-1]), + new_name)) + rename(join(directory, data_file.split("/")[-1]), + new_name) + status = "unzipped" + spec.set_input_state(job, build["build"], status) + spec.set_input_file_name(job, build["build"], + new_name) + except (BadZipfile, RuntimeError) as err: + logging.error("Failed to unzip the file '{0}': {1}.". + format(file_name, str(err))) + except OSError as err: + logging.error("Failed to rename the file '{0}': {1}.". + format(data_file, str(err))) + finally: + if status == "failed": + spec.set_input_file_name(job, build["build"], + None) + else: + raise PresentationError("The file '{0}' does not exist or " + "it is not a zip file". + format(file_name)) + + spec.set_input_state(job, build["build"], status) + + except KeyError: + pass |