aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2019-08-14 08:49:30 +0200
committerTibor Frank <tifrank@cisco.com>2019-08-14 07:23:19 +0000
commit4eba78a1601360187b89848113564c4a4dce3d15 (patch)
treed4cd33c678b2deb51e96baec3fee3a2389bb3198
parentcbf5cf1f4fe42138faa00b3174dc7fd6a851a9eb (diff)
PAL: Archiving of input data
Change-Id: I0902613047d7543378a4b7d46811fc67776c6ad9 Signed-off-by: Tibor Frank <tifrank@cisco.com> (cherry picked from commit 9e476b274acf41b5ac358bc4dba3a0f1e16f04b2)
-rw-r--r--resources/tools/presentation/input_data_files.py32
1 files changed, 24 insertions, 8 deletions
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 0f32ed552c..441a240d79 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -88,32 +88,45 @@ def _download_file(url, file_name, log, arch=False):
success = False
try:
log.append(("INFO", " Connecting to '{0}' ...".format(url)))
- response = requests_retry_session().get(url, stream=True)
+ session = requests_retry_session()
+ response = session.get(url, stream=True)
code = response.status_code
log.append(("INFO", " {0}: {1}".format(code, responses[code])))
if code != codes["OK"]:
+ if session:
+ session.close()
url = url.replace("_info", "")
log.append(("INFO", " Connecting to '{0}' ...".format(url)))
- response = requests_retry_session().get(url, stream=True)
+ session = requests_retry_session()
+ response = session.get(url, stream=True)
code = response.status_code
log.append(("INFO", " {0}: {1}".format(code, responses[code])))
if code != codes["OK"]:
return False, file_name
file_name = file_name.replace("_info", "")
- log.append(("INFO", " Downloading the file '{0}' to '{1}' ...".
- format(url, file_name)))
-
dst_file_name = file_name.replace(".gz", "")
+ log.append(("INFO", " Downloading the file '{0}' to '{1}' ...".
+ format(url, dst_file_name)))
with open(dst_file_name, "wb") as file_handle:
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
if chunk:
file_handle.write(chunk)
if arch and ".gz" in file_name:
- with open(file_name, "wb") as file_handle:
- file_handle.write(response.raw.read())
+ if session:
+ session.close()
+ log.append(("INFO", " Downloading the file '{0}' to '{1}' ...".
+ format(url, file_name)))
+ session = requests_retry_session()
+ response = session.get(url, stream=True)
+ if response.status_code == codes["OK"]:
+ with open(file_name, "wb") as file_handle:
+ file_handle.write(response.raw.read())
+ else:
+ log.append(("ERROR", "Not possible to download the file '{0}' "
+ "to '{1}' ...".format(url, file_name)))
success = True
except ConnectionError as err:
@@ -135,6 +148,9 @@ def _download_file(url, file_name, log, arch=False):
except (IOError, ValueError, KeyError) as err:
log.append(("ERROR", "Download failed."))
log.append(("DEBUG", repr(err)))
+ finally:
+ if session:
+ session.close()
log.append(("INFO", " Download finished."))
return success, file_name
@@ -218,7 +234,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
job=job, sep=SEPARATOR, build=build["build"],
name=file_name))
- logging.info("Downloading {0}".format(url))
+ logging.info("Trying to download {0}".format(url))
arch = True if spec.configuration.get("archive-inputs", True) else False
success, downloaded_name = _download_file(url, new_name, log, arch=arch)