diff options
| author |   Tibor Frank <tifrank@cisco.com> | 2021-03-05 14:10:08 +0100 |
|---|---|---|
| committer | Tibor Frank <tifrank@cisco.com> | 2021-03-05 14:00:33 +0000 |
| commit | 36eb135f679c1ab2a41b6f5bb510ed9da3bdda2c (patch) | |
| tree | d187c63358dc618234c97e86bee96a3b9467152e | |
| parent | 623ec6407652534c2062b6446f4378809c221477 (diff) | |
PAL: repeat the download if it fails 2
Change-Id: I9bba50281bfe65f62a1cfab76364d834dc10d80c
Signed-off-by: Tibor Frank <tifrank@cisco.com>
(cherry picked from commit b83a029d3c6524b7f0139746fcc4325e692976b6)
| -rw-r--r-- | resources/tools/presentation/input_data_files.py | 12 | ||||
| -rw-r--r-- | resources/tools/presentation/requirements.txt | 1 |
2 files changed, 9 insertions, 4 deletions
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index 023d52a0d6..e5b26e910c 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -21,7 +21,7 @@ import gzip from os import rename, mkdir from os.path import join -from http.client import responses, IncompleteRead +from http.client import responses, HTTPException from zipfile import ZipFile, is_zipfile, BadZipfile import requests @@ -30,6 +30,8 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import RequestException from requests import codes +from urllib3.exceptions import HTTPError + from pal_errors import PresentationError @@ -89,9 +91,9 @@ def _download_file(url, file_name, arch=False, verify=True, repeat=1): session.mount(u"https://", adapter) return session + success = False while repeat: repeat -= 1 - success = False session = None try: logging.info(f" Connecting to {url} ...") @@ -137,7 +139,7 @@ def _download_file(url, file_name, arch=False, verify=True, repeat=1): success = True repeat = 0 - except IncompleteRead as err: + except (HTTPException, HTTPError) as err: logging.error(f"Connection broken:\n{repr(err)}") except RequestException as err: logging.error(f"HTTP Request exception:\n{repr(err)}") @@ -247,7 +249,9 @@ def download_and_unzip_data_file(spec, job, build, pid): logging.info(f"Trying to download {url}") arch = bool(spec.configuration.get(u"archive-inputs", True)) - success, downloaded_name = _download_file(url, new_name, arch=arch) + success, downloaded_name = _download_file( + url, new_name, arch=arch, verify=True, repeat=3 + ) if not success: diff --git a/resources/tools/presentation/requirements.txt b/resources/tools/presentation/requirements.txt index 0063926a8c..8e3d170978 100644 --- a/resources/tools/presentation/requirements.txt +++ b/resources/tools/presentation/requirements.txt @@ -8,3 +8,4 @@ pandas==0.25.3 plotly==4.1.1 PTable==0.9.2 hdrhistogram==0.6.1 +urllib3==1.25.6 |