aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2021-03-05 14:10:08 +0100
committerTibor Frank <tifrank@cisco.com>2021-03-05 14:00:33 +0000
commit36eb135f679c1ab2a41b6f5bb510ed9da3bdda2c (patch)
treed187c63358dc618234c97e86bee96a3b9467152e
parent623ec6407652534c2062b6446f4378809c221477 (diff)
PAL: repeat the download if it fails 2
Change-Id: I9bba50281bfe65f62a1cfab76364d834dc10d80c Signed-off-by: Tibor Frank <tifrank@cisco.com> (cherry picked from commit b83a029d3c6524b7f0139746fcc4325e692976b6)
-rw-r--r--resources/tools/presentation/input_data_files.py12
-rw-r--r--resources/tools/presentation/requirements.txt1
2 files changed, 9 insertions, 4 deletions
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 023d52a0d6..e5b26e910c 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -21,7 +21,7 @@ import gzip
from os import rename, mkdir
from os.path import join
-from http.client import responses, IncompleteRead
+from http.client import responses, HTTPException
from zipfile import ZipFile, is_zipfile, BadZipfile
import requests
@@ -30,6 +30,8 @@ from requests.adapters import HTTPAdapter, Retry
from requests.exceptions import RequestException
from requests import codes
+from urllib3.exceptions import HTTPError
+
from pal_errors import PresentationError
@@ -89,9 +91,9 @@ def _download_file(url, file_name, arch=False, verify=True, repeat=1):
session.mount(u"https://", adapter)
return session
+ success = False
while repeat:
repeat -= 1
- success = False
session = None
try:
logging.info(f" Connecting to {url} ...")
@@ -137,7 +139,7 @@ def _download_file(url, file_name, arch=False, verify=True, repeat=1):
success = True
repeat = 0
- except IncompleteRead as err:
+ except (HTTPException, HTTPError) as err:
logging.error(f"Connection broken:\n{repr(err)}")
except RequestException as err:
logging.error(f"HTTP Request exception:\n{repr(err)}")
@@ -247,7 +249,9 @@ def download_and_unzip_data_file(spec, job, build, pid):
logging.info(f"Trying to download {url}")
arch = bool(spec.configuration.get(u"archive-inputs", True))
- success, downloaded_name = _download_file(url, new_name, arch=arch)
+ success, downloaded_name = _download_file(
+ url, new_name, arch=arch, verify=True, repeat=3
+ )
if not success:
diff --git a/resources/tools/presentation/requirements.txt b/resources/tools/presentation/requirements.txt
index 0063926a8c..8e3d170978 100644
--- a/resources/tools/presentation/requirements.txt
+++ b/resources/tools/presentation/requirements.txt
@@ -8,3 +8,4 @@ pandas==0.25.3
plotly==4.1.1
PTable==0.9.2
hdrhistogram==0.6.1
+urllib3==1.25.6