PAL: Add s3_storage as the data source

Priorities for download:
1. s3_storage
2. logs.fd.io
3. docs.fd.io release
4. docs.fd.io master
5. jenkins

Change-Id: I7114715c3ec9a33acc8edf3c5c473f4eaffa639d
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 21 insertions, 5 deletions

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 8b941f2f94..47249b1323 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -200,11 +200,10 @@ def download_and_unzip_data_file(spec, job, build, pid):
     :rtype: bool
     """
 
-    # Try to download .gz from logs.fd.io
-
+    # Try to download .gz from s3_storage
     file_name = spec.input[u"file-name"]
     url = u"{0}/{1}".format(
-        spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+        spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'],
         spec.input[u'download-path'].format(
             job=job, build=build[u'build'], filename=file_name
         )
@@ -220,6 +219,25 @@ def download_and_unzip_data_file(spec, job, build, pid):
     success, downloaded_name = _download_file(url, new_name, arch=arch)
 
     if not success:
+        # Try to download .gz from logs.fd.io
+        file_name = spec.input[u"file-name"]
+        url = u"{0}/{1}".format(
+            spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+            spec.input[u'download-path'].format(
+                job=job, build=build[u'build'], filename=file_name
+            )
+        )
+        new_name = join(
+            spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
+            f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
+        )
+
+        logging.info(f"Trying to download {url}")
+
+        arch = bool(spec.configuration.get(u"archive-inputs", True))
+        success, downloaded_name = _download_file(url, new_name, arch=arch)
+
+    if not success:
 
         # Try to download .gz or .zip from docs.fd.io
         file_name = (spec.input[u"file-name"], spec.input[u"zip-file-name"])
@@ -260,8 +278,6 @@ def download_and_unzip_data_file(spec, job, build, pid):
         download_path = spec.input[u"zip-download-path"]
         if job.startswith(u"csit-"):
             url = spec.environment[u"urls"][u"URL[JENKINS,CSIT]"]
-        elif job.startswith(u"hc2vpp-"):
-            url = spec.environment[u"urls"][u"URL[JENKINS,HC]"]
         else:
             raise PresentationError(f"No url defined for the job {job}.")