aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2021-03-04 09:52:20 +0100
committerTibor Frank <tifrank@cisco.com>2021-03-05 05:38:27 +0000
commitdf0263faee6171805e8bc579a4e0a786e63c3a8d (patch)
treed5ba1f23212cd6d33905bbc81dc26c43f9e6097a
parentbb7e8747d67176367fabadac85bc4e2571a47c93 (diff)
PAL: Add s3_storage as the data source
Priorities for download: 1. s3_storage 2. logs.fd.io 3. docs.fd.io release 4. docs.fd.io master 5. jenkins Change-Id: I7114715c3ec9a33acc8edf3c5c473f4eaffa639d Signed-off-by: Tibor Frank <tifrank@cisco.com>
-rw-r--r--resources/tools/presentation/doc/pal_lld.rst5
-rw-r--r--resources/tools/presentation/input_data_files.py26
-rw-r--r--resources/tools/presentation/specification.yaml2
-rw-r--r--resources/tools/presentation/specification_CPTA.yaml1
-rw-r--r--resources/tools/presentation/specification_local.yaml2
5 files changed, 28 insertions, 8 deletions
diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst
index 9569433760..38ab6d9bcb 100644
--- a/resources/tools/presentation/doc/pal_lld.rst
+++ b/resources/tools/presentation/doc/pal_lld.rst
@@ -187,7 +187,10 @@ The structure of the section "Environment" is as follows (example):
urls:
URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
- URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+ URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+ URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+ URL[NEXUS,DOC]: "https://docs.fd.io/csit"
+ DIR[NEXUS,DOC]: "report/_static/archive"
make-dirs:
# List the directories which are created while preparing the environment.
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 8b941f2f94..47249b1323 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -200,11 +200,10 @@ def download_and_unzip_data_file(spec, job, build, pid):
:rtype: bool
"""
- # Try to download .gz from logs.fd.io
-
+ # Try to download .gz from s3_storage
file_name = spec.input[u"file-name"]
url = u"{0}/{1}".format(
- spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+ spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'],
spec.input[u'download-path'].format(
job=job, build=build[u'build'], filename=file_name
)
@@ -220,6 +219,25 @@ def download_and_unzip_data_file(spec, job, build, pid):
success, downloaded_name = _download_file(url, new_name, arch=arch)
if not success:
+ # Try to download .gz from logs.fd.io
+ file_name = spec.input[u"file-name"]
+ url = u"{0}/{1}".format(
+ spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+ spec.input[u'download-path'].format(
+ job=job, build=build[u'build'], filename=file_name
+ )
+ )
+ new_name = join(
+ spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
+ f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
+ )
+
+ logging.info(f"Trying to download {url}")
+
+ arch = bool(spec.configuration.get(u"archive-inputs", True))
+ success, downloaded_name = _download_file(url, new_name, arch=arch)
+
+ if not success:
# Try to download .gz or .zip from docs.fd.io
file_name = (spec.input[u"file-name"], spec.input[u"zip-file-name"])
@@ -260,8 +278,6 @@ def download_and_unzip_data_file(spec, job, build, pid):
download_path = spec.input[u"zip-download-path"]
if job.startswith(u"csit-"):
url = spec.environment[u"urls"][u"URL[JENKINS,CSIT]"]
- elif job.startswith(u"hc2vpp-"):
- url = spec.environment[u"urls"][u"URL[JENKINS,HC]"]
else:
raise PresentationError(f"No url defined for the job {job}.")
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 72afd35297..5acef6502d 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -104,7 +104,7 @@
urls:
URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
- URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+ URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
URL[NEXUS,DOC]: "https://docs.fd.io/csit"
DIR[NEXUS,DOC]: "report/_static/archive"
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 50cedf4fbc..96584923e8 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -44,6 +44,7 @@
urls:
URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
+ URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
URL[NEXUS,DOC]: "https://docs.fd.io/csit"
DIR[NEXUS,DOC]: "trending/_static/archive"
diff --git a/resources/tools/presentation/specification_local.yaml b/resources/tools/presentation/specification_local.yaml
index 393228c1a5..95970a2fdf 100644
--- a/resources/tools/presentation/specification_local.yaml
+++ b/resources/tools/presentation/specification_local.yaml
@@ -106,7 +106,7 @@
urls:
# Urls where the input files are downloaded from
URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
- URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+ URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
URL[NEXUS,DOC]: "https://docs.fd.io/csit"
DIR[NEXUS,DOC]: "report/_static/archive"