diff options
author | Tibor Frank <tifrank@cisco.com> | 2021-03-04 09:52:20 +0100 |
---|---|---|
committer | Tibor Frank <tifrank@cisco.com> | 2021-03-05 05:38:27 +0000 |
commit | df0263faee6171805e8bc579a4e0a786e63c3a8d (patch) | |
tree | d5ba1f23212cd6d33905bbc81dc26c43f9e6097a /resources/tools/presentation | |
parent | bb7e8747d67176367fabadac85bc4e2571a47c93 (diff) |
PAL: Add s3_storage as the data source
Priorities for download:
1. s3_storage
2. logs.fd.io
3. docs.fd.io release
4. docs.fd.io master
5. jenkins
Change-Id: I7114715c3ec9a33acc8edf3c5c473f4eaffa639d
Signed-off-by: Tibor Frank <tifrank@cisco.com>
Diffstat (limited to 'resources/tools/presentation')
5 files changed, 28 insertions, 8 deletions
diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst index 9569433760..38ab6d9bcb 100644 --- a/resources/tools/presentation/doc/pal_lld.rst +++ b/resources/tools/presentation/doc/pal_lld.rst @@ -187,7 +187,10 @@ The structure of the section "Environment" is as follows (example): urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" - URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + URL[NEXUS,DOC]: "https://docs.fd.io/csit" + DIR[NEXUS,DOC]: "report/_static/archive" make-dirs: # List the directories which are created while preparing the environment. diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index 8b941f2f94..47249b1323 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -200,11 +200,10 @@ def download_and_unzip_data_file(spec, job, build, pid): :rtype: bool """ - # Try to download .gz from logs.fd.io - + # Try to download .gz from s3_storage file_name = spec.input[u"file-name"] url = u"{0}/{1}".format( - spec.environment[u'urls'][u'URL[NEXUS,LOG]'], + spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'], spec.input[u'download-path'].format( job=job, build=build[u'build'], filename=file_name ) @@ -220,6 +219,25 @@ def download_and_unzip_data_file(spec, job, build, pid): success, downloaded_name = _download_file(url, new_name, arch=arch) if not success: + # Try to download .gz from logs.fd.io + file_name = spec.input[u"file-name"] + url = u"{0}/{1}".format( + spec.environment[u'urls'][u'URL[NEXUS,LOG]'], + spec.input[u'download-path'].format( + job=job, build=build[u'build'], filename=file_name + ) + ) + new_name = join( + spec.environment[u"paths"][u"DIR[WORKING,DATA]"], + f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}" + ) + + logging.info(f"Trying to download {url}") + + arch = bool(spec.configuration.get(u"archive-inputs", True)) + success, downloaded_name = _download_file(url, new_name, arch=arch) + + if not success: # Try to download .gz or .zip from docs.fd.io file_name = (spec.input[u"file-name"], spec.input[u"zip-file-name"]) @@ -260,8 +278,6 @@ def download_and_unzip_data_file(spec, job, build, pid): download_path = spec.input[u"zip-download-path"] if job.startswith(u"csit-"): url = spec.environment[u"urls"][u"URL[JENKINS,CSIT]"] - elif job.startswith(u"hc2vpp-"): - url = spec.environment[u"urls"][u"URL[JENKINS,HC]"] else: raise PresentationError(f"No url defined for the job {job}.") diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 72afd35297..5acef6502d 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -104,7 +104,7 @@ urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" - URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" URL[NEXUS,DOC]: "https://docs.fd.io/csit" DIR[NEXUS,DOC]: "report/_static/archive" diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 50cedf4fbc..96584923e8 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -44,6 +44,7 @@ urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" URL[NEXUS,DOC]: "https://docs.fd.io/csit" DIR[NEXUS,DOC]: "trending/_static/archive" diff --git a/resources/tools/presentation/specification_local.yaml b/resources/tools/presentation/specification_local.yaml index 393228c1a5..95970a2fdf 100644 --- a/resources/tools/presentation/specification_local.yaml +++ b/resources/tools/presentation/specification_local.yaml @@ -106,7 +106,7 @@ urls: # Urls where the input files are downloaded from URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" - URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" URL[NEXUS,DOC]: "https://docs.fd.io/csit" DIR[NEXUS,DOC]: "report/_static/archive" |