From 9336e2d1f5678dda74d5740418e616fe5837609e Mon Sep 17 00:00:00 2001 From: Dave Wallace Date: Thu, 17 Nov 2022 14:27:18 -0500 Subject: Enable upload of vpp & hicn verify docs to s3 7day retention bucket - Use the same bucket path as logs so that the docs can be viewed by s/s3-logs/s3-docs-7day/ in the URL after selecting the logs URL from the jenkins job page. - Also, fix error output of get_gerrit_refspec bash function. Change-Id: I73e8b7a1f310dbfb031afe9d164b114021c2cfe3 Signed-off-by: Dave Wallace --- extras/bash/sandbox_test_functions.sh | 3 ++- jjb/global-macros.yaml | 45 ++++++++++++++++++++++++++++++-- jjb/hicn/hicn-docs.yaml | 1 + jjb/scripts/hicn/docs.sh | 8 +++--- jjb/scripts/publish_docs.sh | 48 +++++++++++++++++++++++------------ jjb/scripts/terraform_s3_docs_ship.sh | 3 +-- jjb/vpp/docs.yaml | 1 + 7 files changed, 83 insertions(+), 26 deletions(-) diff --git a/extras/bash/sandbox_test_functions.sh b/extras/bash/sandbox_test_functions.sh index 90736e964..3993069e1 100644 --- a/extras/bash/sandbox_test_functions.sh +++ b/extras/bash/sandbox_test_functions.sh @@ -131,7 +131,8 @@ get_gerrit_refspec() { local query="$(ssh -p 29418 gerrit.fd.io gerrit query status:merged project:$project branch:$branch limit:1 --format=JSON --current-patch-set | tr ',' '\n' | grep refs | cut -d'"' -f4)" if [ -z "$query" ] ; then - echo "ERROR: Invalid project ($1) or branch ($2)" + echo "ERROR: Invalid argument(s): branch ($1) project ($2)" + echo "Usage: $0 " else echo "$query" fi diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index c2c24ca13..ca925ebc0 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -754,10 +754,10 @@ - publisher: name: fdio-infra-publish-docs - # macro to finish up a build. + # macro to finish up a docs build. # # Handles the following: - # - Shipping docs S3 logs repository + # - Mapping docs S3 bucket credentials for merge job docs upload # - Cleanup workspace publishers: - postbuildscript: @@ -779,6 +779,33 @@ - "**/*.jenkins-trigger" fail-build: false +- publisher: + name: fdio-infra-publish-docs-7day + # macro to finish up a verify docs build. + # + # Handles the following: + # - Mapping 7-day retention S3 bucket credentials for verify job docs upload + # - Cleanup workspace + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - fdio-infra-ship-docs-7day + mark-unstable-if-failed: true + - workspace-cleanup: + exclude: + # Do not clean up *.jenkins-trigger files for jobs that use a + # properties file as input for triggering another build. + - "**/*.jenkins-trigger" + fail-build: false + - publisher: name: fdio-infra-publish # macro to finish up a build. @@ -863,6 +890,20 @@ - shell: !include-raw: - ../global-jjb/shell/logs-clear-credentials.sh +- builder: + name: fdio-infra-ship-docs-7day + builders: + - config-file-provider: + files: + - file-id: "jenkins-s3-vpp-docs-ship" + target: $HOME/.aws/credentials + - shell: !include-raw: + - scripts/terraform_s3_docs_ship.sh + - shell: !include-raw: + - scripts/publish_docs.sh + - shell: !include-raw: + - ../global-jjb/shell/logs-clear-credentials.sh + - builder: name: fdio-infra-ship-logs builders: diff --git a/jjb/hicn/hicn-docs.yaml b/jjb/hicn/hicn-docs.yaml index 89fb9fcc7..27bdbe93e 100644 --- a/jjb/hicn/hicn-docs.yaml +++ b/jjb/hicn/hicn-docs.yaml @@ -63,6 +63,7 @@ - shell: !include-raw-escape: ../scripts/hicn/docs.sh publishers: + - fdio-infra-publish-docs-7day - fdio-infra-publish - job-template: diff --git a/jjb/scripts/hicn/docs.sh b/jjb/scripts/hicn/docs.sh index 46e835082..e927f0bbc 100644 --- a/jjb/scripts/hicn/docs.sh +++ b/jjb/scripts/hicn/docs.sh @@ -9,8 +9,6 @@ echo "---> jjb/scripts/hicn/docs.sh" bash scripts/build-packages.sh sphinx -if [[ "${JOB_NAME}" == *merge* ]]; then - mkdir -p "${SITE_DIR_ROOT}" - mv -f "${DOC_DIR}" "${SITE_DIR}" - find "${SITE_DIR}" -type f '(' -name '*.md5' -o -name '*.dot' -o -name '*.map' ')' -delete -fi +mkdir -p "${SITE_DIR_ROOT}" +mv -f "${DOC_DIR}" "${SITE_DIR}" +find "${SITE_DIR}" -type f '(' -name '*.md5' -o -name '*.dot' -o -name '*.map' ')' -delete diff --git a/jjb/scripts/publish_docs.sh b/jjb/scripts/publish_docs.sh index 891e57a7b..1cf01a4bb 100755 --- a/jjb/scripts/publish_docs.sh +++ b/jjb/scripts/publish_docs.sh @@ -17,12 +17,8 @@ echo "---> publish_docs.sh" set -exuo pipefail -if [[ "${SILO}" != "production" ]] ; then - echo "WARNING: Doc upload not supported on Jenkins '${SILO}'..." - exit 0 -fi - CDN_URL="s3-docs.fd.io" +bucket="fdio-docs-s3-cloudfront-index" if [[ ${JOB_NAME} == *merge* ]]; then case "${JOB_NAME}" in @@ -51,17 +47,37 @@ if [[ ${JOB_NAME} == *merge* ]]; then *) die "Unknown job: ${JOB_NAME}" esac +elif [[ ${JOB_NAME} == *verify* ]]; then + bucket="vpp-docs-7day-retention" + # Use the same bucket path as logs so that the docs can be viewed by + # s/s3-logs/s3-docs-7day/ in the URL after selecting the logs URL from + # the jenkins job page. + bucket_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/" + case "${JOB_NAME}" in + *"hicn-docs"*) + workspace_dir="${WORKSPACE}/build/doc/deploy-site" + ;; + *"vpp-docs"*) + CDN_URL="s3-docs-7day.fd.io" + workspace_dir="${WORKSPACE}/build-root/docs/html" + ;; + *) + die "Unknown job: ${JOB_NAME}" + esac +else + die "Unknown job: ${JOB_NAME}" +fi - export TF_VAR_workspace_dir=$workspace_dir - export TF_VAR_bucket_path=$bucket_path - export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/credentials - export AWS_DEFAULT_REGION="us-east-1" +export TF_VAR_workspace_dir="$workspace_dir" +export TF_VAR_bucket_path="$bucket_path" +export TF_VAR_bucket="$bucket" +export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/credentials +export AWS_DEFAULT_REGION="us-east-1" - echo "INFO: archiving docs to S3" - pushd .. - terraform init -no-color - terraform apply -no-color -auto-approve - popd +echo "INFO: archiving docs to S3 bucket '$bucket'" +pushd .. +terraform init -no-color +terraform apply -no-color -auto-approve +popd - echo "S3 docs: https://${CDN_URL}${bucket_path}" -fi +echo "S3 docs: https://${CDN_URL}${bucket_path}" diff --git a/jjb/scripts/terraform_s3_docs_ship.sh b/jjb/scripts/terraform_s3_docs_ship.sh index 5989e3e9a..de177d35c 100644 --- a/jjb/scripts/terraform_s3_docs_ship.sh +++ b/jjb/scripts/terraform_s3_docs_ship.sh @@ -21,7 +21,7 @@ cat >"/w/workspace/main.tf" <<'END_OF_TERRAFORM_SCRIPT' provider "aws" { region = "us-east-1" profile = "default" - s3_force_path_style = false + s3_use_path_style = false skip_credentials_validation = true skip_metadata_api_check = true skip_requesting_account_id = true @@ -65,7 +65,6 @@ variable "file_match_pattern" { variable "bucket" { description = "S3 bucket name" type = string - default = "fdio-docs-s3-cloudfront-index" } variable "bucket_path" { diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml index 25a2dde72..e61c0a688 100644 --- a/jjb/vpp/docs.yaml +++ b/jjb/vpp/docs.yaml @@ -85,6 +85,7 @@ - ../scripts/vpp/docs.sh publishers: + - fdio-infra-publish-docs-7day - fdio-infra-publish - job-template: -- cgit 1.2.3-korg