diff options
| -rw-r--r-- | jjb/csit/csit.yaml | 65 | ||||
| -rw-r--r-- | jjb/global-macros.yaml | 49 | ||||
| -rw-r--r-- | jjb/scripts/csit/docs.sh | 65 | ||||
| -rw-r--r-- | jjb/scripts/csit/report.sh | 65 | ||||
| -rw-r--r-- | jjb/scripts/csit/trending.sh | 21 | ||||
| -rw-r--r-- | jjb/scripts/publish_docs.sh | 48 | ||||
| -rw-r--r-- | jjb/scripts/publish_library_py.sh (renamed from jjb/scripts/logs_publish.sh) | 60 | ||||
| -rw-r--r-- | jjb/scripts/publish_logs.sh | 39 |
8 files changed, 200 insertions, 212 deletions
diff --git a/jjb/csit/csit.yaml b/jjb/csit/csit.yaml index a3ab6f452..45b1aa225 100644 --- a/jjb/csit/csit.yaml +++ b/jjb/csit/csit.yaml @@ -68,8 +68,6 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - maven-exec: - maven-version: 'mvn33-new' scm: - gerrit-trigger-scm: @@ -103,16 +101,6 @@ notbuilt: true builders: - - maven-target: - maven-version: 'mvn33-new' - goals: '--version' - settings: 'csit-settings' - settings-type: cfp - global-settings: 'global-settings' - global-settings-type: cfp - - provide-maven-settings: - settings-file: 'csit-settings' - global-settings-file: 'global-settings' - shell: !include-raw-escape: - ../scripts/setup_executor_env.sh @@ -121,6 +109,7 @@ - ../scripts/csit/docs.sh publishers: + - fdio-infra-publish-docs - fdio-infra-publish - job-template: @@ -129,10 +118,7 @@ project-type: freestyle node: 'builder-{os}-prod-{executor-arch}' concurrent: false - allow-empty: 'true' - fingerprint: false - only-if-success: true - default-excludes: false + latest-only: false build-discarder: daysToKeep: '{build-days-to-keep}' @@ -145,14 +131,13 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - maven-exec: - maven-version: 'mvn33-new' + - gerrit-refspec-parameter scm: - gerrit-trigger-scm: credentials-id: 'jenkins-gerrit-credentials' refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'default' + choosing-strategy: 'gerrit' wrappers: - fdio-infra-wrappers-non-activity-timeout: @@ -177,16 +162,6 @@ notbuilt: true builders: - - maven-target: - maven-version: 'mvn33-new' - goals: '--version' - settings: 'csit-settings' - settings-type: cfp - global-settings: 'global-settings' - global-settings-type: cfp - - provide-maven-settings: - settings-file: 'csit-settings' - global-settings-file: 'global-settings' - shell: !include-raw-escape: - ../scripts/setup_executor_env.sh @@ -195,6 +170,7 @@ - ../scripts/csit/report.sh publishers: + - fdio-infra-publish-docs - fdio-infra-publish - job-template: @@ -215,8 +191,6 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - maven-exec: - maven-version: 'mvn33-new' scm: - gerrit-trigger-scm: @@ -232,22 +206,12 @@ - timed: '0 2,14 * * *' builders: - - maven-target: - maven-version: 'mvn33-new' - goals: '--version' - settings: 'csit-settings' - settings-type: cfp - global-settings: 'global-settings' - global-settings-type: cfp - - provide-maven-settings: - settings-file: 'csit-settings' - global-settings-file: 'global-settings' - shell: !include-raw-escape: - ../scripts/setup_executor_env.sh - shell: !include-raw-escape: - - ../scripts/csit/cpta.sh + - ../scripts/csit/trending.sh publishers: - email-ext: @@ -295,6 +259,7 @@ always: true failure: false + - fdio-infra-publish-docs - fdio-infra-publish - job-template: @@ -315,8 +280,6 @@ project: '{project}' - gerrit-parameter: branch: 'rls2009_lts' - - maven-exec: - maven-version: 'mvn33-new' scm: - gerrit-trigger-scm: @@ -332,22 +295,12 @@ - timed: '0 0 * * 1' builders: - - maven-target: - maven-version: 'mvn33-new' - goals: '--version' - settings: 'csit-settings' - settings-type: cfp - global-settings: 'global-settings' - global-settings-type: cfp - - provide-maven-settings: - settings-file: 'csit-settings' - global-settings-file: 'global-settings' - shell: !include-raw-escape: - ../scripts/setup_executor_env.sh - shell: !include-raw-escape: - - ../scripts/csit/cpta.sh + - ../scripts/csit/trending.sh publishers: - email-ext: @@ -395,8 +348,10 @@ always: true failure: false + - fdio-infra-publish-docs - fdio-infra-publish + - project: name: csit-info project-name: csit diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index eed5c0bed..8059ac550 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -648,6 +648,33 @@ - recipients - publisher: + name: fdio-infra-publish-docs + # macro to finish up a build. + # + # Handles the following: + # - Shipping docs S3 logs repository + # - Cleanup workspace + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - fdio-infra-ship-docs + mark-unstable-if-failed: true + - workspace-cleanup: + exclude: + # Do not clean up *.jenkins-trigger files for jobs that use a + # properties file as input for triggering another build. + - "**/*.jenkins-trigger" + fail-build: false + +- publisher: name: fdio-infra-publish # macro to finish up a build. # @@ -725,6 +752,24 @@ num-to-keep: '{build-num-to-keep}' - builder: + name: fdio-infra-ship-docs + builders: + - config-file-provider: + files: + - file-id: "jenkins-log-archives-settings" + variable: "SETTINGS_FILE" + - config-file-provider: + files: + - file-id: "jenkins-s3-docs-ship" + target: $HOME/.aws/credentials + - shell: !include-raw: + - scripts/publish_library_py.sh + - shell: !include-raw: + - scripts/publish_docs.sh + - shell: !include-raw: + - ../global-jjb/shell/logs-clear-credentials.sh + +- builder: name: fdio-infra-ship-backup-logs builders: - config-file-provider: @@ -732,7 +777,9 @@ - file-id: "jenkins-s3-log-ship" variable: $HOME/.aws/credentials - shell: !include-raw: - - scripts/logs_publish.sh + - scripts/publish_library_py.sh + - shell: !include-raw: + - scripts/publish_logs.sh - shell: !include-raw: - ../global-jjb/shell/logs-clear-credentials.sh diff --git a/jjb/scripts/csit/docs.sh b/jjb/scripts/csit/docs.sh index 55048f2fd..9ecc78cd1 100644 --- a/jjb/scripts/csit/docs.sh +++ b/jjb/scripts/csit/docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,64 +15,7 @@ echo "---> jjb/scripts/csit/docs.sh" -set -xe -o pipefail +set -exuo pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit -[ "$DOC_DIR" ] || DOC_DIR=resources/tools/doc_gen -[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/doc -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" - -cd ${DOC_DIR} -chmod +x ./run_doc.sh -./run_doc.sh ${GERRIT_BRANCH} - -retval=$? -if [ ${retval} -ne "0" ]; then - echo "Documentation generation failed!" -exit ${retval} -fi - -if [[ ${JOB_NAME} == *merge* ]]; then - - cd ${WORKSPACE} - - mkdir -p ${RESOURCES_DIR} - mv -f ${BUILD_DIR}/* ${RESOURCES_DIR} - cd ${SITE_DIR} - - cat > pom.xml << EOF - <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>io.fd.csit</groupId> - <artifactId>docs</artifactId> - <version>1.0.0</version> - <packaging>pom</packaging> - <properties> - <generateReports>false</generateReports> - </properties> - <build> - <extensions> - <extension> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-webdav-jackrabbit</artifactId> - <version>2.10</version> - </extension> - </extensions> - </build> - <distributionManagement> - <site> - <id>fdio-site</id> - <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url> - </site> - </distributionManagement> - </project> -EOF - - ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C - - cd - - -fi +csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" +source "${csit_entry_dir}/bootstrap_docs.sh" diff --git a/jjb/scripts/csit/report.sh b/jjb/scripts/csit/report.sh index 16018fdc9..0bd2e81bd 100644 --- a/jjb/scripts/csit/report.sh +++ b/jjb/scripts/csit/report.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,64 +15,7 @@ echo "---> jjb/scripts/csit/report.sh" -set -xe -o pipefail +set -exuo pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit -[ "$DOC_DIR" ] || DOC_DIR=resources/tools/presentation -[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/report -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" - -cd ${DOC_DIR} -chmod +x ./run_report.sh -./run_report.sh ${GERRIT_BRANCH} - -retval=$? -if [ ${retval} -ne "0" ]; then - echo "Report generation failed!" -exit ${retval} -fi - -if [[ ${JOB_NAME} == *merge* ]]; then - - cd ${WORKSPACE} - - mkdir -p ${RESOURCES_DIR} - mv -f ${BUILD_DIR}/* ${RESOURCES_DIR} - cd ${SITE_DIR} - - cat > pom.xml << EOF - <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>io.fd.csit</groupId> - <artifactId>docs</artifactId> - <version>1.0.0</version> - <packaging>pom</packaging> - <properties> - <generateReports>false</generateReports> - </properties> - <build> - <extensions> - <extension> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-webdav-jackrabbit</artifactId> - <version>2.10</version> - </extension> - </extensions> - </build> - <distributionManagement> - <site> - <id>fdio-site</id> - <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url> - </site> - </distributionManagement> - </project> -EOF - - ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C - - cd - - -fi +csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" +source "${csit_entry_dir}/bootstrap_report.sh" diff --git a/jjb/scripts/csit/trending.sh b/jjb/scripts/csit/trending.sh new file mode 100644 index 000000000..715d15bf1 --- /dev/null +++ b/jjb/scripts/csit/trending.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/csit/trending.sh" + +set -exuo pipefail + +csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" +source "${csit_entry_dir}/bootstrap_trending.sh" diff --git a/jjb/scripts/publish_docs.sh b/jjb/scripts/publish_docs.sh new file mode 100644 index 000000000..d6b843cbe --- /dev/null +++ b/jjb/scripts/publish_docs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> publish_docs.sh" + +set -exuo pipefail + +S3_BUCKET="fdio-docs-s3-cloudfront-index" +CDN_URL="s3-docs.fd.io" +PYTHON_SCRIPT="/w/workspace/publish_library.py" + +if [[ ${JOB_NAME} == *merge* ]]; then + case "${JOB_NAME}" in + *"csit-trending"*) + SITE_DIR="${WORKSPACE}/resources/tools/trending/_build" + s3_path="csit/${GERRIT_BRANCH}/trending" + ;; + *"csit-report"*) + SITE_DIR="${WORKSPACE}/resources/tools/presentation/_build" + s3_path="csit/${GERRIT_BRANCH}/report" + ;; + *"csit-docs"*) + SITE_DIR="${WORKSPACE}/resources/tools/doc_gen/_build" + s3_path="csit/${GERRIT_BRANCH}/docs" + ;; + *) + die "Unknown job: ${JOB_NAME}" + esac + + echo "INFO: S3 path $s3_path" + + echo "INFO: archiving docs to S3" + python3 $PYTHON_SCRIPT deploy_docs "$S3_BUCKET" "$s3_path" "$SITE_DIR" + + echo "S3 docs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>" +fi diff --git a/jjb/scripts/logs_publish.sh b/jjb/scripts/publish_library_py.sh index da3593c66..f8430ff0b 100644 --- a/jjb/scripts/logs_publish.sh +++ b/jjb/scripts/publish_library_py.sh @@ -13,24 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo "---> logs_publish.sh" +echo "---> publish_library_py.sh" -CDN_URL="logs.nginx.service.consul" -export AWS_ENDPOINT_URL="http://storage.service.consul:9000" +set -exuo pipefail -# FIXME: s3 config (until migrated to config provider, then pwd will be reset) -mkdir -p ${HOME}/.aws -echo "[default] -aws_access_key_id = storage -aws_secret_access_key = Storage1234" >> "$HOME/.aws/credentials" - -PYTHON_SCRIPT="/w/workspace/test-logs/logs_publish.py" - -# This script uploads the artifacts to a backup upload location -if [ -f "$PYTHON_SCRIPT" ]; then - echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done" - exit 0 -fi +PYTHON_SCRIPT="/w/workspace/publish_library.py" pip3 install boto3 mkdir -p $(dirname "$PYTHON_SCRIPT") @@ -38,7 +25,7 @@ mkdir -p $(dirname "$PYTHON_SCRIPT") cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT' #!/usr/bin/python3 -"""Storage utilities library.""" +"""S3 publish library.""" import gzip import logging @@ -130,7 +117,7 @@ def upload(s3_resource, s3_bucket, src_fpath, s3_path): if not mime: mime = u"application/octet-stream" - if s3_bucket not in u"docs.fd.io": + if u"logs" in s3_bucket: if mime in COMPRESS_MIME and encoding != u"gzip": compress(src_fpath) src_fpath = src_fpath + u".gz" @@ -178,6 +165,27 @@ def upload_recursive(s3_resource, s3_bucket, src_fpath, s3_path): ) +def deploy_docs(s3_bucket, s3_path, docs_dir): + """Ship docs dir content to S3 bucket. Requires the s3 bucket to exist. + + :param s3_bucket: Name of S3 bucket. Eg: lf-project-date + :param s3_path: Path on S3 bucket to place the docs. Eg: + csit/${GERRIT_BRANCH}/report + :param docs_dir: Directory in which to recursively upload content. + :type s3_bucket: Object + :type s3_path: str + :type docs_dir: str + """ + s3_resource = boto3.resource(u"s3") + + upload_recursive( + s3_resource=s3_resource, + s3_bucket=s3_bucket, + src_fpath=docs_dir, + s3_path=s3_path + ) + + def deploy_s3(s3_bucket, s3_path, build_url, workspace): """Add logs and archives to temp directory to be shipped to S3 bucket. Fetches logs and system information and pushes them and archives to S3 @@ -273,19 +281,3 @@ if __name__ == u"__main__": globals()[sys.argv[1]](*sys.argv[2:]) END_OF_PYTHON_SCRIPT - -# The 'deploy_s3' command below expects the archives -# directory to exist. Normally lf-infra-sysstat or similar would -# create it and add content, but to make sure this script is -# self-contained, we ensure it exists here. -mkdir -p "$WORKSPACE/archives" - -s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/" -echo "INFO: S3 path $s3_path" - -echo "INFO: archiving backup logs to S3" -# shellcheck disable=SC2086 -python3 $PYTHON_SCRIPT deploy_s3 "logs.fd.io" "$s3_path" \ - "$BUILD_URL" "$WORKSPACE" - -echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>" diff --git a/jjb/scripts/publish_logs.sh b/jjb/scripts/publish_logs.sh new file mode 100644 index 000000000..bc1e24c15 --- /dev/null +++ b/jjb/scripts/publish_logs.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> publish_logs.sh" + +S3_BUCKET="logs.fd.io" +CDN_URL="logs.nginx.service.consul" +export AWS_ENDPOINT_URL="http://storage.service.consul:9000" +PYTHON_SCRIPT="/w/workspace/publish_library.py" + +# FIXME: s3 config (until migrated to config provider, then pwd will be reset) +mkdir -p ${HOME}/.aws +echo "[default] +aws_access_key_id = storage +aws_secret_access_key = Storage1234" > "$HOME/.aws/credentials" + +mkdir -p "$WORKSPACE/archives" + +s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/" + +echo "INFO: S3 path $s3_path" + +echo "INFO: archiving backup logs to S3" +python3 $PYTHON_SCRIPT deploy_s3 "$S3_BUCKET" "$s3_path" \ + "$BUILD_URL" "$WORKSPACE" + +echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>" |