diff options
Diffstat (limited to 'jjb/scripts')
| -rw-r--r-- | jjb/scripts/csit/docs.sh | 65 | ||||
| -rw-r--r-- | jjb/scripts/csit/report.sh | 65 | ||||
| -rw-r--r-- | jjb/scripts/csit/trending.sh | 21 | ||||
| -rw-r--r-- | jjb/scripts/publish_docs.sh | 48 | ||||
| -rw-r--r-- | jjb/scripts/publish_library_py.sh (renamed from jjb/scripts/logs_publish.sh) | 60 | ||||
| -rw-r--r-- | jjb/scripts/publish_logs.sh | 39 |
6 files changed, 142 insertions, 156 deletions
diff --git a/jjb/scripts/csit/docs.sh b/jjb/scripts/csit/docs.sh index 55048f2fd..9ecc78cd1 100644 --- a/jjb/scripts/csit/docs.sh +++ b/jjb/scripts/csit/docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,64 +15,7 @@ echo "---> jjb/scripts/csit/docs.sh" -set -xe -o pipefail +set -exuo pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit -[ "$DOC_DIR" ] || DOC_DIR=resources/tools/doc_gen -[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/doc -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" - -cd ${DOC_DIR} -chmod +x ./run_doc.sh -./run_doc.sh ${GERRIT_BRANCH} - -retval=$? -if [ ${retval} -ne "0" ]; then - echo "Documentation generation failed!" -exit ${retval} -fi - -if [[ ${JOB_NAME} == *merge* ]]; then - - cd ${WORKSPACE} - - mkdir -p ${RESOURCES_DIR} - mv -f ${BUILD_DIR}/* ${RESOURCES_DIR} - cd ${SITE_DIR} - - cat > pom.xml << EOF - <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>io.fd.csit</groupId> - <artifactId>docs</artifactId> - <version>1.0.0</version> - <packaging>pom</packaging> - <properties> - <generateReports>false</generateReports> - </properties> - <build> - <extensions> - <extension> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-webdav-jackrabbit</artifactId> - <version>2.10</version> - </extension> - </extensions> - </build> - <distributionManagement> - <site> - <id>fdio-site</id> - <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url> - </site> - </distributionManagement> - </project> -EOF - - ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C - - cd - - -fi +csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" +source "${csit_entry_dir}/bootstrap_docs.sh" diff --git a/jjb/scripts/csit/report.sh b/jjb/scripts/csit/report.sh index 16018fdc9..0bd2e81bd 100644 --- a/jjb/scripts/csit/report.sh +++ b/jjb/scripts/csit/report.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,64 +15,7 @@ echo "---> jjb/scripts/csit/report.sh" -set -xe -o pipefail +set -exuo pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit -[ "$DOC_DIR" ] || DOC_DIR=resources/tools/presentation -[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/report -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" - -cd ${DOC_DIR} -chmod +x ./run_report.sh -./run_report.sh ${GERRIT_BRANCH} - -retval=$? -if [ ${retval} -ne "0" ]; then - echo "Report generation failed!" -exit ${retval} -fi - -if [[ ${JOB_NAME} == *merge* ]]; then - - cd ${WORKSPACE} - - mkdir -p ${RESOURCES_DIR} - mv -f ${BUILD_DIR}/* ${RESOURCES_DIR} - cd ${SITE_DIR} - - cat > pom.xml << EOF - <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>io.fd.csit</groupId> - <artifactId>docs</artifactId> - <version>1.0.0</version> - <packaging>pom</packaging> - <properties> - <generateReports>false</generateReports> - </properties> - <build> - <extensions> - <extension> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-webdav-jackrabbit</artifactId> - <version>2.10</version> - </extension> - </extensions> - </build> - <distributionManagement> - <site> - <id>fdio-site</id> - <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url> - </site> - </distributionManagement> - </project> -EOF - - ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C - - cd - - -fi +csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" +source "${csit_entry_dir}/bootstrap_report.sh" diff --git a/jjb/scripts/csit/trending.sh b/jjb/scripts/csit/trending.sh new file mode 100644 index 000000000..715d15bf1 --- /dev/null +++ b/jjb/scripts/csit/trending.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/csit/trending.sh" + +set -exuo pipefail + +csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" +source "${csit_entry_dir}/bootstrap_trending.sh" diff --git a/jjb/scripts/publish_docs.sh b/jjb/scripts/publish_docs.sh new file mode 100644 index 000000000..d6b843cbe --- /dev/null +++ b/jjb/scripts/publish_docs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> publish_docs.sh" + +set -exuo pipefail + +S3_BUCKET="fdio-docs-s3-cloudfront-index" +CDN_URL="s3-docs.fd.io" +PYTHON_SCRIPT="/w/workspace/publish_library.py" + +if [[ ${JOB_NAME} == *merge* ]]; then + case "${JOB_NAME}" in + *"csit-trending"*) + SITE_DIR="${WORKSPACE}/resources/tools/trending/_build" + s3_path="csit/${GERRIT_BRANCH}/trending" + ;; + *"csit-report"*) + SITE_DIR="${WORKSPACE}/resources/tools/presentation/_build" + s3_path="csit/${GERRIT_BRANCH}/report" + ;; + *"csit-docs"*) + SITE_DIR="${WORKSPACE}/resources/tools/doc_gen/_build" + s3_path="csit/${GERRIT_BRANCH}/docs" + ;; + *) + die "Unknown job: ${JOB_NAME}" + esac + + echo "INFO: S3 path $s3_path" + + echo "INFO: archiving docs to S3" + python3 $PYTHON_SCRIPT deploy_docs "$S3_BUCKET" "$s3_path" "$SITE_DIR" + + echo "S3 docs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>" +fi diff --git a/jjb/scripts/logs_publish.sh b/jjb/scripts/publish_library_py.sh index da3593c66..f8430ff0b 100644 --- a/jjb/scripts/logs_publish.sh +++ b/jjb/scripts/publish_library_py.sh @@ -13,24 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo "---> logs_publish.sh" +echo "---> publish_library_py.sh" -CDN_URL="logs.nginx.service.consul" -export AWS_ENDPOINT_URL="http://storage.service.consul:9000" +set -exuo pipefail -# FIXME: s3 config (until migrated to config provider, then pwd will be reset) -mkdir -p ${HOME}/.aws -echo "[default] -aws_access_key_id = storage -aws_secret_access_key = Storage1234" >> "$HOME/.aws/credentials" - -PYTHON_SCRIPT="/w/workspace/test-logs/logs_publish.py" - -# This script uploads the artifacts to a backup upload location -if [ -f "$PYTHON_SCRIPT" ]; then - echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done" - exit 0 -fi +PYTHON_SCRIPT="/w/workspace/publish_library.py" pip3 install boto3 mkdir -p $(dirname "$PYTHON_SCRIPT") @@ -38,7 +25,7 @@ mkdir -p $(dirname "$PYTHON_SCRIPT") cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT' #!/usr/bin/python3 -"""Storage utilities library.""" +"""S3 publish library.""" import gzip import logging @@ -130,7 +117,7 @@ def upload(s3_resource, s3_bucket, src_fpath, s3_path): if not mime: mime = u"application/octet-stream" - if s3_bucket not in u"docs.fd.io": + if u"logs" in s3_bucket: if mime in COMPRESS_MIME and encoding != u"gzip": compress(src_fpath) src_fpath = src_fpath + u".gz" @@ -178,6 +165,27 @@ def upload_recursive(s3_resource, s3_bucket, src_fpath, s3_path): ) +def deploy_docs(s3_bucket, s3_path, docs_dir): + """Ship docs dir content to S3 bucket. Requires the s3 bucket to exist. + + :param s3_bucket: Name of S3 bucket. Eg: lf-project-date + :param s3_path: Path on S3 bucket to place the docs. Eg: + csit/${GERRIT_BRANCH}/report + :param docs_dir: Directory in which to recursively upload content. + :type s3_bucket: Object + :type s3_path: str + :type docs_dir: str + """ + s3_resource = boto3.resource(u"s3") + + upload_recursive( + s3_resource=s3_resource, + s3_bucket=s3_bucket, + src_fpath=docs_dir, + s3_path=s3_path + ) + + def deploy_s3(s3_bucket, s3_path, build_url, workspace): """Add logs and archives to temp directory to be shipped to S3 bucket. Fetches logs and system information and pushes them and archives to S3 @@ -273,19 +281,3 @@ if __name__ == u"__main__": globals()[sys.argv[1]](*sys.argv[2:]) END_OF_PYTHON_SCRIPT - -# The 'deploy_s3' command below expects the archives -# directory to exist. Normally lf-infra-sysstat or similar would -# create it and add content, but to make sure this script is -# self-contained, we ensure it exists here. -mkdir -p "$WORKSPACE/archives" - -s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/" -echo "INFO: S3 path $s3_path" - -echo "INFO: archiving backup logs to S3" -# shellcheck disable=SC2086 -python3 $PYTHON_SCRIPT deploy_s3 "logs.fd.io" "$s3_path" \ - "$BUILD_URL" "$WORKSPACE" - -echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>" diff --git a/jjb/scripts/publish_logs.sh b/jjb/scripts/publish_logs.sh new file mode 100644 index 000000000..bc1e24c15 --- /dev/null +++ b/jjb/scripts/publish_logs.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> publish_logs.sh" + +S3_BUCKET="logs.fd.io" +CDN_URL="logs.nginx.service.consul" +export AWS_ENDPOINT_URL="http://storage.service.consul:9000" +PYTHON_SCRIPT="/w/workspace/publish_library.py" + +# FIXME: s3 config (until migrated to config provider, then pwd will be reset) +mkdir -p ${HOME}/.aws +echo "[default] +aws_access_key_id = storage +aws_secret_access_key = Storage1234" > "$HOME/.aws/credentials" + +mkdir -p "$WORKSPACE/archives" + +s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/" + +echo "INFO: S3 path $s3_path" + +echo "INFO: archiving backup logs to S3" +python3 $PYTHON_SCRIPT deploy_s3 "$S3_BUCKET" "$s3_path" \ + "$BUILD_URL" "$WORKSPACE" + +echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>" |