diff options
author | Vanessa Rene Valderrama <vvalderrama@linuxfoundation.org> | 2021-04-12 16:03:55 +0000 |
---|---|---|
committer | Gerrit Code Review <gerrit@fd.io> | 2021-04-12 16:03:55 +0000 |
commit | 20320eefc4b796dcfb5d24de1244ab724b8eb3be (patch) | |
tree | bd8600ec23d422e69abebd03100fd330a9677484 /jjb | |
parent | 563cca7716264d422882ae5fd3e81d48959f33d1 (diff) | |
parent | 221efe6a33aa75e8359f46fdb016500bbcbd3fa4 (diff) |
Merge "Use fdio-infra-publish for nomad executor based jobs"
Diffstat (limited to 'jjb')
-rw-r--r-- | jjb/ci-management/packagecloud-repo-cleanup.yaml | 12 | ||||
-rw-r--r-- | jjb/cicn/cicn.yaml | 8 | ||||
-rw-r--r-- | jjb/csit/csit-perf.yaml | 30 | ||||
-rw-r--r-- | jjb/csit/csit-tox.yaml | 3 | ||||
-rw-r--r-- | jjb/csit/csit-vpp-device.yaml | 9 | ||||
-rw-r--r-- | jjb/csit/csit.yaml | 12 | ||||
-rw-r--r-- | jjb/global-macros.yaml | 2 | ||||
-rw-r--r-- | jjb/hicn/hicn-docs.yaml | 8 | ||||
-rw-r--r-- | jjb/hicn/hicn.yaml | 12 | ||||
-rwxr-xr-x | jjb/scripts/post_build_deploy_archives.sh | 30 | ||||
-rwxr-xr-x | jjb/scripts/post_build_executor_info.sh | 39 | ||||
-rw-r--r-- | jjb/vpp/docs.yaml | 18 | ||||
-rw-r--r-- | jjb/vpp/vpp.yaml | 12 |
13 files changed, 107 insertions, 88 deletions
diff --git a/jjb/ci-management/packagecloud-repo-cleanup.yaml b/jjb/ci-management/packagecloud-repo-cleanup.yaml index 7b8dae452..517c2cc7d 100644 --- a/jjb/ci-management/packagecloud-repo-cleanup.yaml +++ b/jjb/ci-management/packagecloud-repo-cleanup.yaml @@ -85,12 +85,14 @@ target: '/root/packagecloud_api' - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: + !include-raw-escape: - package-list.sh - packagecloud-repo-cleanup.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish - job-template: name: 'packagecloud-repo-cleanup-hicn-{os}-{executor-arch}' @@ -148,9 +150,11 @@ target: '/root/packagecloud_api' - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: + !include-raw-escape: - package-list.sh - packagecloud-repo-cleanup.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish diff --git a/jjb/cicn/cicn.yaml b/jjb/cicn/cicn.yaml index ff296d130..24b1c32d2 100644 --- a/jjb/cicn/cicn.yaml +++ b/jjb/cicn/cicn.yaml @@ -142,6 +142,8 @@ global-settings: "global-settings" global-settings-type: cfp - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-cicn-libparc-build.sh - provide-maven-settings: settings-file: "cicn-settings" @@ -193,6 +195,8 @@ builders: - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-cicn-libparc-build.sh publishers: @@ -261,6 +265,8 @@ global-settings: "global-settings" global-settings-type: cfp - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-cicn-viper-build.sh - provide-maven-settings: settings-file: "cicn-settings" @@ -312,6 +318,8 @@ builders: - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-cicn-viper-build.sh publishers: diff --git a/jjb/csit/csit-perf.yaml b/jjb/csit/csit-perf.yaml index 1c44e8940..d8e8418f1 100644 --- a/jjb/csit/csit-perf.yaml +++ b/jjb/csit/csit-perf.yaml @@ -442,8 +442,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-perf-mrr-daily-master{node-arch}' @@ -520,8 +519,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-perf-mrr-weekly-{stream}-{node-arch}' @@ -598,8 +596,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}' @@ -671,8 +668,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-dpdk-perf-verify-{stream}-{node-arch}' @@ -772,8 +768,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-dpdk-perf-mrr-weekly-master{node-arch}' @@ -850,8 +845,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-perf-report-iterative-{stream_report}-{node-arch}' @@ -952,8 +946,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-dpdk-perf-report-iterative-{stream_report}-{node-arch}' @@ -1054,8 +1047,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-perf-report-coverage-{stream_report}-{node-arch}' @@ -1156,8 +1148,7 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-dpdk-perf-report-coverage-{stream_report}-{node-arch}' @@ -1258,5 +1249,4 @@ other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish diff --git a/jjb/csit/csit-tox.yaml b/jjb/csit/csit-tox.yaml index f5f7b2b1f..cd1b85592 100644 --- a/jjb/csit/csit-tox.yaml +++ b/jjb/csit/csit-tox.yaml @@ -120,5 +120,4 @@ max: 250 unstable: 250 pattern: 'pylint.log' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish diff --git a/jjb/csit/csit-vpp-device.yaml b/jjb/csit/csit-vpp-device.yaml index fae559dc4..99c16598a 100644 --- a/jjb/csit/csit-vpp-device.yaml +++ b/jjb/csit/csit-vpp-device.yaml @@ -130,8 +130,7 @@ output-path: 'archives' other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-device-{stream}-{os}-{executor-arch}-{node-arch}-semiweekly' @@ -188,8 +187,7 @@ output-path: 'archives' other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-vpp-device-{stream}-{os}-{executor-arch}-{node-arch}-weekly' @@ -248,5 +246,4 @@ output-path: 'archives' other-files: - '*.*' - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish diff --git a/jjb/csit/csit.yaml b/jjb/csit/csit.yaml index 75da7df52..ea2d55045 100644 --- a/jjb/csit/csit.yaml +++ b/jjb/csit/csit.yaml @@ -125,8 +125,7 @@ - ../scripts/csit/docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-report-merge-{stream}-{os}-{executor-arch}' @@ -204,8 +203,7 @@ - ../scripts/csit/report.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-trending-daily-master-{os}-{executor-arch}' @@ -305,8 +303,7 @@ always: true failure: false - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - job-template: name: 'csit-trending-weekly-2009-lts-{os}-{executor-arch}' @@ -406,8 +403,7 @@ always: true failure: false - - fdio-infra-shiplogs: - maven-version: 'mvn33-new' + - fdio-infra-publish - project: name: csit-info diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 1146607f1..42ed90782 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -673,6 +673,7 @@ - SUCCESS - UNSTABLE build-steps: + - shell: !include-raw: scripts/post_build_executor_info.sh - shell: !include-raw: scripts/post_build_deploy_archives.sh - shell: !include-raw: scripts/backup_upload_archives.sh - fdio-infra-ship-logs @@ -707,7 +708,6 @@ - UNSTABLE build-steps: - shell: !include-raw: include-raw-deploy-archives.sh - - shell: !include-raw: scripts/backup_upload_archives.sh - maven-target: maven-version: '{maven-version}' pom: '.archives/deploy-archives.xml' diff --git a/jjb/hicn/hicn-docs.yaml b/jjb/hicn/hicn-docs.yaml index 24810be0d..5d748edc0 100644 --- a/jjb/hicn/hicn-docs.yaml +++ b/jjb/hicn/hicn-docs.yaml @@ -86,11 +86,11 @@ - provide-maven-settings: settings-file: "hicn-settings" global-settings-file: "global-settings" + - shell: !include-raw-escape: ../scripts/setup_executor_env.sh - shell: !include-raw-escape: include-raw-hicn-docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: false max-failed-builds: 1 @@ -176,11 +176,11 @@ - provide-maven-settings: settings-file: "hicn-settings" global-settings-file: "global-settings" + - shell: !include-raw-escape: ../scripts/setup_executor_env.sh - shell: !include-raw-escape: include-raw-hicn-docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: false max-failed-builds: 1 diff --git a/jjb/hicn/hicn.yaml b/jjb/hicn/hicn.yaml index 15f66acac..c3bc9107e 100644 --- a/jjb/hicn/hicn.yaml +++ b/jjb/hicn/hicn.yaml @@ -286,7 +286,9 @@ builders: - shell: !include-raw-escape: - include-raw-hicn-checkstyle.sh + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: + - include-raw-hicn-checkstyle.sh publishers: - fdio-infra-publish @@ -339,6 +341,8 @@ builders: - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-hicn-build.sh publishers: @@ -410,6 +414,8 @@ global-settings: "global-settings" global-settings-type: cfp - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-hicn-checkstyle.sh - shell: !include-raw-escape: - include-raw-hicn-build.sh @@ -495,6 +501,8 @@ builders: - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-hicn-extras-build.sh publishers: @@ -574,6 +582,8 @@ global-settings: "global-settings" global-settings-type: cfp - shell: !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: !include-raw-escape: - include-raw-hicn-extras-build.sh - provide-maven-settings: settings-file: "hicn-settings" diff --git a/jjb/scripts/post_build_deploy_archives.sh b/jjb/scripts/post_build_deploy_archives.sh index b7e7cec45..79fa05a10 100755 --- a/jjb/scripts/post_build_deploy_archives.sh +++ b/jjb/scripts/post_build_deploy_archives.sh @@ -19,20 +19,6 @@ set +e # Do not affect the build result if some part of archiving fails. WS_ARCHIVES_DIR="$WORKSPACE/archives" BUILD_ENV_LOG="$WS_ARCHIVES_DIR/_build-enviroment-variables.log" -# Output executor runtime attributes [again] in case the job fails prior to -# running setup_executor_env.sh -long_line="************************************************************************" -OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') -OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') -OS_ARCH=$(uname -m) -echo "$long_line" -echo "Executor Runtime Attributes:" -echo "OS: $OS_ID-$OS_VERSION_ID" -echo "Arch: $OS_ARCH" -echo "Nomad Hostname: $(grep search /etc/resolv.conf | cut -d' ' -f2 | head -1)" -echo "Container ID: $(hostname)" -echo "$long_line" - # Generate gdb-command script to output vpp stack traceback from core files. gdb_cmdfile="/tmp/gdb-commands" cat >$gdb_cmdfile <<'__END__' @@ -83,21 +69,17 @@ generate_vpp_stacktrace_and_delete_core() { fi } -# Delete existing archives dir to ensure current artifact upload -rm -rf "$WS_ARCHIVES_DIR" mkdir -p "$WS_ARCHIVES_DIR" # Log the build environment variables echo "Logging build environment variables in '$BUILD_ENV_LOG'..." env > $BUILD_ENV_LOG -echo "WS_ARCHIVE_ARTIFACTS = '$WS_ARCHIVE_ARTIFACTS'" -if [ -n "${WS_ARCHIVE_ARTIFACTS}" ]; then +echo "ARCHIVE_ARTIFACTS = '$ARCHIVE_ARTIFACTS'" +if [ -n "${ARCHIVE_ARTIFACTS:-}" ] ; then pushd $WORKSPACE shopt -s globstar # Enable globstar to copy archives - archive_artifacts=$(echo ${WS_ARCHIVE_ARTIFACTS}) - shopt -u globstar # Disable globstar - for file in $archive_artifacts; do + for file in $ARCHIVE_ARTIFACTS ; do if [ -f "$file" ] ; then fname="$(basename $file)" # Decompress core.gz file @@ -121,11 +103,15 @@ if [ -n "${WS_ARCHIVE_ARTIFACTS}" ]; then echo "Archiving '$file' to '$destfile'" destdir="$(dirname $destfile)" mkdir -p $destdir - mv $file $destfile + mv -f $file $destfile else echo "Not archiving '$file'" + if ! grep -qe '*' <<<"$file" ; then + echo "WARNING: No artifacts detected in ARCHIVE_ARTIFACTS '$ARCHIVE_ARTIFACTS'!" + fi fi done + shopt -u globstar # Disable globstar popd fi diff --git a/jjb/scripts/post_build_executor_info.sh b/jjb/scripts/post_build_executor_info.sh new file mode 100755 index 000000000..f36190045 --- /dev/null +++ b/jjb/scripts/post_build_executor_info.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/post_build_executor_info.sh" + +# Output executor runtime attributes [again] in case the job fails prior to +# running setup_executor_env.sh +long_line="************************************************************************" +OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_ARCH=$(uname -m) +echo "$long_line" +echo "Executor Runtime Attributes:" +echo "OS: $OS_ID-$OS_VERSION_ID" +echo "Arch: $OS_ARCH" +echo "Nomad Hostname: $(grep search /etc/resolv.conf | cut -d' ' -f2 | head -1)" +echo "Container ID: $(hostname)" + +if [ -n "$(which ccache)" ] ; then + echo "$long_line" + [ -n "${CCACHE_DISABLE:-}" ] && echo "CCACHE_DISABLE = '$CCACHE_DISABLE'" + [ -n "${CCACHE_DIR:-}" ] && echo "CCACHE_DIR = '$CCACHE_DIR'" + echo "ccache statistics:" + ccache -s +fi + +echo "$long_line" diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml index e5337b753..eac0bc463 100644 --- a/jjb/vpp/docs.yaml +++ b/jjb/vpp/docs.yaml @@ -107,8 +107,7 @@ - ../scripts/vpp/docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish - job-template: name: 'vpp-docs-merge-{stream}-{os}-{executor-arch}' @@ -196,8 +195,7 @@ - ../scripts/vpp/docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish - job-template: name: 'vpp-sphinx-docs-verify-{stream}-{os}-{executor-arch}' @@ -280,8 +278,7 @@ - ../scripts/vpp/sphinx-docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish - job-template: name: 'vpp-sphinx-docs-merge-{stream}-{os}-{executor-arch}' @@ -361,8 +358,7 @@ - ../scripts/vpp/sphinx-docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish - job-template: name: 'vpp-make-test-docs-verify-{stream}-{os}-{executor-arch}' @@ -432,8 +428,7 @@ - ../scripts/vpp/make-test-docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish - job-template: name: 'vpp-make-test-docs-merge-{stream}-{os}-{executor-arch}' @@ -522,5 +517,4 @@ - ../scripts/vpp/make-test-docs.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index 8031d4240..2f015d4b8 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -858,8 +858,7 @@ - robot-report: output-path: 'archives' - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-CSIT-VERIFY-DEVICE-PERIODIC JOB TEMPLATE # VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE @@ -946,8 +945,7 @@ - robot-report: output-path: 'archives' - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE # VPP-CSIT-VERIFY-PERF JOB TEMPLATE @@ -1044,8 +1042,7 @@ - robot-report: output-path: 'csit_current/0/' - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-CSIT-VERIFY-PERF JOB TEMPLATE # VPP-CSIT-VERIFY-API-CRC JOB TEMPLATE @@ -1109,7 +1106,6 @@ # TODO: Add sending e-mail notifications to vpp-api-dev@lists.fd.io publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-CSIT-VERIFY-API-CRC JOB TEMPLATE # [end] JOB TEMPLATE DEFINITIONS |