diff options
Diffstat (limited to 'jjb/scripts')
| -rwxr-xr-x | jjb/scripts/backup_upload_archives.sh | 195 | ||||
| -rw-r--r-- | jjb/scripts/packagecloud_push.sh | 23 | ||||
| -rwxr-xr-x | jjb/scripts/post_build_deploy_archives.sh | 126 | ||||
| -rw-r--r-- | jjb/scripts/setup_executor_env.sh | 4 | ||||
| -rw-r--r-- | jjb/scripts/setup_vpp_dpdk_dev_env.sh | 3 | ||||
| -rw-r--r-- | jjb/scripts/vpp/build.sh | 98 | ||||
| -rw-r--r-- | jjb/scripts/vpp/debug-build.sh | 46 | ||||
| -rw-r--r-- | jjb/scripts/vpp/docs.sh | 32 | ||||
| -rw-r--r-- | jjb/scripts/vpp/gcc-build.sh | 62 | ||||
| -rw-r--r-- | jjb/scripts/vpp/make-test-docs.sh | 28 | ||||
| -rw-r--r-- | jjb/scripts/vpp/sphinx-docs.sh | 30 |
11 files changed, 564 insertions, 83 deletions
diff --git a/jjb/scripts/backup_upload_archives.sh b/jjb/scripts/backup_upload_archives.sh new file mode 100755 index 000000000..1a9c577e1 --- /dev/null +++ b/jjb/scripts/backup_upload_archives.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/backup_upload_archives.sh" + +PYTHON_SCRIPT="/w/workspace/test-logs/artifact.py" + +# This script uploads the artifacts to a backup upload location +if [ -f "$PYTHON_SCRIPT" ]; then + echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done" + exit 0 +fi + +# TEMPORARY: talk to consul-aware resolver rather than external ones +echo "nameserver 172.17.0.1" >/etc/resolv.conf + +# the Python code below needs boto3 installed +python3 -m pip install boto3 +mkdir -p $(dirname "$PYTHON_SCRIPT") + +cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT' +#!/usr/bin/python3 + +"""Storage utilities library.""" + +import argparse +import gzip +import os +from mimetypes import MimeTypes + +from boto3 import resource +from botocore.client import Config + +ENDPOINT_URL = u"http://storage.service.consul:9000" +AWS_ACCESS_KEY_ID = u"storage" +AWS_SECRET_ACCESS_KEY = u"Storage1234" +REGION_NAME = u"yul1" +COMPRESS_MIME = ( + u"text/html", + u"text/xml", + u"application/octet-stream" +) + + +def compress(src_fpath): + """Compress a single file. + + :param src_fpath: Input file path. + :type src_fpath: str + """ + with open(src_fpath, u"rb") as orig_file: + with gzip.open(src_fpath + ".gz", u"wb") as zipped_file: + zipped_file.writelines(orig_file) + + +def upload(storage, bucket, src_fpath, dst_fpath): + """Upload single file to destination bucket. + + :param storage: S3 storage resource. + :param bucket: S3 bucket name. + :param src_fpath: Input file path. + :param dst_fpath: Destination file path on remote storage. + :type storage: Object + :type bucket: str + :type src_fpath: str + :type dst_fpath: str + """ + mime_guess = MimeTypes().guess_type(src_fpath) + mime = mime_guess[0] + encoding = mime_guess[1] + if not mime: + mime = "application/octet-stream" + + if mime in COMPRESS_MIME and bucket in "logs" and encoding != "gzip": + compress(src_fpath) + src_fpath = src_fpath + ".gz" + dst_fpath = dst_fpath + ".gz" + + extra_args = dict() + extra_args['ContentType'] = mime + + storage.Bucket(bucket + ".fd.io").upload_file( + src_fpath, + dst_fpath, + ExtraArgs=extra_args + ) + print("https://" + bucket + ".nginx.service.consul/" + dst_fpath) + + +def upload_recursive(storage, bucket, src_fpath): + """Recursively uploads input folder to destination. + + Example: + - bucket: logs + - src_fpath: /home/user + - dst_fpath: logs.fd.io/home/user + + :param storage: S3 storage resource. + :param bucket: S3 bucket name. + :param src_fpath: Input folder path. + :type storage: Object + :type bucket: str + :type src_fpath: str + """ + for path, _, files in os.walk(src_fpath): + for file in files: + _path = path.replace(src_fpath, u"") + _dir = src_fpath[1:] if src_fpath[0] == "/" else src_fpath + _dst_fpath = os.path.normpath(_dir + "/" + _path + "/" + file) + _src_fpath = os.path.join(path, file) + upload(storage, bucket, _src_fpath, _dst_fpath) + + +def main(): + """Main function for storage manipulation.""" + + parser = argparse.ArgumentParser() + parser.add_argument( + u"-d", u"--dir", required=True, type=str, + help=u"Directory to upload to storage." + ) + parser.add_argument( + u"-b", u"--bucket", required=True, type=str, + help=u"Target bucket on storage." + ) + args = parser.parse_args() + + # Create main storage resource. + storage = resource( + u"s3", + endpoint_url=ENDPOINT_URL, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + config=Config( + signature_version=u"s3v4" + ), + region_name=REGION_NAME + ) + + upload_recursive( + storage=storage, + bucket=args.bucket, + src_fpath=args.dir + ) + + +if __name__ == u"__main__": + main() + +END_OF_PYTHON_SCRIPT + +WS_ARCHIVES_DIR="$WORKSPACE/archives" +ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER" +ARCHIVES_ZIP_FILE="$WORKSPACE/.archives/archives.zip" + +TMP_ARCHIVES_DIR="/tmp/archives" +mkdir -p $TMP_ARCHIVES_DIR +pushd $TMP_ARCHIVES_DIR + +if [ -f "$ARCHIVES_ZIP_FILE" ]; then + # FIXME: this branch to be removed upon full transition to lftools: + # we are include_raw from fdio-infra-shiplogs publisher, and + # it already packed everything into archives.zip and removed + # the original archives directory. So just unpack and use that. + echo "Detected existing archives.zip, uploading the contents of it" + unzip "$ARCHIVES_ZIP_FILE" +else + # the include_raw is done from fdio-infra-publish publisher + mkdir -p "$ARCHIVES_DIR" + if [ -e "$WS_ARCHIVES_DIR" ]; then + echo "Found $WS_ARCHIVES_DIR, uploading its contents" + cp -r "$WS_ARCHIVES_DIR" . + else + echo "No $WS_ARCHIVES_DIR found. Creating a dummy file." + echo "No archives found while doing backup upload" > "$ARCHIVES_DIR/no-archives-found.txt" + fi +fi + +echo "Contents of the archives dir:" +ls -alR $TMP_ARCHIVES_DIR +echo "Running uploader script $PYTHON_SCRIPT:" +python3 $PYTHON_SCRIPT -d . -b logs || echo "Failed to upload logs" +popd diff --git a/jjb/scripts/packagecloud_push.sh b/jjb/scripts/packagecloud_push.sh index af3bf48ff..a4ae9b166 100644 --- a/jjb/scripts/packagecloud_push.sh +++ b/jjb/scripts/packagecloud_push.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,6 +17,22 @@ echo "---> jjb/scripts/packagecloud_push.sh" set -euxo pipefail +line="*************************************************************************" + +# Nothing was built if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision ID +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping package push. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DRYRUN="${DRYRUN:-}" +if [ "${DRYRUN,,}" = "true" ] ; then + echo -e "\n$line\nSkipping package push because DRYRUN is '${DRYRUN,,}'.\n$line\n" + exit 0 +fi + echo "STARTING PACKAGECLOUD PUSH" sleep 10 @@ -58,8 +74,7 @@ if [ -f ~/.packagecloud ]; then fi ;; *) - echo "ERROR: Unsupported OS '$FACTER_OS'" - echo "PACKAGECLOUD PUSH FAILED!" + echo -e "\n$line\n* ERROR: Unsupported OS '$FACTER_OS'\n* PACKAGECLOUD PUSH FAILED!\n$line\n" exit 1 ;; esac @@ -83,4 +98,4 @@ else exit 1 fi -echo "PACKAGECLOUD PUSH COMPLETE" +echo -e "\n$line\n* PACKAGECLOUD PUSH COMPLETE\n$line\n" diff --git a/jjb/scripts/post_build_deploy_archives.sh b/jjb/scripts/post_build_deploy_archives.sh new file mode 100755 index 000000000..0a47903e8 --- /dev/null +++ b/jjb/scripts/post_build_deploy_archives.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/post_build_deploy_archives.sh" + +set +e # Do not affect the build result if some part of archiving fails. +WS_ARCHIVES_DIR="$WORKSPACE/archives" +BUILD_ENV_LOG="$WS_ARCHIVES_DIR/_build-enviroment-variables.log" + +# Generate gdb-command script to output vpp stack traceback from core files. +gdb_cmdfile="/tmp/gdb-commands" +cat >$gdb_cmdfile <<'__END__' +# Usage: +# gdb $BINFILE $CORE -ex 'source -v gdb-commands' -ex quit + +set pagination off +thread apply all bt + +define printstack + set $i=0 + while $i < 15 + frame $i + x/i $pc + info locals + info reg + set $i = $i + 1 + end +end +thread apply all printstack + +# info proc mappings + +__END__ + +STACKTRACE="" +# Returns stacktrace filename in STACKTRACE +generate_vpp_stacktrace_and_delete_core() { + local corefile="$1" + if grep -qe 'debug' <<< "$WORKSPACE" ; then + local binfile="$WORKSPACE/build-root/install-vpp_debug-native/vpp/bin/vpp" + else + local binfile="$WORKSPACE/build-root/install-vpp-native/vpp/bin/vpp" + fi + + echo "Generating stack trace from core file: $corefile" + STACKTRACE="${corefile}.stacktrace" + gdb "$binfile" $corefile -ex 'source -v /tmp/gdb-commands' -ex quit > $STACKTRACE + # remove the core to save space + echo "Removing core file: $corefile" + rm -f "$corefile" + # Dump stacktrace to console log + if [ -f $STACKTRACE ] ; then + echo -e "\n=====[ $STACKTRACE ]=====\n$(cat $STACKTRACE)\n=====[ $STACKTRACE ]=====\n" + else + echo "Stacktrace file not generated!" + STACKTRACE="" + fi +} + +# Delete existing archives dir to ensure current artifact upload +rm -rf "$WS_ARCHIVES_DIR" +mkdir -p "$WS_ARCHIVES_DIR" + +# Log the build environment variables +echo "Logging build environment variables in '$BUILD_ENV_LOG'..." +env > $BUILD_ENV_LOG + +echo "WS_ARCHIVE_ARTIFACTS = '$WS_ARCHIVE_ARTIFACTS'" +if [ -n "${WS_ARCHIVE_ARTIFACTS}" ]; then + pushd $WORKSPACE + shopt -s globstar # Enable globstar to copy archives + archive_artifacts=$(echo ${WS_ARCHIVE_ARTIFACTS}) + shopt -u globstar # Disable globstar + for file in $archive_artifacts; do + if [ -f "$file" ] ; then + fname="$(basename $file)" + # Decompress core.gz file + if grep -qe '^core.*\.gz$' <<<"$fname" ; then + echo "Uncompressing core file $file" + gunzip "$file" + file="${file::(-3)}" + fi + # Convert core file to stacktrace + if [ "${fname::4}" = "core" ] ; then + generate_vpp_stacktrace_and_delete_core $file + [ -z "$STACKTRACE" ] && continue + file=$STACKTRACE + fi + # Set destination filename + if [ "${file::26}" = "/tmp/vpp-failed-unittests/" ] ; then + destfile=$WS_ARCHIVES_DIR${file:25} + else + destfile=$WS_ARCHIVE_DIR$file + fi + echo "Archiving '$file' to '$destfile'" + destdir="$(dirname $destfile)" + mkdir -p $destdir + mv $file $destfile + else + echo "Not archiving '$file'" + fi + done + popd +fi + +# find and gzip any 'text' files +find $WS_ARCHIVES_DIR -type f -print0 \ + | xargs -0r file \ + | egrep -e ':.*text.*' \ + | cut -d: -f1 \ + | xargs -d'\n' -r gzip + +echo "Workspace archived artifacts:" +ls -alR $WS_ARCHIVES_DIR diff --git a/jjb/scripts/setup_executor_env.sh b/jjb/scripts/setup_executor_env.sh index 08b37873c..2bfbe8b32 100644 --- a/jjb/scripts/setup_executor_env.sh +++ b/jjb/scripts/setup_executor_env.sh @@ -49,6 +49,10 @@ elif [ "$OS_ID" == "centos" ] ; then fi echo "$long_line" +echo "Python3 package list:" +pip3 list 2>/dev/null | column -t || true + +echo "$long_line" echo "Executor Downloads cache '$downloads_cache':" ls -lh "$downloads_cache" || true echo "$long_line" diff --git a/jjb/scripts/setup_vpp_dpdk_dev_env.sh b/jjb/scripts/setup_vpp_dpdk_dev_env.sh index cd5bab447..2c55f166b 100644 --- a/jjb/scripts/setup_vpp_dpdk_dev_env.sh +++ b/jjb/scripts/setup_vpp_dpdk_dev_env.sh @@ -48,6 +48,9 @@ function setup { force_opts="$force_opts --allow-change-held-packages" sudo apt-get -y $force_opts install vpp-ext-deps || true fi + echo "Removing packagecloud.io repository references and running apt-get update" + sudo rm -f /etc/apt/sources.list.d/fdio_*.list + sudo apt-get update -qq || true elif [ "${OS_ID,,}" == "centos" ] ; then if [ "${STREAM}" != "master" ] ; then echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list" diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh index 52cee0978..bfeaacea9 100644 --- a/jjb/scripts/vpp/build.sh +++ b/jjb/scripts/vpp/build.sh @@ -1,7 +1,6 @@ #!/bin/bash -# basic build script example -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,11 +15,30 @@ echo "---> jjb/scripts/vpp/build.sh" -set -xe -o pipefail +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping build. A newer patch has been merged.\n$line\n" + exit 0 +fi OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +IS_CSIT_VPP_JOB="${IS_CSIT_VPP_JOB:-}" +MAKE_PARALLEL_FLAGS="${MAKE_PARALLEL_FLAGS:-}" +MAKE_PARALLEL_JOBS="${MAKE_PARALLEL_JOBS:-}" +MAKE_TEST_OS="${MAKE_TEST_OS:-ubuntu-18.04}" +MAKE_TEST_MULTIWORKER_OS="${MAKE_TEST_MULTIWORKER_OS:-debian-10}" +VPPAPIGEN_TEST_OS="${VPPAPIGEN_TEST_OS:-${MAKE_TEST_OS}}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" echo "sha1sum of this script: ${0}" sha1sum $0 @@ -35,11 +53,11 @@ sha1sum $0 # echo $CCACHE_DIR does not exist. #fi -if [ "x${MAKE_PARALLEL_FLAGS}" != "x" ] +if [ -n "${MAKE_PARALLEL_FLAGS}" ] then echo "Building VPP. Number of cores for build set with" \ "MAKE_PARALLEL_FLAGS='${MAKE_PARALLEL_FLAGS}'." -elif [ "x${MAKE_PARALLEL_JOBS}" != "x" ] +elif [ -n "${MAKE_PARALLEL_JOBS}" ] then echo "Building VPP. Number of cores for build set with" \ "MAKE_PARALLEL_JOBS='${MAKE_PARALLEL_JOBS}'." @@ -48,15 +66,24 @@ else "using build default ($(grep -c ^processor /proc/cpuinfo))." fi -echo "CC=${CC}" -echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" - -# If we are not a CSIT job just building packages, then use make verify, -# else use make pkg-verify. -if [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] -then - if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] - then +make_build_test() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make UNATTENDED=yes pkg-verify ; then + BUILD_ERROR="FAILED 'make pkg-verify'" + return + fi + if [ "${IS_CSIT_VPP_JOB,,}" == "true" ]; then + # CSIT jobs don't need to run make test + return + fi + if [ -n "${MAKE_PARALLEL_JOBS}" ]; then export TEST_JOBS="${MAKE_PARALLEL_JOBS}" echo "Testing VPP with ${TEST_JOBS} cores." else @@ -64,13 +91,38 @@ then echo "Testing VPP with automatically calculated number of cores. " \ "See test logs for the exact number." fi - echo "Building using \"make verify\"" - [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes verify -else - echo "Building using \"make pkg-verify\"" - [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes pkg-verify -fi + if [ "${OS_ID}-${OS_VERSION_ID}" == "${VPPAPIGEN_TEST_OS}" ]; then + if ! src/tools/vppapigen/test_vppapigen.py; then + BUILD_ERROR="FAILED src/tools/vppapigen/test_vppapigen.py" + return + fi + fi + if [ "${OS_ID}-${OS_VERSION_ID}" == "${MAKE_TEST_OS}" ]; then + if ! make COMPRESS_FAILED_TEST_LOGS=yes RETRIES=3 test; then + BUILD_ERROR="FAILED 'make test'" + return + fi + else + echo "Skip running 'make test' on ${OS_ID}-${OS_VERSION_ID}" + fi + if [ "${OS_ID}-${OS_VERSION_ID}" == "${MAKE_TEST_MULTIWORKER_OS}" ] && git grep VPP_WORKER_CONFIG; then + if ! make VPP_WORKER_CONFIG="workers 2" COMPRESS_FAILED_TEST_LOGS=yes RETRIES=3 test; then + BUILD_ERROR="FAILED 'make test' with VPP_WORKER_CONFIG='workers 2'" + return + else + echo -e "\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} MULTIWORKER MAKE TEST SUCCESSFULLY COMPLETED\n" + fi + else + echo "Skip running 'make test' with VPP_WORKER_CONFIG='workers 2' on ${OS_ID}-${OS_VERSION_ID}" + fi +} -echo "*******************************************************************" -echo "* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED" -echo "*******************************************************************" +if [ "${DRYRUN,,}" != "true" ] ; then + make_build_test +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh index a9887340d..dba984f21 100644 --- a/jjb/scripts/vpp/debug-build.sh +++ b/jjb/scripts/vpp/debug-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,11 +15,16 @@ echo "---> jjb/scripts/vpp/debug-build.sh" -set -xe -o pipefail +set -euxo pipefail +line="*************************************************************************" OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" echo "sha1sum of this script: ${0}" sha1sum $0 @@ -27,15 +32,34 @@ sha1sum $0 # run with ASAN on export VPP_EXTRA_CMAKE_ARGS='-DVPP_ENABLE_SANITIZE_ADDR=ON' +make_build_test_debug() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make UNATTENDED=yes build ; then + BUILD_ERROR="FAILED 'make build'" + return + fi + if ! make UNATTENDED=yes TEST_JOBS=auto test-debug ; then + BUILD_ERROR="FAILED 'make test-debug'" + return + fi +} + # clang is not working with ASAN right now - see change 27268 # also, it does not work with gcc-7, we need gcc-8 at least # on ubuntu 20.04 executor the gcc is gcc9 - -make UNATTENDED=yes install-dep -make UNATTENDED=yes install-ext-deps -make UNATTENDED=yes build -make UNATTENDED=yes TEST_JOBS=auto test-debug - -echo "*******************************************************************" -echo "* VPP debug/asan test BUILD on ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} SUCCESSFULLY COMPLETED" -echo "*******************************************************************" +if [ "${DRYRUN,,}" != "true" ] ; then + make_build_test_debug +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} DEBUG BUILD $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/docs.sh b/jjb/scripts/vpp/docs.sh index 0899b6615..866b32217 100644 --- a/jjb/scripts/vpp/docs.sh +++ b/jjb/scripts/vpp/docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,19 +15,29 @@ echo "---> jjb/scripts/vpp/docs.sh" -set -xe -o pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp -[ "$DOC_FILE" ] || DOC_FILE=vpp.docs.zip -[ "$DOC_DIR" ] || DOC_DIR=build-root/docs/html -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site/ -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -[ "$VERSION" ] || VERSION=$(./build-root/scripts/version rpm-version) +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping doxygen docs build. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DOCS_REPO_URL=${DOCS_REPO_URL:-"https://nexus.fd.io/content/sites/site"} +PROJECT_PATH=${PROJECT_PATH:-"io/fd/vpp"} +DOC_FILE=${DOC_FILE:-"vpp.docs.zip"} +DOC_DIR=${DOC_DIR:-"build-root/docs/html"} +SITE_DIR=${SITE_DIR:-"build-root/docs/deploy-site"} +RESOURCES_DIR=${RESOURCES_DIR:-"${SITE_DIR}/src/site/resources"} +MVN=${MVN:-"/opt/apache/maven/bin/mvn"} +VERSION=${VERSION:-"$(./build-root/scripts/version rpm-version)"} make doxygen -if [[ ${JOB_NAME} == *merge* ]]; then +if [[ ${JOB_NAME} == *merge* ]] ; then mkdir -p $(dirname ${RESOURCES_DIR}) mv -f ${DOC_DIR} ${RESOURCES_DIR} cd ${SITE_DIR} diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh index 5cdbc2633..488e8d7d7 100644 --- a/jjb/scripts/vpp/gcc-build.sh +++ b/jjb/scripts/vpp/gcc-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,27 +15,59 @@ echo "---> jjb/scripts/vpp/gcc-build.sh" -set -xe -o pipefail +set -euxo pipefail +line="*************************************************************************" OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" echo "sha1sum of this script: ${0}" sha1sum $0 export CC=gcc -make UNATTENDED=yes install-dep -make UNATTENDED=yes install-ext-deps -make UNATTENDED=yes build -# TODO: Add 'smoke test' env var to select smoke test cases -# then update this accordingly. For now pick a few basic suites... -MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash" -for suite in $MAKE_TEST_SUITES ; do - make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test - make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug -done +make_build_release_build_test_gcov_sanity() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make UNATTENDED=yes build-release ; then + BUILD_ERROR="FAILED 'make build'" + return + fi + if ! make UNATTENDED=yes build ; then + BUILD_ERROR="FAILED 'make build'" + return + fi + # TODO: Add 'smoke test' env var to select smoke test cases + # then update this accordingly. For now pick a few basic suites... + MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash" + for suite in $MAKE_TEST_SUITES ; do + if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test ; then + BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test'!" + return + fi + if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug ; then + BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug'!" + return + fi + done +} -echo "*******************************************************************" -echo "* VPP GCC on ${OS_ID^^}-${OS_VERSION}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED" -echo "*******************************************************************" +if [ "${DRYRUN,,}" != "true" ] ; then + make_build_release_build_test_gcov_sanity +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} GCC BUILD $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/make-test-docs.sh b/jjb/scripts/vpp/make-test-docs.sh index d15cac380..d20a7b817 100644 --- a/jjb/scripts/vpp/make-test-docs.sh +++ b/jjb/scripts/vpp/make-test-docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,14 +15,24 @@ echo "---> jjb/scripts/vpp/make-test-docs.sh" -set -xe -o pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp -[ "$DOC_DIR" ] || DOC_DIR=build-root/build-test/doc/html -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/vpp_make_test -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -[ "$VERSION" ] || VERSION=$(./build-root/scripts/version rpm-version) +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping 'make test' doxygen docs build. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DOCS_REPO_URL=${DOCS_REPO_URL:-"https://nexus.fd.io/content/sites/site"} +PROJECT_PATH=${PROJECT_PATH:-"io/fd/vpp"} +DOC_DIR=${DOC_DIR:-"build-root/build-test/doc/html"} +SITE_DIR=${SITE_DIR:-"build-root/docs/deploy-site"} +RESOURCES_DIR=${RESOURCES_DIR:-"${SITE_DIR}/src/site/resources/vpp_make_test"} +MVN=${MVN:-"/opt/apache/maven/bin/mvn"} +VERSION=${VERSION:-"$(./build-root/scripts/version rpm-version)"} make test-doc diff --git a/jjb/scripts/vpp/sphinx-docs.sh b/jjb/scripts/vpp/sphinx-docs.sh index b9eb64bc7..d80b4cd38 100644 --- a/jjb/scripts/vpp/sphinx-docs.sh +++ b/jjb/scripts/vpp/sphinx-docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,15 +15,25 @@ echo "---> jjb/scripts/vpp/sphinx-docs.sh" -set -xe -o pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp -[ "$DOC_FILE" ] || DOC_FILE=vpp.docs.zip -[ "$DOC_DIR" ] || DOC_DIR=./docs/_build/html -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -[ "$VERSION" ] || VERSION=$(./build-root/scripts/version rpm-version) +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping sphinx docs build. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DOCS_REPO_URL=${DOCS_REPO_URL:-"https://nexus.fd.io/content/sites/site"} +PROJECT_PATH=${PROJECT_PATH:-"io/fd/vpp"} +DOC_FILE=${DOC_FILE:-"vpp.docs.zip"} +DOC_DIR=${DOC_DIR:-"./docs/_build/html"} +SITE_DIR=${SITE_DIR:-"build-root/docs/deploy-site"} +RESOURCES_DIR=${RESOURCES_DIR:-"${SITE_DIR}/src/site/resources"} +MVN=${MVN:-"/opt/apache/maven/bin/mvn"} +VERSION=${VERSION:-"$(./build-root/scripts/version rpm-version)"} make docs-venv CONFIRM=-y FORCE=--force-yes make docs |