From 2db30aa5a938494a710a89cab5943675c1143918 Mon Sep 17 00:00:00 2001 From: Dave Wallace Date: Thu, 1 Oct 2020 20:38:47 +0000 Subject: Clean up vpp jenkins scripts & docker dir - Remove docker/vpp Dockerfile definitions and colateral files for image now being build by https://gerrit.fd.io/r/c/ci-management/+/28022 - Remove opensuse support which has been elided from the vpp build in https://gerrit.fd.io/r/c/vpp/+/28999 - Remove unused scripts. - Output script name marker for all jjb scripts following LFIT practices. - Remove old/unused output from scripts. - Add logic to use vpp-ext-deps download cache in the executor images to speed up build jobs. - Add copyright notifcations to modified files without an existing copyright. - Make all vpp/docs jobs use '-{stream}-{os}-{executor}' suffix in their names. - Consolidate executor logging into setup_executor_env.sh and call it first for all jobs. Change-Id: I4ffb67ed476659f0127ca8c84f20a305d06149bf Signed-off-by: Dave Wallace --- jjb/scripts/vpp/api-checkstyle.sh | 15 ++++++ jjb/scripts/vpp/build.sh | 96 ++++++++++++++++---------------------- jjb/scripts/vpp/check_crc.sh | 2 + jjb/scripts/vpp/checkstyle.sh | 16 ++++++- jjb/scripts/vpp/commitmsg.sh | 19 +++++++- jjb/scripts/vpp/csit-device.sh | 2 + jjb/scripts/vpp/csit-perf.sh | 2 + jjb/scripts/vpp/debug-build.sh | 44 +++++++---------- jjb/scripts/vpp/docs.sh | 16 +++++++ jjb/scripts/vpp/gcc-build.sh | 40 +++++++--------- jjb/scripts/vpp/make-test-docs.sh | 16 +++++++ jjb/scripts/vpp/maven-push.sh | 56 ---------------------- jjb/scripts/vpp/sphinx-docs.sh | 16 +++++++ jjb/scripts/vpp/test-checkstyle.sh | 16 ++++++- 14 files changed, 193 insertions(+), 163 deletions(-) delete mode 100644 jjb/scripts/vpp/maven-push.sh (limited to 'jjb/scripts/vpp') diff --git a/jjb/scripts/vpp/api-checkstyle.sh b/jjb/scripts/vpp/api-checkstyle.sh index 90740337e..38017fea1 100644 --- a/jjb/scripts/vpp/api-checkstyle.sh +++ b/jjb/scripts/vpp/api-checkstyle.sh @@ -1,5 +1,20 @@ #!/bin/bash +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/api-checkstyle.sh" + VPP_CRC_CHECKER="extras/scripts/crcchecker.py" VPP_CRC_CHECKER_CMD="$VPP_CRC_CHECKER --check-patchset" diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh index 68fa30d1a..52cee0978 100644 --- a/jjb/scripts/vpp/build.sh +++ b/jjb/scripts/vpp/build.sh @@ -1,45 +1,39 @@ #!/bin/bash # basic build script example -set -xe -o pipefail -OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') -OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -echo OS_ID: $OS_ID -echo OS_VERSION_ID: $OS_VERSION_ID +echo "---> jjb/scripts/vpp/build.sh" -# do nothing but print the current slave hostname -hostname -export CCACHE_DIR=/tmp/ccache -if [ -d $CCACHE_DIR ];then - echo $CCACHE_DIR exists - du -sk $CCACHE_DIR -else - echo $CCACHE_DIR does not exist. This must be a new slave. -fi - -echo "cat /etc/bootstrap.sha" -if [ -f /etc/bootstrap.sha ];then - cat /etc/bootstrap.sha -else - echo "Cannot find cat /etc/bootstrap.sha" -fi +set -xe -o pipefail -echo "cat /etc/bootstrap-functions.sha" -if [ -f /etc/bootstrap-functions.sha ];then - cat /etc/bootstrap-functions.sha -else - echo "Cannot find cat /etc/bootstrap-functions.sha" -fi +OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_ARCH=$(uname -m) echo "sha1sum of this script: ${0}" sha1sum $0 -echo "CC=${CC}" -echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" -# If and only if we are doing verify *after* make verify was made to work -# and we are not a CSIT job just building packages, then use make verify, -# else use make pkg-verify. +# TODO: Mount ccache volume into docker container, then enable this. +# +#export CCACHE_DIR=/scratch/docker-build/ccache +#if [ -d $CCACHE_DIR ];then +# echo "ccache size:" +# du -sh $CCACHE_DIR +#else +# echo $CCACHE_DIR does not exist. +#fi if [ "x${MAKE_PARALLEL_FLAGS}" != "x" ] then @@ -54,19 +48,22 @@ else "using build default ($(grep -c ^processor /proc/cpuinfo))." fi -if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] -then - export TEST_JOBS="${MAKE_PARALLEL_JOBS}" - echo "Testing VPP with ${TEST_JOBS} cores." -else - export TEST_JOBS="auto" - echo "Testing VPP with automatically calculated number of cores. " \ - "See test logs for the exact number." -fi +echo "CC=${CC}" +echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" -if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \ - [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] +# If we are not a CSIT job just building packages, then use make verify, +# else use make pkg-verify. +if [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] then + if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] + then + export TEST_JOBS="${MAKE_PARALLEL_JOBS}" + echo "Testing VPP with ${TEST_JOBS} cores." + else + export TEST_JOBS="auto" + echo "Testing VPP with automatically calculated number of cores. " \ + "See test logs for the exact number." + fi echo "Building using \"make verify\"" [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes verify else @@ -74,17 +71,6 @@ else [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes pkg-verify fi -if [ "x${VPP_REPO}" == "x1" ]; then - if [ "x${REBASE_NEEDED}" == "x1" ]; then - echo "This patch to vpp is based on an old point in the tree that is likely" - echo "to fail verify." - echo "PLEASE REBASE PATCH ON THE CURRENT HEAD OF THE VPP REPO" - exit 1 - fi -fi - -local_arch=$(uname -m) - echo "*******************************************************************" -echo "* VPP ${local_arch^^} BUILD SUCCESSFULLY COMPLETED" +echo "* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED" echo "*******************************************************************" diff --git a/jjb/scripts/vpp/check_crc.sh b/jjb/scripts/vpp/check_crc.sh index 04e53966d..2e07c2b5e 100644 --- a/jjb/scripts/vpp/check_crc.sh +++ b/jjb/scripts/vpp/check_crc.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +echo "---> jjb/scripts/vpp/check_crc.sh" + set -exuo pipefail # Clone CSIT git repository and proceed with entry script located there. diff --git a/jjb/scripts/vpp/checkstyle.sh b/jjb/scripts/vpp/checkstyle.sh index 7c520dd99..55aa62a53 100644 --- a/jjb/scripts/vpp/checkstyle.sh +++ b/jjb/scripts/vpp/checkstyle.sh @@ -1,5 +1,19 @@ #!/bin/bash -# jjb/vpp/include-raw-vpp-checkstyle.sh + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/checkstyle.sh" if [ -n "$(grep -E '^checkstyle:' Makefile)" ] then diff --git a/jjb/scripts/vpp/commitmsg.sh b/jjb/scripts/vpp/commitmsg.sh index d926ff5dd..479f35136 100644 --- a/jjb/scripts/vpp/commitmsg.sh +++ b/jjb/scripts/vpp/commitmsg.sh @@ -1,7 +1,22 @@ #!/bin/bash -if [ -f extras/scripts/check_commit_msg.sh ];then - echo "Running extras/scripts/check_commit_msg.sh" +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/commitmsg.sh" + +if [ -f extras/scripts/check_commit_msg.sh ] ; then + echo "Running extras/scripts/check_commit_msg.sh" extras/scripts/check_commit_msg.sh else echo "Cannot find cat extras/scripts/check_commit_msg.sh - skipping commit message check" diff --git a/jjb/scripts/vpp/csit-device.sh b/jjb/scripts/vpp/csit-device.sh index af54c9dc4..6d4beb80f 100644 --- a/jjb/scripts/vpp/csit-device.sh +++ b/jjb/scripts/vpp/csit-device.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +echo "---> jjb/scripts/vpp/csit-device.sh" + set -exuo pipefail # Clone CSIT git repository and proceed with entry script located there. diff --git a/jjb/scripts/vpp/csit-perf.sh b/jjb/scripts/vpp/csit-perf.sh index 17a9d39bc..e57306651 100644 --- a/jjb/scripts/vpp/csit-perf.sh +++ b/jjb/scripts/vpp/csit-perf.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +echo "---> jjb/scripts/vpp/csit-perf.sh" + set -exuo pipefail # Clone CSIT git repository and proceed with entry script located there. diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh index cdf1d0760..e7f185218 100644 --- a/jjb/scripts/vpp/debug-build.sh +++ b/jjb/scripts/vpp/debug-build.sh @@ -1,34 +1,29 @@ #!/bin/bash -# basic build script example + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/debug-build.sh" + set -xe -o pipefail OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') - -echo OS_ID: $OS_ID -echo OS_VERSION_ID: $OS_VERSION_ID - -# do nothing but print the current slave hostname -hostname - -echo "cat /etc/bootstrap.sha" -if [ -f /etc/bootstrap.sha ];then - cat /etc/bootstrap.sha -else - echo "Cannot find cat /etc/bootstrap.sha" -fi - -echo "cat /etc/bootstrap-functions.sha" -if [ -f /etc/bootstrap-functions.sha ];then - cat /etc/bootstrap-functions.sha -else - echo "Cannot find cat /etc/bootstrap-functions.sha" -fi +OS_ARCH=$(uname -m) echo "sha1sum of this script: ${0}" sha1sum $0 - # run with ASAN on # export VPP_EXTRA_CMAKE_ARGS='-DVPP_ENABLE_SANITIZE_ADDR=ON' @@ -36,14 +31,11 @@ sha1sum $0 # apparently gcc neither... # export CC=gcc - - make UNATTENDED=yes install-dep make UNATTENDED=yes install-ext-deps make UNATTENDED=yes build make UNATTENDED=yes TEST_JOBS=auto test-debug - echo "*******************************************************************" -echo "* VPP debug/asan test BUILD SUCCESSFULLY COMPLETED" +echo "* VPP debug/asan test BUILD on ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} SUCCESSFULLY COMPLETED" echo "*******************************************************************" diff --git a/jjb/scripts/vpp/docs.sh b/jjb/scripts/vpp/docs.sh index 608f8f979..0899b6615 100644 --- a/jjb/scripts/vpp/docs.sh +++ b/jjb/scripts/vpp/docs.sh @@ -1,4 +1,20 @@ #!/bin/bash + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/docs.sh" + set -xe -o pipefail [ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" [ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh index 6f8b8f686..5cdbc2633 100644 --- a/jjb/scripts/vpp/gcc-build.sh +++ b/jjb/scripts/vpp/gcc-build.sh @@ -1,29 +1,25 @@ #!/bin/bash -# basic build script example + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/gcc-build.sh" + set -xe -o pipefail OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') - -echo OS_ID: $OS_ID -echo OS_VERSION_ID: $OS_VERSION_ID - -# do nothing but print the current slave hostname -hostname - -echo "cat /etc/bootstrap.sha" -if [ -f /etc/bootstrap.sha ];then - cat /etc/bootstrap.sha -else - echo "Cannot find cat /etc/bootstrap.sha" -fi - -echo "cat /etc/bootstrap-functions.sha" -if [ -f /etc/bootstrap-functions.sha ];then - cat /etc/bootstrap-functions.sha -else - echo "Cannot find cat /etc/bootstrap-functions.sha" -fi +OS_ARCH=$(uname -m) echo "sha1sum of this script: ${0}" sha1sum $0 @@ -41,5 +37,5 @@ for suite in $MAKE_TEST_SUITES ; do done echo "*******************************************************************" -echo "* VPP GCC BUILD SUCCESSFULLY COMPLETED" +echo "* VPP GCC on ${OS_ID^^}-${OS_VERSION}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED" echo "*******************************************************************" diff --git a/jjb/scripts/vpp/make-test-docs.sh b/jjb/scripts/vpp/make-test-docs.sh index 1f82ab502..d15cac380 100644 --- a/jjb/scripts/vpp/make-test-docs.sh +++ b/jjb/scripts/vpp/make-test-docs.sh @@ -1,4 +1,20 @@ #!/bin/bash + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/make-test-docs.sh" + set -xe -o pipefail [ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" [ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp diff --git a/jjb/scripts/vpp/maven-push.sh b/jjb/scripts/vpp/maven-push.sh deleted file mode 100644 index 6ce3b2c06..000000000 --- a/jjb/scripts/vpp/maven-push.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -xe -o pipefail -echo "*******************************************************************" -echo "* STARTING PUSH OF PACKAGES TO REPOS" -echo "* NOTHING THAT HAPPENS BELOW THIS POINT IS RELATED TO BUILD FAILURE" -echo "*******************************************************************" - -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -GROUP_ID="io.fd.${PROJECT}" -BASEURL="${NEXUSPROXY}/content/repositories/fd.io." -BASEREPOID='fdio-' - -if [ "${OS}" == "ubuntu1604" ]; then - # Find the files - JARS=$(find . -type f -iname '*.jar') - DEBS=$(find . -type f -iname '*.deb') - for i in $JARS - do - push_jar "$i" - done - - for i in $DEBS - do - push_deb "$i" - done -elif [ "${OS}" == "ubuntu1804" ]; then - # Find the files - JARS=$(find . -type f -iname '*.jar') - DEBS=$(find . -type f -iname '*.deb') - for i in $JARS - do - push_jar "$i" - done - - for i in $DEBS - do - push_deb "$i" - done -elif [ "${OS}" == "centos7" ]; then - # Find the files - RPMS=$(find . -type f -iname '*.rpm') - SRPMS=$(find . -type f -iname '*.srpm') - SRCRPMS=$(find . -type f -name '*.src.rpm') - for i in $RPMS $SRPMS $SRCRPMS - do - push_rpm "$i" - done -elif [ "${OS}" == "opensuse" ]; then - # Find the files - RPMS=$(find . -type f -iname '*.rpm') - for i in $RPMS - do - push_rpm "$i" - done -fi -# vim: ts=4 sw=4 sts=4 et ft=sh : diff --git a/jjb/scripts/vpp/sphinx-docs.sh b/jjb/scripts/vpp/sphinx-docs.sh index 749728731..b9eb64bc7 100644 --- a/jjb/scripts/vpp/sphinx-docs.sh +++ b/jjb/scripts/vpp/sphinx-docs.sh @@ -1,4 +1,20 @@ #!/bin/bash + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/sphinx-docs.sh" + set -xe -o pipefail [ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" [ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp diff --git a/jjb/scripts/vpp/test-checkstyle.sh b/jjb/scripts/vpp/test-checkstyle.sh index bdc843114..3f0bc9a06 100644 --- a/jjb/scripts/vpp/test-checkstyle.sh +++ b/jjb/scripts/vpp/test-checkstyle.sh @@ -1,5 +1,19 @@ #!/bin/bash -# jjb/vpp/include-raw-vpp-test-checkstyle.sh + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/test-checkstyle.sh" if [ -n "$(grep -E '^test-checkstyle:' Makefile)" ] then -- cgit 1.2.3-korg