diff options
author | 2021-10-28 21:38:49 -0400 | |
---|---|---|
committer | 2021-10-29 12:43:26 -0400 | |
commit | e62c4bc9708a906d2dcd504cdcca5964b232d3b3 (patch) | |
tree | 0dd88ce97e13ef4fb3973aed929bc3198ae6018e | |
parent | 54f10a6f38996907cf3dbcb698810757b4dfffd5 (diff) |
Fix upload of vpp-ext-deps cached in executor
- Refactor installation of vpp-ext-deps
Change-Id: Ifd22ceb0481c9b2f2c6be223dae64e775d59fa12
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
-rwxr-xr-x[-rw-r--r--] | jjb/scripts/packagecloud_push.sh | 64 | ||||
-rw-r--r-- | jjb/scripts/setup_vpp_dpdk_dev_env.sh | 80 | ||||
-rwxr-xr-x | jjb/scripts/setup_vpp_ext_deps.sh | 75 | ||||
-rwxr-xr-x[-rw-r--r--] | jjb/scripts/setup_vpp_ubuntu_docker_test.sh | 16 | ||||
-rw-r--r-- | jjb/vpp/vpp.yaml | 14 |
5 files changed, 121 insertions, 128 deletions
diff --git a/jjb/scripts/packagecloud_push.sh b/jjb/scripts/packagecloud_push.sh index a4ae9b166..80226aa04 100644..100755 --- a/jjb/scripts/packagecloud_push.sh +++ b/jjb/scripts/packagecloud_push.sh @@ -40,38 +40,53 @@ sleep 10 FACTER_OS=$(/usr/bin/facter operatingsystem) push_cmd="" push_ext_deps_cmd="" +ext_deps_pkg="" +downloads_dir="/root/Downloads" + +create_deb_push_cmds() +{ + local distro="$1" + + if [ "$distro" = "debian" ] || [ "$distro" = "ubuntu" ] ; then + FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename) + DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps | xargs || true) + push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/${distro}/${FACTER_LSBNAME}/main/ ${DEBS}" + ext_deps_ver="$(dpkg -l vpp-ext-deps | mawk '/vpp-ext-deps/{print $3}' || true)" + ext_deps_pkg="$(find . -type f -iname 'vpp-ext-deps*.deb' | grep $ext_deps_ver || find $downloads_dir -type f -iname 'vpp-ext-deps*.deb' | grep $ext_deps_ver || true)" + if [ -n "$ext_deps_pkg}" ] ; then + push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/${distro}/${FACTER_LSBNAME}/main/ ${ext_deps_pkg}" + fi + else + echo "ERROR: Unknown distro: '$distro'" + return 1 + fi +} + +create_rpm_push_cmds() +{ + FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease) + FACTER_ARCH=$(/usr/bin/facter architecture) + RPMS=$(find . -type f -iregex '.*/.*\.\(s\)?rpm' | grep -v vpp-ext-deps | xargs || true) + push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${RPMS}" + ext_deps_ver="$(dnf list vpp-ext-deps | mawk '/vpp-ext-deps/{print $2}' || true)" + ext_deps_pkg="$(find . -type f -iname 'vpp-ext-deps*.rpm' | grep $ext_deps_ver || find $downloads_dir -type f -iname 'vpp-ext-deps*.rpm' | grep $ext_deps_ver || true)" + if [ -n "$ext_deps_pkg" ] ; then + push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${ext_deps_pkg}" + fi +} # PCIO_CO and SILO are Jenkins Global Environment variables defined in # .../ci-management/jenkins-config/global-vars-*.sh if [ -f ~/.packagecloud ]; then case "$FACTER_OS" in Debian) - FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename) - DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps) - push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/debian/${FACTER_LSBNAME}/main/ ${DEBS}" - EXT_DEPS_DEB=$(find . -type f -iname 'vpp-ext-deps*.deb') - if [ -n "$EXT_DEPS_DEB" ] ; then - push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/debian/${FACTER_LSBNAME}/main/ ${EXT_DEPS_DEB}" - fi + create_deb_push_cmds debian ;; Ubuntu) - FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename) - DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps) - push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ ${DEBS}" - EXT_DEPS_DEB=$(find . -type f -iname 'vpp-ext-deps*.deb') - if [ -n "$EXT_DEPS_DEB" ] ; then - push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ ${EXT_DEPS_DEB}" - fi + create_deb_push_cmds ubuntu ;; CentOS) - FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease) - FACTER_ARCH=$(/usr/bin/facter architecture) - RPMS=$(find . -type f -iregex '.*/.*\.\(s\)?rpm' | grep -v vpp-ext-deps) - push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${RPMS}" - EXT_DEPS_RPM=$(find . -type f -iname 'vpp-ext-deps*.rpm') - if [ -n "$EXT_DEPS_RPM" ] ; then - push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${EXT_DEPS_RPM}" - fi + create_rpm_push_cmds ;; *) echo -e "\n$line\n* ERROR: Unsupported OS '$FACTER_OS'\n* PACKAGECLOUD PUSH FAILED!\n$line\n" @@ -86,10 +101,7 @@ if [ -f ~/.packagecloud ]; then else $push_cmd if [ -n "$push_ext_deps_cmd" ] ; then - # Don't fail script if vpp-ext-deps push fails. - set +e - $push_ext_deps_cmd - set -e + $push_ext_deps_cmd || true fi fi else diff --git a/jjb/scripts/setup_vpp_dpdk_dev_env.sh b/jjb/scripts/setup_vpp_dpdk_dev_env.sh deleted file mode 100644 index 2c55f166b..000000000 --- a/jjb/scripts/setup_vpp_dpdk_dev_env.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2020 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "---> jjb/scripts/setup_vpp_dpdk_dev_env.sh" - -set -e -o pipefail - -OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') -OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') - -function setup { - if [ -n "$REPO_NAME" ] ; then - echo "Installing vpp-ext-deps..." - REPO_URL="https://packagecloud.io/fdio/${STREAM}" - echo "REPO_URL: $REPO_URL" - INSTALL_URL="https://packagecloud.io/install/repositories/fdio/${STREAM}" - echo "INSTALL_URL: $INSTALL_URL" - # Setup by installing vpp-dev and vpp-lib - if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then - if [ "${STREAM}" != "master" ]; then - echo "stream '${STREAM}' is not master: replacing packagecloud apt sources list with stream specific list" - sudo apt-get -y remove vpp-ext-deps || true - sudo rm -f /etc/apt/sources.list.d/fdio_master.list - curl -s $INSTALL_URL/script.deb.sh | sudo bash - fi - sudo apt-get update -qq || true - local vpp_ext_deps_version="$(apt-cache show vpp-ext-deps | mawk '/Version/ {print $2}' | head -1)" - local vpp_ext_deps_arch="$(apt-cache show vpp-ext-deps | mawk '/Architecture/ {print $2}' | head -1)" - local vpp_ext_deps_pkg="/root/Downloads/vpp-ext-deps_${vpp_ext_deps_version}_${vpp_ext_deps_arch}.deb" - if [ -f "$vpp_ext_deps_pkg" ] ; then - echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg" - sudo dpkg -i $vpp_ext_deps_pkg - else - echo "Installing vpp-ext-deps from packagecloud.io" - local force_opts="--allow-downgrades --allow-remove-essential" - force_opts="$force_opts --allow-change-held-packages" - sudo apt-get -y $force_opts install vpp-ext-deps || true - fi - echo "Removing packagecloud.io repository references and running apt-get update" - sudo rm -f /etc/apt/sources.list.d/fdio_*.list - sudo apt-get update -qq || true - elif [ "${OS_ID,,}" == "centos" ] ; then - if [ "${STREAM}" != "master" ] ; then - echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list" - sudo yum -y erase vpp-ext-deps || true - sudo yum clean all || true - sudo rm -f /etc/yum.repos.d/fdio_master.repo - curl -s $INSTALL_URL/script.rpm.sh | sudo bash - fi - local vpp_ext_deps_version="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $2}')" - local vpp_ext_deps_pkg="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $1}')" - vpp_ext_deps_pkg="/root/Downloads/${vpp_ext_deps_pkg/./-${vpp_ext_deps_version}.}.rpm" - if [ -f "$vpp_ext_deps_pkg" ] ; then - echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg" - sudo yum -y localinstall $vpp_ext_deps_pkg || true - else - echo "Installing vpp-ext-deps from packagecloud.io" - sudo yum -y install vpp-ext-deps || true - fi - else - echo "ERROR: Unsupported OS '$OS_ID'!" - fi - else - echo "ERROR: REPO_NAME not found!" - fi -} - -setup diff --git a/jjb/scripts/setup_vpp_ext_deps.sh b/jjb/scripts/setup_vpp_ext_deps.sh new file mode 100755 index 000000000..1ca982d45 --- /dev/null +++ b/jjb/scripts/setup_vpp_ext_deps.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/setup_vpp_ext_deps.sh" + +set -e -o pipefail + +OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') + +echo "Installing vpp-ext-deps..." +REPO_URL="https://packagecloud.io/fdio/${STREAM}" +echo "REPO_URL: $REPO_URL" +INSTALL_URL="https://packagecloud.io/install/repositories/fdio/${STREAM}" +echo "INSTALL_URL: $INSTALL_URL" + +downloads_dir="/root/Downloads" + +# Setup by installing vpp-dev and vpp-lib +if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then + if [ "${STREAM}" != "master" ]; then + echo "stream '${STREAM}' is not master: replacing packagecloud apt sources list with stream specific list" + sudo apt-get -y remove vpp-ext-deps || true + sudo rm -f /etc/apt/sources.list.d/fdio_master.list + curl -s $INSTALL_URL/script.deb.sh | sudo bash || true + fi + sudo apt-get update -qq || true + vpp_ext_deps_version="$(apt-cache show vpp-ext-deps | mawk '/Version/ {print $2}' | head -1)" + vpp_ext_deps_arch="$(apt-cache show vpp-ext-deps | mawk '/Architecture/ {print $2}' | head -1)" + vpp_ext_deps_pkg="vpp-ext-deps_${vpp_ext_deps_version}_${vpp_ext_deps_arch}.deb" + if [ -f "$downloads_dir/$vpp_ext_deps_pkg" ] ; then + echo "Installing cached vpp-ext-deps pkg: $downloads_dir/$vpp_ext_deps_pkg" + sudo dpkg -i "$downloads_dir/$vpp_ext_deps_pkg" || true + else + echo "Installing vpp-ext-deps from packagecloud.io" + force_opts="--allow-downgrades --allow-remove-essential --allow-change-held-packages" + sudo apt-get -y $force_opts install vpp-ext-deps || true + fi + echo "Removing packagecloud.io repository references and running apt-get update" + sudo rm -f /etc/apt/sources.list.d/fdio_*.list + sudo apt-get update -qq || true +#TODO: Remove centos when VPP 21.06 is no longer supported +elif [ "${OS_ID,,}" == "centos" ] ; then + if [ "${STREAM}" != "master" ] ; then + echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list" + sudo yum -y erase vpp-ext-deps || true + sudo yum clean all || true + sudo rm -f /etc/yum.repos.d/fdio_master.repo + curl -s $INSTALL_URL/script.rpm.sh | sudo bash || true + fi + vpp_ext_deps_version="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $2}')" + vpp_ext_deps_pkg="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $1}')" + vpp_ext_deps_pkg="${vpp_ext_deps_pkg/./-${vpp_ext_deps_version}.}.rpm" + if [ -f "$vpp_ext_deps_pkg" ] ; then + echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg" + sudo yum -y localinstall "$downloads_dir/$vpp_ext_deps_pkg" || true + else + echo "Installing vpp-ext-deps from packagecloud.io" + sudo yum -y install vpp-ext-deps || true + fi +else + echo "ERROR: Unsupported OS '$OS_ID'!" +fi diff --git a/jjb/scripts/setup_vpp_ubuntu_docker_test.sh b/jjb/scripts/setup_vpp_ubuntu_docker_test.sh index 8ed0acaee..548ac5660 100644..100755 --- a/jjb/scripts/setup_vpp_ubuntu_docker_test.sh +++ b/jjb/scripts/setup_vpp_ubuntu_docker_test.sh @@ -1,7 +1,7 @@ #!/bin/bash ############################################################################## -# Copyright (c) 2018 The Linux Foundation and others. +# Copyright (c) 2021 The Linux Foundation and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Eclipse Public License v1.0 @@ -30,17 +30,3 @@ if [ -n ${DOCKER_TEST} ] ; then sudo mount -o remount /dev/shm -o size=${MEM} || true echo "/dev/shm remounted with size='${MEM}'" fi - -# This will remove any previously installed external packages -# for old branch builds -if [ "${GERRIT_BRANCH}" != "master" ]; then - echo "Removing $OS_ID-$OS_VERSION_ID package 'vpp-ext-deps'" - if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then - sudo apt-get -y remove vpp-ext-deps || true - elif [ "${OS_ID,,}" == "centos" ]; then - sudo yum -y erase vpp-ext-deps || true - sudo yum clean all || true - else - echo "ERROR: Unsupported OS '$OS_ID'!" - fi -fi diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index f5164aa80..be473d872 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -471,7 +471,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/build.sh @@ -535,7 +535,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/debug-build.sh @@ -602,7 +602,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/build.sh @@ -665,7 +665,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/gcc-build.sh @@ -743,7 +743,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/csit-device.sh @@ -820,7 +820,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/csit-device.sh @@ -905,7 +905,7 @@ - ../scripts/setup_vpp_ubuntu_docker_test.sh - shell: !include-raw-escape: - - ../scripts/setup_vpp_dpdk_dev_env.sh + - ../scripts/setup_vpp_ext_deps.sh - shell: !include-raw-escape: - ../scripts/vpp/csit-perf.sh |