summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2021-11-16 06:58:04 +0000
committerGerrit Code Review <gerrit@fd.io>2021-11-16 06:58:04 +0000
commit6a508a4a67b4bda6011340396a64415a20b759fe (patch)
tree01886a1f71c4e97e988077f1c2be767cadaa273e
parentb41db525b39f0d7d31f3381f589bae0f7bcb0478 (diff)
parente62c4bc9708a906d2dcd504cdcca5964b232d3b3 (diff)
Merge "Fix upload of vpp-ext-deps cached in executor"
-rwxr-xr-x[-rw-r--r--]jjb/scripts/packagecloud_push.sh64
-rw-r--r--jjb/scripts/setup_vpp_dpdk_dev_env.sh80
-rwxr-xr-xjjb/scripts/setup_vpp_ext_deps.sh75
-rwxr-xr-x[-rw-r--r--]jjb/scripts/setup_vpp_ubuntu_docker_test.sh16
-rw-r--r--jjb/vpp/vpp.yaml14
5 files changed, 121 insertions, 128 deletions
diff --git a/jjb/scripts/packagecloud_push.sh b/jjb/scripts/packagecloud_push.sh
index a4ae9b166..80226aa04 100644..100755
--- a/jjb/scripts/packagecloud_push.sh
+++ b/jjb/scripts/packagecloud_push.sh
@@ -40,38 +40,53 @@ sleep 10
FACTER_OS=$(/usr/bin/facter operatingsystem)
push_cmd=""
push_ext_deps_cmd=""
+ext_deps_pkg=""
+downloads_dir="/root/Downloads"
+
+create_deb_push_cmds()
+{
+ local distro="$1"
+
+ if [ "$distro" = "debian" ] || [ "$distro" = "ubuntu" ] ; then
+ FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
+ DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps | xargs || true)
+ push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/${distro}/${FACTER_LSBNAME}/main/ ${DEBS}"
+ ext_deps_ver="$(dpkg -l vpp-ext-deps | mawk '/vpp-ext-deps/{print $3}' || true)"
+ ext_deps_pkg="$(find . -type f -iname 'vpp-ext-deps*.deb' | grep $ext_deps_ver || find $downloads_dir -type f -iname 'vpp-ext-deps*.deb' | grep $ext_deps_ver || true)"
+ if [ -n "$ext_deps_pkg}" ] ; then
+ push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/${distro}/${FACTER_LSBNAME}/main/ ${ext_deps_pkg}"
+ fi
+ else
+ echo "ERROR: Unknown distro: '$distro'"
+ return 1
+ fi
+}
+
+create_rpm_push_cmds()
+{
+ FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease)
+ FACTER_ARCH=$(/usr/bin/facter architecture)
+ RPMS=$(find . -type f -iregex '.*/.*\.\(s\)?rpm' | grep -v vpp-ext-deps | xargs || true)
+ push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${RPMS}"
+ ext_deps_ver="$(dnf list vpp-ext-deps | mawk '/vpp-ext-deps/{print $2}' || true)"
+ ext_deps_pkg="$(find . -type f -iname 'vpp-ext-deps*.rpm' | grep $ext_deps_ver || find $downloads_dir -type f -iname 'vpp-ext-deps*.rpm' | grep $ext_deps_ver || true)"
+ if [ -n "$ext_deps_pkg" ] ; then
+ push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${ext_deps_pkg}"
+ fi
+}
# PCIO_CO and SILO are Jenkins Global Environment variables defined in
# .../ci-management/jenkins-config/global-vars-*.sh
if [ -f ~/.packagecloud ]; then
case "$FACTER_OS" in
Debian)
- FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
- DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps)
- push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/debian/${FACTER_LSBNAME}/main/ ${DEBS}"
- EXT_DEPS_DEB=$(find . -type f -iname 'vpp-ext-deps*.deb')
- if [ -n "$EXT_DEPS_DEB" ] ; then
- push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/debian/${FACTER_LSBNAME}/main/ ${EXT_DEPS_DEB}"
- fi
+ create_deb_push_cmds debian
;;
Ubuntu)
- FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
- DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps)
- push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ ${DEBS}"
- EXT_DEPS_DEB=$(find . -type f -iname 'vpp-ext-deps*.deb')
- if [ -n "$EXT_DEPS_DEB" ] ; then
- push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ ${EXT_DEPS_DEB}"
- fi
+ create_deb_push_cmds ubuntu
;;
CentOS)
- FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease)
- FACTER_ARCH=$(/usr/bin/facter architecture)
- RPMS=$(find . -type f -iregex '.*/.*\.\(s\)?rpm' | grep -v vpp-ext-deps)
- push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${RPMS}"
- EXT_DEPS_RPM=$(find . -type f -iname 'vpp-ext-deps*.rpm')
- if [ -n "$EXT_DEPS_RPM" ] ; then
- push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${EXT_DEPS_RPM}"
- fi
+ create_rpm_push_cmds
;;
*)
echo -e "\n$line\n* ERROR: Unsupported OS '$FACTER_OS'\n* PACKAGECLOUD PUSH FAILED!\n$line\n"
@@ -86,10 +101,7 @@ if [ -f ~/.packagecloud ]; then
else
$push_cmd
if [ -n "$push_ext_deps_cmd" ] ; then
- # Don't fail script if vpp-ext-deps push fails.
- set +e
- $push_ext_deps_cmd
- set -e
+ $push_ext_deps_cmd || true
fi
fi
else
diff --git a/jjb/scripts/setup_vpp_dpdk_dev_env.sh b/jjb/scripts/setup_vpp_dpdk_dev_env.sh
deleted file mode 100644
index 2c55f166b..000000000
--- a/jjb/scripts/setup_vpp_dpdk_dev_env.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2020 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "---> jjb/scripts/setup_vpp_dpdk_dev_env.sh"
-
-set -e -o pipefail
-
-OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-
-function setup {
- if [ -n "$REPO_NAME" ] ; then
- echo "Installing vpp-ext-deps..."
- REPO_URL="https://packagecloud.io/fdio/${STREAM}"
- echo "REPO_URL: $REPO_URL"
- INSTALL_URL="https://packagecloud.io/install/repositories/fdio/${STREAM}"
- echo "INSTALL_URL: $INSTALL_URL"
- # Setup by installing vpp-dev and vpp-lib
- if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then
- if [ "${STREAM}" != "master" ]; then
- echo "stream '${STREAM}' is not master: replacing packagecloud apt sources list with stream specific list"
- sudo apt-get -y remove vpp-ext-deps || true
- sudo rm -f /etc/apt/sources.list.d/fdio_master.list
- curl -s $INSTALL_URL/script.deb.sh | sudo bash
- fi
- sudo apt-get update -qq || true
- local vpp_ext_deps_version="$(apt-cache show vpp-ext-deps | mawk '/Version/ {print $2}' | head -1)"
- local vpp_ext_deps_arch="$(apt-cache show vpp-ext-deps | mawk '/Architecture/ {print $2}' | head -1)"
- local vpp_ext_deps_pkg="/root/Downloads/vpp-ext-deps_${vpp_ext_deps_version}_${vpp_ext_deps_arch}.deb"
- if [ -f "$vpp_ext_deps_pkg" ] ; then
- echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg"
- sudo dpkg -i $vpp_ext_deps_pkg
- else
- echo "Installing vpp-ext-deps from packagecloud.io"
- local force_opts="--allow-downgrades --allow-remove-essential"
- force_opts="$force_opts --allow-change-held-packages"
- sudo apt-get -y $force_opts install vpp-ext-deps || true
- fi
- echo "Removing packagecloud.io repository references and running apt-get update"
- sudo rm -f /etc/apt/sources.list.d/fdio_*.list
- sudo apt-get update -qq || true
- elif [ "${OS_ID,,}" == "centos" ] ; then
- if [ "${STREAM}" != "master" ] ; then
- echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list"
- sudo yum -y erase vpp-ext-deps || true
- sudo yum clean all || true
- sudo rm -f /etc/yum.repos.d/fdio_master.repo
- curl -s $INSTALL_URL/script.rpm.sh | sudo bash
- fi
- local vpp_ext_deps_version="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $2}')"
- local vpp_ext_deps_pkg="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $1}')"
- vpp_ext_deps_pkg="/root/Downloads/${vpp_ext_deps_pkg/./-${vpp_ext_deps_version}.}.rpm"
- if [ -f "$vpp_ext_deps_pkg" ] ; then
- echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg"
- sudo yum -y localinstall $vpp_ext_deps_pkg || true
- else
- echo "Installing vpp-ext-deps from packagecloud.io"
- sudo yum -y install vpp-ext-deps || true
- fi
- else
- echo "ERROR: Unsupported OS '$OS_ID'!"
- fi
- else
- echo "ERROR: REPO_NAME not found!"
- fi
-}
-
-setup
diff --git a/jjb/scripts/setup_vpp_ext_deps.sh b/jjb/scripts/setup_vpp_ext_deps.sh
new file mode 100755
index 000000000..1ca982d45
--- /dev/null
+++ b/jjb/scripts/setup_vpp_ext_deps.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/setup_vpp_ext_deps.sh"
+
+set -e -o pipefail
+
+OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+
+echo "Installing vpp-ext-deps..."
+REPO_URL="https://packagecloud.io/fdio/${STREAM}"
+echo "REPO_URL: $REPO_URL"
+INSTALL_URL="https://packagecloud.io/install/repositories/fdio/${STREAM}"
+echo "INSTALL_URL: $INSTALL_URL"
+
+downloads_dir="/root/Downloads"
+
+# Setup by installing vpp-dev and vpp-lib
+if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then
+ if [ "${STREAM}" != "master" ]; then
+ echo "stream '${STREAM}' is not master: replacing packagecloud apt sources list with stream specific list"
+ sudo apt-get -y remove vpp-ext-deps || true
+ sudo rm -f /etc/apt/sources.list.d/fdio_master.list
+ curl -s $INSTALL_URL/script.deb.sh | sudo bash || true
+ fi
+ sudo apt-get update -qq || true
+ vpp_ext_deps_version="$(apt-cache show vpp-ext-deps | mawk '/Version/ {print $2}' | head -1)"
+ vpp_ext_deps_arch="$(apt-cache show vpp-ext-deps | mawk '/Architecture/ {print $2}' | head -1)"
+ vpp_ext_deps_pkg="vpp-ext-deps_${vpp_ext_deps_version}_${vpp_ext_deps_arch}.deb"
+ if [ -f "$downloads_dir/$vpp_ext_deps_pkg" ] ; then
+ echo "Installing cached vpp-ext-deps pkg: $downloads_dir/$vpp_ext_deps_pkg"
+ sudo dpkg -i "$downloads_dir/$vpp_ext_deps_pkg" || true
+ else
+ echo "Installing vpp-ext-deps from packagecloud.io"
+ force_opts="--allow-downgrades --allow-remove-essential --allow-change-held-packages"
+ sudo apt-get -y $force_opts install vpp-ext-deps || true
+ fi
+ echo "Removing packagecloud.io repository references and running apt-get update"
+ sudo rm -f /etc/apt/sources.list.d/fdio_*.list
+ sudo apt-get update -qq || true
+#TODO: Remove centos when VPP 21.06 is no longer supported
+elif [ "${OS_ID,,}" == "centos" ] ; then
+ if [ "${STREAM}" != "master" ] ; then
+ echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list"
+ sudo yum -y erase vpp-ext-deps || true
+ sudo yum clean all || true
+ sudo rm -f /etc/yum.repos.d/fdio_master.repo
+ curl -s $INSTALL_URL/script.rpm.sh | sudo bash || true
+ fi
+ vpp_ext_deps_version="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $2}')"
+ vpp_ext_deps_pkg="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $1}')"
+ vpp_ext_deps_pkg="${vpp_ext_deps_pkg/./-${vpp_ext_deps_version}.}.rpm"
+ if [ -f "$vpp_ext_deps_pkg" ] ; then
+ echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg"
+ sudo yum -y localinstall "$downloads_dir/$vpp_ext_deps_pkg" || true
+ else
+ echo "Installing vpp-ext-deps from packagecloud.io"
+ sudo yum -y install vpp-ext-deps || true
+ fi
+else
+ echo "ERROR: Unsupported OS '$OS_ID'!"
+fi
diff --git a/jjb/scripts/setup_vpp_ubuntu_docker_test.sh b/jjb/scripts/setup_vpp_ubuntu_docker_test.sh
index 8ed0acaee..548ac5660 100644..100755
--- a/jjb/scripts/setup_vpp_ubuntu_docker_test.sh
+++ b/jjb/scripts/setup_vpp_ubuntu_docker_test.sh
@@ -1,7 +1,7 @@
#!/bin/bash
##############################################################################
-# Copyright (c) 2018 The Linux Foundation and others.
+# Copyright (c) 2021 The Linux Foundation and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
@@ -30,17 +30,3 @@ if [ -n ${DOCKER_TEST} ] ; then
sudo mount -o remount /dev/shm -o size=${MEM} || true
echo "/dev/shm remounted with size='${MEM}'"
fi
-
-# This will remove any previously installed external packages
-# for old branch builds
-if [ "${GERRIT_BRANCH}" != "master" ]; then
- echo "Removing $OS_ID-$OS_VERSION_ID package 'vpp-ext-deps'"
- if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then
- sudo apt-get -y remove vpp-ext-deps || true
- elif [ "${OS_ID,,}" == "centos" ]; then
- sudo yum -y erase vpp-ext-deps || true
- sudo yum clean all || true
- else
- echo "ERROR: Unsupported OS '$OS_ID'!"
- fi
-fi
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 880fcb215..ed470580d 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -475,7 +475,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/build.sh
@@ -539,7 +539,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/debug-build.sh
@@ -606,7 +606,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/build.sh
@@ -669,7 +669,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/gcc-build.sh
@@ -747,7 +747,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/csit-device.sh
@@ -824,7 +824,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/csit-device.sh
@@ -909,7 +909,7 @@
- ../scripts/setup_vpp_ubuntu_docker_test.sh
- shell:
!include-raw-escape:
- - ../scripts/setup_vpp_dpdk_dev_env.sh
+ - ../scripts/setup_vpp_ext_deps.sh
- shell:
!include-raw-escape:
- ../scripts/vpp/csit-perf.sh