diff options
| -rw-r--r-- | VPP_REPO_URL | 2 | ||||
| -rw-r--r-- | VPP_REPO_URL_CENTOS | 1 | ||||
| -rw-r--r-- | VPP_REPO_URL_UBUNTU | 1 | ||||
| -rw-r--r-- | bootstrap-vpp-verify-nightly.sh | 5 | ||||
| -rw-r--r-- | bootstrap-vpp-verify-semiweekly.sh | 6 | ||||
| -rw-r--r-- | bootstrap-vpp-verify-weekly.sh | 10 | ||||
| -rwxr-xr-x | bootstrap.sh | 10 | ||||
| -rw-r--r-- | resources/libraries/bash/function/artifacts.sh | 155 | ||||
| -rw-r--r-- | resources/libraries/bash/function/gather.sh | 26 | ||||
| -rwxr-xr-x | resources/tools/scripts/download_install_vpp_pkgs.sh | 179 |
10 files changed, 185 insertions, 210 deletions
diff --git a/VPP_REPO_URL b/VPP_REPO_URL index 56de6b69e9..c41c16cf4f 100644 --- a/VPP_REPO_URL +++ b/VPP_REPO_URL @@ -1 +1 @@ -https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/ +https://packagecloud.io/install/repositories/fdio/master diff --git a/VPP_REPO_URL_CENTOS b/VPP_REPO_URL_CENTOS deleted file mode 100644 index bc11e8d2a1..0000000000 --- a/VPP_REPO_URL_CENTOS +++ /dev/null @@ -1 +0,0 @@ -https://nexus.fd.io/content/repositories/fd.io.master.centos7/io/fd/vpp/ diff --git a/VPP_REPO_URL_UBUNTU b/VPP_REPO_URL_UBUNTU deleted file mode 100644 index 56de6b69e9..0000000000 --- a/VPP_REPO_URL_UBUNTU +++ /dev/null @@ -1 +0,0 @@ -https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/ diff --git a/bootstrap-vpp-verify-nightly.sh b/bootstrap-vpp-verify-nightly.sh index 38f4eeab9c..cc2b36401b 100644 --- a/bootstrap-vpp-verify-nightly.sh +++ b/bootstrap-vpp-verify-nightly.sh @@ -55,7 +55,10 @@ else # Download the latest VPP build install packages rm -f *.${PACKAGE} echo Downloading VPP packages... - bash ${SCRIPT_DIR}/resources/tools/download_install_vpp_pkgs.sh --skip-install + source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh" + download_artifacts + # Need to revert -euo as the rest of script is not optimized for this. + set +euo pipefail fi # Take vpp package and get the vpp version diff --git a/bootstrap-vpp-verify-semiweekly.sh b/bootstrap-vpp-verify-semiweekly.sh index 14453e3003..ecc7114113 100644 --- a/bootstrap-vpp-verify-semiweekly.sh +++ b/bootstrap-vpp-verify-semiweekly.sh @@ -65,8 +65,10 @@ if [ "${#}" -ne "0" ]; then echo ${arr[0]} else # Download the latest VPP build install packages - bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \ - --skip-install + source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh" + download_artifacts + # Need to revert -euo as the rest of script is not optimized for this. + set +euo pipefail fi VIRL_DIR_LOC="/tmp/" diff --git a/bootstrap-vpp-verify-weekly.sh b/bootstrap-vpp-verify-weekly.sh index 2ee9a9792c..d96e2d1f37 100644 --- a/bootstrap-vpp-verify-weekly.sh +++ b/bootstrap-vpp-verify-weekly.sh @@ -59,10 +59,12 @@ if [ "${#}" -ne "0" ]; then echo ${arr[0]} else # Download the specific VPP build install packages - DPDK_STABLE_VER=$(cat ${SCRIPT_DIR}/DPDK_STABLE_VER) - VPP_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO}) - bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \ - --skip-install --vpp ${VPP_STABLE_VER} --dkms ${DPDK_STABLE_VER} + DKMS_VERSION=$(< ${SCRIPT_DIR}/DPDK_STABLE_VER) + VPP_VERSION=$(< ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO}) + source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh" + download_artifacts + # Need to revert -euo as the rest of script is not optimized for this. + set +euo pipefail fi VIRL_DIR_LOC="/tmp/" diff --git a/bootstrap.sh b/bootstrap.sh index 02aa7133e5..015858b1fd 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -44,10 +44,12 @@ if [ "${#}" -ne "0" ]; then echo ${arr[0]} SKIP_PATCH="skip_patchORskip_vpp_patch" else - DPDK_STABLE_VER=$(cat ${SCRIPT_DIR}/DPDK_STABLE_VER) - VPP_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO}) - bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \ - --skip-install --vpp ${VPP_STABLE_VER} --dkms ${DPDK_STABLE_VER} + DKMS_VERSION=$(< ${SCRIPT_DIR}/DPDK_STABLE_VER) + VPP_VERSION=$(< ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO}) + source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh" + download_artifacts + # Need to revert -euo as the rest of script is not optimized for this. + set +euo pipefail fi VIRL_DIR_LOC="/tmp/" diff --git a/resources/libraries/bash/function/artifacts.sh b/resources/libraries/bash/function/artifacts.sh new file mode 100644 index 0000000000..945bdfe824 --- /dev/null +++ b/resources/libraries/bash/function/artifacts.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +function download_artifacts () { + # Get and/or install VPP artifacts from packagecloud.io. + # + # Variables set: + # - REPO_URL - FD.io Packagecloud repository. + + set -exuo pipefail + + os_id=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') || { + die "Get OS release failed." + } + + repo_url_path="./VPP_REPO_URL" + if [ -e "${repo_url_path}" ]; then + REPO_URL="$(<${repo_url_path})" || { + die "Read repo URL from ${repo_url_path} failed." + } + else + REPO_URL="https://packagecloud.io/install/repositories/fdio/master" + fi + + if [ "${os_id}" == "ubuntu" ]; then + download_ubuntu_artifacts || die + elif [ "${os_id}" == "centos" ]; then + download_centos_artifacts || die + elif [ "${os_id}" == "opensuse" ]; then + download_opensuse_artifacts || die + else + die "${os_id} is not yet supported." + fi +} + +function download_ubuntu_artifacts () { + # Get and/or install Ubuntu VPP artifacts from packagecloud.io. + # + # Variables read: + # - REPO_URL - FD.io Packagecloud repository. + # - VPP_VERSION - VPP version. + # - DKMS_VERSION - DKMS version. + # - INSTALL - If install packages or download only. Default: download + + set -exuo pipefail + + curl -s "${REPO_URL}"/script.deb.sh | sudo bash || { + die "Packagecloud FD.io repo fetch failed." + } + # If version is set we will add suffix. + artifacts=() + vpp=(vpp vpp-dbg vpp-dev vpp-lib vpp-plugins) + if [ -z "${VPP_VERSION-}" ]; then + artifacts+=(${vpp[@]}) + else + artifacts+=(${vpp[@]/%/=${VPP_VERSION-}}) + fi + dkms=(vpp-dpdk-dkms) + if [ -z "${DKMS_VERSION-}" ]; then + artifacts+=(${dkms[@]}) + else + artifacts+=(${dkms[@]/%/=${DKMS_VERSION-}}) + fi + + if [ "${INSTALL:-false}" = true ]; then + sudo apt-get -y install "${artifacts[@]}" || { + die "Install VPP artifacts failed." + } + else + apt-get -y download "${artifacts[@]}" || { + die "Download VPP artifacts failed." + } + fi +} + +function download_centos_artifacts () { + # Get and/or install CentOS VPP artifacts from packagecloud.io. + # + # Variables read: + # - REPO_URL - FD.io Packagecloud repository. + # - VPP_VERSION - VPP version. + # - INSTALL - If install packages or download only. Default: download + + set -exuo pipefail + + curl -s "${REPO_URL}"/script.rpm.sh | sudo bash || { + die "Packagecloud FD.io repo fetch failed." + } + # If version is set we will add suffix. + artifacts=() + vpp=(vpp vpp-selinux-policy vpp-devel vpp-lib vpp-plugins) + if [ -z "${VPP_VERSION-}" ]; then + artifacts+=(${vpp[@]}) + else + artifacts+=(${vpp[@]/%/-${VPP_VERSION-}}) + fi + + if [ "${INSTALL:-false}" = true ]; then + sudo yum -y install "${artifacts[@]}" || { + die "Install VPP artifact failed." + } + else + sudo yum -y install --downloadonly --downloaddir=. "${artifacts[@]}" || { + die "Download VPP artifacts failed." + } + fi +} + +function download_opensuse_artifacts () { + # Get and/or install OpenSuSE VPP artifacts from packagecloud.io. + # + # Variables read: + # - REPO_URL - FD.io Packagecloud repository. + # - VPP_VERSION - VPP version. + # - INSTALL - If install packages or download only. Default: download + + set -exuo pipefail + + curl -s "${REPO_URL}"/script.rpm.sh | sudo bash || { + die "Packagecloud FD.io repo fetch failed." + } + # If version is set we will add suffix. + artifacts=() + vpp=(vpp vpp-devel vpp-lib vpp-plugins libvpp0) + if [ -z "${VPP_VERSION-}" ]; then + artifacts+=(${vpp[@]}) + else + artifacts+=(${vpp[@]/%/-${VPP_VERSION-}}) + fi + + if [ "${INSTALL:-false}" = true ]; then + sudo yum -y install "${artifacts[@]}" || { + die "Install VPP artifact failed." + } + else + sudo yum -y install --downloadonly --downloaddir=. "${artifacts[@]}" || { + die "Download VPP artifacts failed." + } + fi +} + diff --git a/resources/libraries/bash/function/gather.sh b/resources/libraries/bash/function/gather.sh index f490c80110..e030652494 100644 --- a/resources/libraries/bash/function/gather.sh +++ b/resources/libraries/bash/function/gather.sh @@ -246,6 +246,7 @@ function gather_vpp () { set -exuo pipefail # Variables read: + # - BASH_FUNCTION_DIR - Bash directory with functions. # - TEST_CODE - The test selection string from environment or argument. # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from. # - CSIT_DIR - Path to existing root of local CSIT git repository. @@ -267,32 +268,23 @@ function gather_vpp () { case "${TEST_CODE}" in # Not csit-vpp as this code is re-used by ligato gathering. "csit-"*) - install_script="${CSIT_DIR}/resources/tools/scripts/" - install_script+="download_install_vpp_pkgs.sh" # Use downloaded packages with specific version if [[ "${TEST_CODE}" == *"daily"* ]] || \ [[ "${TEST_CODE}" == *"weekly"* ]] || \ [[ "${TEST_CODE}" == *"timed"* ]]; then - echo "Downloading latest VPP packages from NEXUS..." - # TODO: Can we source? - bash "${install_script}" --skip-install || { - die "Failed to get VPP packages!" - } + warn "Downloading latest VPP packages from Packagecloud." else - echo "Downloading VPP packages of specific version from NEXUS." - dpdk_stable_ver="$(cat "${CSIT_DIR}/DPDK_STABLE_VER")" || { - die "Cat failed." - } - vpp_stable_ver="$(cat "${CSIT_DIR}/VPP_STABLE_VER_UBUNTU")" || { - die "Cat failed." + warn "Downloading stable VPP packages from Packagecloud." + DKMS_VERSION="$(<"${CSIT_DIR}/DPDK_STABLE_VER")" || { + die "Read DPDK stable version failed." } - install_args=("--skip-install" "--vpp" "${vpp_stable_ver}") - install_args+=("--dkms" "${dpdk_stable_ver}") - bash "${install_script}" "${install_args[@]}" || { - die "Failed to get VPP packages!" + VPP_VERSION="$(<"${CSIT_DIR}/VPP_STABLE_VER_UBUNTU")" || { + die "Read VPP stable version failed." } fi + source "${BASH_FUNCTION_DIR}/artifacts.sh" || die "Source failed." + download_artifacts || die ;; "vpp-csit-"*) # Use local built packages. diff --git a/resources/tools/scripts/download_install_vpp_pkgs.sh b/resources/tools/scripts/download_install_vpp_pkgs.sh deleted file mode 100755 index ce7530af83..0000000000 --- a/resources/tools/scripts/download_install_vpp_pkgs.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') -OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') - -NEXUSPROXY="https://nexus.fd.io" - -function artifacts { - if [ "$OS_ID" == "ubuntu" ]; then - VPP_REPO_URL_PATH="./VPP_REPO_URL_UBUNTU" - if [ -e "$VPP_REPO_URL_PATH" ]; then - VPP_REPO_URL=$(cat $VPP_REPO_URL_PATH) - REPO_NAME=$(echo ${VPP_REPO_URL#https://nexus.fd.io/content/repositories/}) - REPO_NAME=$(echo ${REPO_NAME%io/fd/vpp/}) - else - OS_VERSION_CODENAME=$(grep '^VERSION_CODENAME=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') - REPO_NAME="fd.io.master.ubuntu.${OS_VERSION_CODENAME}.main" - fi - REPO_URL="${NEXUSPROXY}/content/repositories/${REPO_NAME}" - - echo "deb ${REPO_URL} ./" | sudo tee /etc/apt/sources.list.d/99fd.io.list - sudo apt-get -y update \ - -o Dir::Etc::sourcelist="sources.list.d/99fd.io.list" \ - -o Acquire::AllowInsecureRepositories=true \ - -o Dir::Etc::sourceparts="-" \ - -o APT::Get::AllowUnauthenticated=true \ - -o APT::Get::List-Cleanup="0" - - # If version is set we will add suffix - VPP=(vpp vpp-dbg vpp-dev vpp-lib vpp-plugins) - DKMS=(vpp-dpdk-dkms) - if [ -z "${VPP_VERSION}" ]; then - ARTIFACTS+=(${VPP[@]/%/${VPP_VERSION}}) - ARTIFACTS+=(${DKMS[@]/%/${DKMS_VERSION}}) - else - ARTIFACTS+=(${VPP[@]/%/=${VPP_VERSION}}) - ARTIFACTS+=(${DKMS[@]/%/=${DKMS_VERSION}}) - fi - - if [ "$INSTALL" = true ]; then - echo Installing VPP - sudo apt-get -y install ${ARTIFACTS[@]} \ - -o Acquire::AllowInsecureRepositories=true \ - -o APT::Get::AllowUnauthenticated=true - else - echo Downloading VPP - apt-get -y download ${ARTIFACTS[@]} \ - -o Acquire::AllowInsecureRepositories=true \ - -o APT::Get::AllowUnauthenticated=true - fi - - elif [ "$OS_ID" == "centos" ]; then - VPP_REPO_URL_PATH="./VPP_REPO_URL_CENTOS" - if [ -e "$VPP_REPO_URL_PATH" ]; then - VPP_REPO_URL=$(cat $VPP_REPO_URL_PATH) - REPO_NAME=$(echo ${VPP_REPO_URL#https://nexus.fd.io/content/repositories/}) - REPO_NAME=$(echo ${REPO_NAME%/io/fd/vpp/}) - else - REPO_NAME="fd.io.master.centos7" - fi - REPO_URL="${NEXUSPROXY}/content/repositories/${REPO_NAME}" - - sudo cat << EOF > fdio-master.repo -[fdio-master] -name=fd.io master branch latest merge -baseurl=${REPO_URL} -enabled=1 -gpgcheck=0 -EOF - sudo mv fdio-master.repo /etc/yum.repos.d/fdio-master.repo - - # If version is set we will add suffix - VPP=(vpp vpp-selinux-policy vpp-devel vpp-lib vpp-plugins) - if [ -z "${VPP_VERSION}" ]; then - ARTIFACTS+=(${VPP[@]/%/${VPP_VERSION}}) - else - ARTIFACTS+=(${VPP[@]/%/-${VPP_VERSION}}) - fi - - if [ "$INSTALL" = true ]; then - echo Installing VPP - sudo yum -y install ${ARTIFACTS[@]} - else - echo Downloading VPP - sudo yum -y install --downloadonly --downloaddir=. ${ARTIFACTS[@]} - fi - elif [ "$OS_ID" == "opensuse" ]; then - VPP_REPO_URL_PATH="./VPP_REPO_URL_OPENSUSE" - if [ -e "$VPP_REPO_URL_PATH" ]; then - VPP_REPO_URL=$(cat $VPP_REPO_URL_PATH) - REPO_NAME=$(echo ${VPP_REPO_URL#https://nexus.fd.io/content/repositories/}) - REPO_NAME=$(echo ${REPO_NAME%/io/fd/vpp/}) - else - REPO_NAME='fd.io.master.opensuse' - fi - REPO_URL="${NEXUSPROXY}/content/repositories/${REPO_NAME}" - - sudo cat << EOF > fdio-master.repo -[fdio-master] -name=fd.io master branch latest merge -baseurl=${REPO_URL} -enabled=1 -gpgcheck=0 -EOF - sudo mv fdio-master.repo /etc/yum/repos.d/fdio-master.repo - - # If version is set we will add suffix - VPP=(vpp vpp-devel vpp-lib vpp-plugins libvpp0) - if [ -z "${VPP_VERSION}" ]; then - ARTIFACTS+=(${VPP[@]/%/${VPP_VERSION}}) - else - ARTIFACTS+=(${VPP[@]/%/-${VPP_VERSION}}) - fi - - if [ "$INSTALL" = true ]; then - echo Installing VPP - sudo yum -y install ${ARTIFACTS[@]} - else - echo Downloading VPP - sudo yum -y install --downloadonly --downloaddir=. ${ARTIFACTS[@]} - fi - else - echo "$OS_ID is not yet supported." - exit 1 - fi -} - -function display_help () { - echo "Usage: $0 [--skip-install] [--vpp <version>] [--dkms <version>]" -} - -# Whether to install artifacts or not -INSTALL=true -# VPP version (default empty = latest) -VPP_VERSION="" -# DKMS version (default empty = latest) -DKMS_VERSION="" - -while : -do - case "$1" in - -h | --help) - display_help - exit 0 - ;; - -s | --skip-install) - INSTALL=false - shift 1 - ;; - -v | --vpp) - VPP_VERSION="$2" - shift 2 - ;; - -d | --dkms) - DKMS_VERSION="$2" - shift 2 - ;; - *) - break - ;; - esac -done - -artifacts "${INSTALL}" "${VPP_VERSION}" "${DKMS_VERSION}" |