CSIT-1282 Migrate from Nexus.fd.io to packagecloud.io

Change-Id: Ib4e72dc8e3c0fedfcd702f4f97bc4f26cbc642e9
Signed-off-by: Peter Mikus <pmikus@cisco.com>

10 files changed, 185 insertions, 210 deletions

diff --git a/VPP_REPO_URL b/VPP_REPO_URL
index 56de6b69e9..c41c16cf4f 100644
--- a/VPP_REPO_URL
+++ b/VPP_REPO_URL
@@ -1 +1 @@
-https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/
+https://packagecloud.io/install/repositories/fdio/master
diff --git a/VPP_REPO_URL_CENTOS b/VPP_REPO_URL_CENTOS
deleted file mode 100644
index bc11e8d2a1..0000000000
--- a/VPP_REPO_URL_CENTOS
+++ /dev/null
@@ -1 +0,0 @@
-https://nexus.fd.io/content/repositories/fd.io.master.centos7/io/fd/vpp/
diff --git a/VPP_REPO_URL_UBUNTU b/VPP_REPO_URL_UBUNTU
deleted file mode 100644
index 56de6b69e9..0000000000
--- a/VPP_REPO_URL_UBUNTU
+++ /dev/null
@@ -1 +0,0 @@
-https://nexus.fd.io/content/repositories/fd.io.master.ubuntu.xenial.main/io/fd/vpp/
diff --git a/bootstrap-vpp-verify-nightly.sh b/bootstrap-vpp-verify-nightly.sh
index 38f4eeab9c..cc2b36401b 100644
--- a/bootstrap-vpp-verify-nightly.sh
+++ b/bootstrap-vpp-verify-nightly.sh
@@ -55,7 +55,10 @@ else
     # Download the latest VPP build install packages
     rm -f *.${PACKAGE}
     echo Downloading VPP packages...
-    bash ${SCRIPT_DIR}/resources/tools/download_install_vpp_pkgs.sh --skip-install
+    source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh"
+    download_artifacts
+    # Need to revert -euo as the rest of script is not optimized for this.
+    set +euo pipefail
 fi
 
 # Take vpp package and get the vpp version
diff --git a/bootstrap-vpp-verify-semiweekly.sh b/bootstrap-vpp-verify-semiweekly.sh
index 14453e3003..ecc7114113 100644
--- a/bootstrap-vpp-verify-semiweekly.sh
+++ b/bootstrap-vpp-verify-semiweekly.sh
@@ -65,8 +65,10 @@ if [ "${#}" -ne "0" ]; then
     echo ${arr[0]}
 else
     # Download the latest VPP build install packages
-    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-        --skip-install
+    source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh"
+    download_artifacts
+    # Need to revert -euo as the rest of script is not optimized for this.
+    set +euo pipefail
 fi
 
 VIRL_DIR_LOC="/tmp/"
diff --git a/bootstrap-vpp-verify-weekly.sh b/bootstrap-vpp-verify-weekly.sh
index 2ee9a9792c..d96e2d1f37 100644
--- a/bootstrap-vpp-verify-weekly.sh
+++ b/bootstrap-vpp-verify-weekly.sh
@@ -59,10 +59,12 @@ if [ "${#}" -ne "0" ]; then
     echo ${arr[0]}
 else
     # Download the specific VPP build install packages
-    DPDK_STABLE_VER=$(cat ${SCRIPT_DIR}/DPDK_STABLE_VER)
-    VPP_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO})
-    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-        --skip-install --vpp ${VPP_STABLE_VER} --dkms ${DPDK_STABLE_VER}
+    DKMS_VERSION=$(< ${SCRIPT_DIR}/DPDK_STABLE_VER)
+    VPP_VERSION=$(< ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO})
+    source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh"
+    download_artifacts
+    # Need to revert -euo as the rest of script is not optimized for this.
+    set +euo pipefail
 fi
 
 VIRL_DIR_LOC="/tmp/"
diff --git a/bootstrap.sh b/bootstrap.sh
index 02aa7133e5..015858b1fd 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -44,10 +44,12 @@ if [ "${#}" -ne "0" ]; then
     echo ${arr[0]}
     SKIP_PATCH="skip_patchORskip_vpp_patch"
 else
-    DPDK_STABLE_VER=$(cat ${SCRIPT_DIR}/DPDK_STABLE_VER)
-    VPP_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO})
-    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-        --skip-install --vpp ${VPP_STABLE_VER} --dkms ${DPDK_STABLE_VER}
+    DKMS_VERSION=$(< ${SCRIPT_DIR}/DPDK_STABLE_VER)
+    VPP_VERSION=$(< ${SCRIPT_DIR}/VPP_STABLE_VER_${DISTRO})
+    source "${SCRIPT_DIR}/resources/libraries/bash/function/artifacts.sh"
+    download_artifacts
+    # Need to revert -euo as the rest of script is not optimized for this.
+    set +euo pipefail
 fi
 
 VIRL_DIR_LOC="/tmp/"
diff --git a/resources/libraries/bash/function/artifacts.sh b/resources/libraries/bash/function/artifacts.sh
new file mode 100644
index 0000000000..945bdfe824
--- /dev/null
+++ b/resources/libraries/bash/function/artifacts.sh
@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+function download_artifacts () {
+    # Get and/or install VPP artifacts from packagecloud.io.
+    #
+    # Variables set:
+    # - REPO_URL - FD.io Packagecloud repository.
+
+    set -exuo pipefail
+
+    os_id=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') || {
+        die "Get OS release failed."
+    }
+
+    repo_url_path="./VPP_REPO_URL"
+    if [ -e "${repo_url_path}" ]; then
+        REPO_URL="$(<${repo_url_path})" || {
+            die "Read repo URL from ${repo_url_path} failed."
+        }
+    else
+        REPO_URL="https://packagecloud.io/install/repositories/fdio/master"
+    fi
+
+    if [ "${os_id}" == "ubuntu" ]; then
+        download_ubuntu_artifacts || die
+    elif [ "${os_id}" == "centos" ]; then
+        download_centos_artifacts || die
+    elif [ "${os_id}" == "opensuse" ]; then
+        download_opensuse_artifacts || die
+    else
+        die "${os_id} is not yet supported."
+    fi
+}
+
+function download_ubuntu_artifacts () {
+    # Get and/or install Ubuntu VPP artifacts from packagecloud.io.
+    #
+    # Variables read:
+    # - REPO_URL - FD.io Packagecloud repository.
+    # - VPP_VERSION - VPP version.
+    # - DKMS_VERSION - DKMS version.
+    # - INSTALL - If install packages or download only. Default: download
+
+    set -exuo pipefail
+
+    curl -s "${REPO_URL}"/script.deb.sh | sudo bash || {
+        die "Packagecloud FD.io repo fetch failed."
+    }
+    # If version is set we will add suffix.
+    artifacts=()
+    vpp=(vpp vpp-dbg vpp-dev vpp-lib vpp-plugins)
+    if [ -z "${VPP_VERSION-}" ]; then
+        artifacts+=(${vpp[@]})
+    else
+        artifacts+=(${vpp[@]/%/=${VPP_VERSION-}})
+    fi
+    dkms=(vpp-dpdk-dkms)
+    if [ -z "${DKMS_VERSION-}" ]; then
+        artifacts+=(${dkms[@]})
+    else
+        artifacts+=(${dkms[@]/%/=${DKMS_VERSION-}})
+    fi
+
+    if [ "${INSTALL:-false}" = true ]; then
+        sudo apt-get -y install "${artifacts[@]}" || {
+            die "Install VPP artifacts failed."
+        }
+    else
+        apt-get -y download "${artifacts[@]}" || {
+            die "Download VPP artifacts failed."
+        }
+    fi
+}
+
+function download_centos_artifacts () {
+    # Get and/or install CentOS VPP artifacts from packagecloud.io.
+    #
+    # Variables read:
+    # - REPO_URL - FD.io Packagecloud repository.
+    # - VPP_VERSION - VPP version.
+    # - INSTALL - If install packages or download only. Default: download
+
+    set -exuo pipefail
+
+    curl -s "${REPO_URL}"/script.rpm.sh | sudo bash || {
+        die "Packagecloud FD.io repo fetch failed."
+    }
+    # If version is set we will add suffix.
+    artifacts=()
+    vpp=(vpp vpp-selinux-policy vpp-devel vpp-lib vpp-plugins)
+    if [ -z "${VPP_VERSION-}" ]; then
+        artifacts+=(${vpp[@]})
+    else
+        artifacts+=(${vpp[@]/%/-${VPP_VERSION-}})
+    fi
+
+    if [ "${INSTALL:-false}" = true ]; then
+        sudo yum -y install "${artifacts[@]}" || {
+            die "Install VPP artifact failed."
+        }
+    else
+        sudo yum -y install --downloadonly --downloaddir=. "${artifacts[@]}" || {
+            die "Download VPP artifacts failed."
+        }
+    fi
+}
+
+function download_opensuse_artifacts () {
+    # Get and/or install OpenSuSE VPP artifacts from packagecloud.io.
+    #
+    # Variables read:
+    # - REPO_URL - FD.io Packagecloud repository.
+    # - VPP_VERSION - VPP version.
+    # - INSTALL - If install packages or download only. Default: download
+
+    set -exuo pipefail
+
+    curl -s "${REPO_URL}"/script.rpm.sh | sudo bash || {
+        die "Packagecloud FD.io repo fetch failed."
+    }
+    # If version is set we will add suffix.
+    artifacts=()
+    vpp=(vpp vpp-devel vpp-lib vpp-plugins libvpp0)
+    if [ -z "${VPP_VERSION-}" ]; then
+        artifacts+=(${vpp[@]})
+    else
+        artifacts+=(${vpp[@]/%/-${VPP_VERSION-}})
+    fi
+
+    if [ "${INSTALL:-false}" = true ]; then
+        sudo yum -y install "${artifacts[@]}" || {
+            die "Install VPP artifact failed."
+        }
+    else
+        sudo yum -y install --downloadonly --downloaddir=. "${artifacts[@]}" || {
+            die "Download VPP artifacts failed."
+        }
+    fi
+}
+
diff --git a/resources/libraries/bash/function/gather.sh b/resources/libraries/bash/function/gather.sh
index f490c80110..e030652494 100644
--- a/resources/libraries/bash/function/gather.sh
+++ b/resources/libraries/bash/function/gather.sh
@@ -246,6 +246,7 @@ function gather_vpp () {
     set -exuo pipefail
 
     # Variables read:
+    # - BASH_FUNCTION_DIR - Bash directory with functions.
     # - TEST_CODE - The test selection string from environment or argument.
     # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
     # - CSIT_DIR - Path to existing root of local CSIT git repository.
@@ -267,32 +268,23 @@ function gather_vpp () {
     case "${TEST_CODE}" in
         # Not csit-vpp as this code is re-used by ligato gathering.
         "csit-"*)
-            install_script="${CSIT_DIR}/resources/tools/scripts/"
-            install_script+="download_install_vpp_pkgs.sh"
             # Use downloaded packages with specific version
             if [[ "${TEST_CODE}" == *"daily"* ]] || \
                [[ "${TEST_CODE}" == *"weekly"* ]] || \
                [[ "${TEST_CODE}" == *"timed"* ]];
             then
-                echo "Downloading latest VPP packages from NEXUS..."
-                # TODO: Can we source?
-                bash "${install_script}" --skip-install || {
-                    die "Failed to get VPP packages!"
-                }
+                warn "Downloading latest VPP packages from Packagecloud."
             else
-                echo "Downloading VPP packages of specific version from NEXUS."
-                dpdk_stable_ver="$(cat "${CSIT_DIR}/DPDK_STABLE_VER")" || {
-                    die "Cat failed."
-                }
-                vpp_stable_ver="$(cat "${CSIT_DIR}/VPP_STABLE_VER_UBUNTU")" || {
-                    die "Cat failed."
+                warn "Downloading stable VPP packages from Packagecloud."
+                DKMS_VERSION="$(<"${CSIT_DIR}/DPDK_STABLE_VER")" || {
+                    die "Read DPDK stable version failed."
                 }
-                install_args=("--skip-install" "--vpp" "${vpp_stable_ver}")
-                install_args+=("--dkms" "${dpdk_stable_ver}")
-                bash "${install_script}" "${install_args[@]}" || {
-                    die "Failed to get VPP packages!"
+                VPP_VERSION="$(<"${CSIT_DIR}/VPP_STABLE_VER_UBUNTU")" || {
+                    die "Read VPP stable version failed."
                 }
             fi
+            source "${BASH_FUNCTION_DIR}/artifacts.sh" || die "Source failed."
+            download_artifacts || die
             ;;
         "vpp-csit-"*)
             # Use local built packages.
diff --git a/resources/tools/scripts/download_install_vpp_pkgs.sh b/resources/tools/scripts/download_install_vpp_pkgs.sh
deleted file mode 100755
index ce7530af83..0000000000
--- a/resources/tools/scripts/download_install_vpp_pkgs.sh
+++ /dev/null
@@ -1,179 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e -o pipefail
-
-OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-
-NEXUSPROXY="https://nexus.fd.io"
-
-function artifacts {
-    if [ "$OS_ID" == "ubuntu" ]; then
-        VPP_REPO_URL_PATH="./VPP_REPO_URL_UBUNTU"
-        if [ -e "$VPP_REPO_URL_PATH" ]; then
-            VPP_REPO_URL=$(cat $VPP_REPO_URL_PATH)
-            REPO_NAME=$(echo ${VPP_REPO_URL#https://nexus.fd.io/content/repositories/})
-            REPO_NAME=$(echo ${REPO_NAME%io/fd/vpp/})
-        else
-            OS_VERSION_CODENAME=$(grep '^VERSION_CODENAME=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-            REPO_NAME="fd.io.master.ubuntu.${OS_VERSION_CODENAME}.main"
-        fi
-        REPO_URL="${NEXUSPROXY}/content/repositories/${REPO_NAME}"
-
-        echo "deb ${REPO_URL} ./" | sudo tee /etc/apt/sources.list.d/99fd.io.list
-        sudo apt-get -y update \
-            -o Dir::Etc::sourcelist="sources.list.d/99fd.io.list" \
-            -o Acquire::AllowInsecureRepositories=true \
-            -o Dir::Etc::sourceparts="-" \
-            -o APT::Get::AllowUnauthenticated=true \
-            -o APT::Get::List-Cleanup="0"
-
-        # If version is set we will add suffix
-        VPP=(vpp vpp-dbg vpp-dev vpp-lib vpp-plugins)
-        DKMS=(vpp-dpdk-dkms)
-        if [ -z "${VPP_VERSION}" ]; then
-            ARTIFACTS+=(${VPP[@]/%/${VPP_VERSION}})
-            ARTIFACTS+=(${DKMS[@]/%/${DKMS_VERSION}})
-        else
-            ARTIFACTS+=(${VPP[@]/%/=${VPP_VERSION}})
-            ARTIFACTS+=(${DKMS[@]/%/=${DKMS_VERSION}})
-        fi
-
-        if [ "$INSTALL" = true ]; then
-            echo Installing VPP
-            sudo apt-get -y install ${ARTIFACTS[@]} \
-                -o Acquire::AllowInsecureRepositories=true \
-                -o APT::Get::AllowUnauthenticated=true
-        else
-            echo Downloading VPP
-            apt-get -y download ${ARTIFACTS[@]} \
-                -o Acquire::AllowInsecureRepositories=true \
-                -o APT::Get::AllowUnauthenticated=true
-        fi
-
-    elif [ "$OS_ID" == "centos" ]; then
-        VPP_REPO_URL_PATH="./VPP_REPO_URL_CENTOS"
-        if [ -e "$VPP_REPO_URL_PATH" ]; then
-            VPP_REPO_URL=$(cat $VPP_REPO_URL_PATH)
-            REPO_NAME=$(echo ${VPP_REPO_URL#https://nexus.fd.io/content/repositories/})
-            REPO_NAME=$(echo ${REPO_NAME%/io/fd/vpp/})
-        else
-            REPO_NAME="fd.io.master.centos7"
-        fi
-        REPO_URL="${NEXUSPROXY}/content/repositories/${REPO_NAME}"
-
-        sudo cat << EOF > fdio-master.repo
-[fdio-master]
-name=fd.io master branch latest merge
-baseurl=${REPO_URL}
-enabled=1
-gpgcheck=0
-EOF
-        sudo mv fdio-master.repo /etc/yum.repos.d/fdio-master.repo
-
-        # If version is set we will add suffix
-        VPP=(vpp vpp-selinux-policy vpp-devel vpp-lib vpp-plugins)
-        if [ -z "${VPP_VERSION}" ]; then
-            ARTIFACTS+=(${VPP[@]/%/${VPP_VERSION}})
-        else
-            ARTIFACTS+=(${VPP[@]/%/-${VPP_VERSION}})
-        fi
-
-        if [ "$INSTALL" = true ]; then
-            echo Installing VPP
-            sudo yum -y install ${ARTIFACTS[@]}
-        else
-            echo Downloading VPP
-            sudo yum -y install --downloadonly --downloaddir=. ${ARTIFACTS[@]}
-        fi
-    elif [ "$OS_ID" == "opensuse" ]; then
-        VPP_REPO_URL_PATH="./VPP_REPO_URL_OPENSUSE"
-        if [ -e "$VPP_REPO_URL_PATH" ]; then
-            VPP_REPO_URL=$(cat $VPP_REPO_URL_PATH)
-            REPO_NAME=$(echo ${VPP_REPO_URL#https://nexus.fd.io/content/repositories/})
-            REPO_NAME=$(echo ${REPO_NAME%/io/fd/vpp/})
-        else
-            REPO_NAME='fd.io.master.opensuse'
-        fi
-        REPO_URL="${NEXUSPROXY}/content/repositories/${REPO_NAME}"
-
-        sudo cat << EOF > fdio-master.repo
-[fdio-master]
-name=fd.io master branch latest merge
-baseurl=${REPO_URL}
-enabled=1
-gpgcheck=0
-EOF
-        sudo mv fdio-master.repo /etc/yum/repos.d/fdio-master.repo
-
-        # If version is set we will add suffix
-        VPP=(vpp vpp-devel vpp-lib vpp-plugins libvpp0)
-        if [ -z "${VPP_VERSION}" ]; then
-            ARTIFACTS+=(${VPP[@]/%/${VPP_VERSION}})
-        else
-            ARTIFACTS+=(${VPP[@]/%/-${VPP_VERSION}})
-        fi
-
-        if [ "$INSTALL" = true ]; then
-            echo Installing VPP
-            sudo yum -y install ${ARTIFACTS[@]}
-        else
-            echo Downloading VPP
-            sudo yum -y install --downloadonly --downloaddir=. ${ARTIFACTS[@]}
-        fi
-    else
-        echo "$OS_ID is not yet supported."
-        exit 1
-    fi
-}
-
-function display_help () {
-    echo "Usage: $0 [--skip-install] [--vpp <version>] [--dkms <version>]"
-}
-
-# Whether to install artifacts or not
-INSTALL=true
-# VPP version (default empty = latest)
-VPP_VERSION=""
-# DKMS version (default empty = latest)
-DKMS_VERSION=""
-
-while :
-do
-    case "$1" in
-        -h | --help)
-            display_help
-            exit 0
-            ;;
-        -s | --skip-install)
-            INSTALL=false
-            shift 1
-            ;;
-        -v | --vpp)
-            VPP_VERSION="$2"
-            shift 2
-            ;;
-        -d | --dkms)
-            DKMS_VERSION="$2"
-            shift 2
-            ;;
-        *)
-            break
-            ;;
-     esac
-done
-
-artifacts "${INSTALL}" "${VPP_VERSION}" "${DKMS_VERSION}"