CSIT-1362: migrate package download scripts for hc2vpp jobs

As part of Bionic update for honeycomb (HC2VPP-372)
scripts need to be updated. Switching to Packagecloud instead
of nexus simplifies this process.


Change-Id: I243367e8c1d7c3de5890d6846c199563cef457bf
Signed-off-by: Michal Cmarada <mcmarada@cisco.com>

2 files changed, 142 insertions, 101 deletions

diff --git a/resources/tools/scripts/download_hc_build_pkgs.sh b/resources/tools/scripts/download_hc_build_pkgs.sh
index 48ef1eb4b0..6a56db910e 100755
--- a/resources/tools/scripts/download_hc_build_pkgs.sh
+++ b/resources/tools/scripts/download_hc_build_pkgs.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -18,68 +18,81 @@ set -ex
 STREAM=$1
 OS=$2
 
-URL="https://nexus.fd.io/service/local/artifact/maven/content"
-VPP_GROUP="io.fd.vpp"
-# TODO(CSIT-994): reenable NSH
-# NSH_GROUP="io.fd.nsh_sfc"
-# NSH_ARTIFACTS="vpp-nsh-plugin"
-VPP_ARTIFACTS="vpp vpp-lib vpp-plugins vpp-api-java"
-
-if [ "${OS}" == "ubuntu1604" ]; then
-    OS="ubuntu.xenial.main"
-    PACKAGE="deb deb.md5"
-    CLASS="deb"
-elif [ "${OS}" == "centos7" ]; then
-    OS="centos7"
-    PACKAGE="rpm rpm.md5"
-    CLASS=""
+# Figure out what system we are running on
+if [[ -f /etc/os-release ]];then
+    source /etc/os-release
+else
+    echo "Cannot determine OS version"
+    exit 1
 fi
-
-REPO="fd.io.${STREAM}.${OS}"
-
-# Use vpp packages based on vpp-version file from hc2vpp project
-VER=`../vpp-version`
-if [ "${VER}" != 'RELEASE' ]; then
-    if [ "${OS}" == "centos7" ]; then
-        VER="${VER}.x86_64"
+echo "----- OS INFO -----"
+echo DISTRIB_ID: ${ID}
+echo DISTRIB_RELEASE: ${VERSION_ID}
+echo DISTRIB_CODENAME: ${VERSION_CODENAME}
+echo DISTRIB_DESCRIPTION: ${PRETTY_NAME}
+
+VERSION=`../vpp-version`
+JVPP_VERSION=`../jvpp-version`
+VPP_DEB_NEW_ARTIFACTS="vpp libvppinfra vpp-plugin-core vpp-api-java"
+VPP_DEB_ARTIFACTS="vpp vpp-lib vpp-plugins vpp-api-java"
+VPP_RPM_ARTIFACTS="vpp vpp-lib vpp-plugins vpp-api-java"
+IGNORE_DEPS=""
+# Check OS and stream to set correct packages
+if [[ "$ID" == "centos" ]]; then
+    VPP_ARTIFACTS=${VPP_RPM_ARTIFACTS}
+elif [[ "$ID" == "ubuntu" ]]; then
+    if [[ "1807 1810 1901" =~ .*$STREAM.* ]]; then
+        VPP_ARTIFACTS=${VPP_DEB_ARTIFACTS}
+        IGNORE_DEPS="vpp,vpp-lib,vpp-plugins"
     else
-        VER="${VER}_amd64"
+        VPP_ARTIFACTS=${VPP_DEB_NEW_ARTIFACTS}
+        IGNORE_DEPS="vpp,libvppinfra,vpp-plugin-core"
     fi
 fi
-
+VPP_DEB_PACKAGES=""
+VPP_RPM_PACKAGES=""
 for ART in ${VPP_ARTIFACTS}; do
-    for PAC in ${PACKAGE}; do
-        curl "${URL}?r=${REPO}&g=${VPP_GROUP}&a=${ART}&p=${PAC}&v=${VER}&c=${CLASS}" -O -J || exit
-    done
+    if [[ "${VERSION}" != 'RELEASE' ]]; then
+        if [[ "$ART" == "vpp-api-java" ]]; then
+            VPP_DEB_PACKAGES="$VPP_DEB_PACKAGES $ART=$JVPP_VERSION"
+            VPP_RPM_PACKAGES="$VPP_RPM_PACKAGES $ART-$JVPP_VERSION"
+        else
+            VPP_DEB_PACKAGES="$VPP_DEB_PACKAGES $ART=$VERSION"
+            VPP_RPM_PACKAGES="$VPP_RPM_PACKAGES $ART-$VERSION"
+        fi
+    else
+        VPP_DEB_PACKAGES="$VPP_DEB_PACKAGES $ART"
+        VPP_RPM_PACKAGES="$VPP_RPM_PACKAGES $ART"
+    fi
 done
 
-# TODO(CSIT-994): reenable NSH
-# for ART in ${NSH_ARTIFACTS}; do
-#     for PAC in ${PACKAGE}; do
-#         curl "${URL}?r=${REPO}&g=${NSH_GROUP}&a=${ART}&p=${PAC}&v=${VER}&c=${CLASS}" -O -J || exit
-#     done
-# done
-
-# verify downloaded packages
-if [ "${OS}" == "centos7" ]; then
-    FILES=*.rpm
-else
-    FILES=*.deb
+echo "----- DOWNLOADING PACKAGES -----"
+REPO_URL="https://packagecloud.io/fdio/${STREAM}"
+echo "REPO_URL: ${REPO_URL}"
+if [[ "$ID" == "ubuntu" ]]; then
+    if [[ -f /etc/apt/sources.list.d/99fd.io.list ]];then
+        echo "Deleting: /etc/apt/sources.list.d/99fd.io.list"
+        sudo rm /etc/apt/sources.list.d/99fd.io.list
+    fi
+    curl -s https://packagecloud.io/install/repositories/fdio/${STREAM}/script.deb.sh | sudo bash
+    apt-get download ${VPP_DEB_PACKAGES} || true
+elif [[ "$ID" == "centos" ]]; then
+    if [[ -f /etc/yum.repos.d/fdio-master.repo ]]; then
+        echo "Deleting: /etc/yum.repos.d/fdio-master.repo"
+        sudo rm /etc/yum.repos.d/fdio-master.repo
+    fi
+    curl -s https://packagecloud.io/install/repositories/fdio/${STREAM}/script.rpm.sh | sudo bash
+    sudo yum -y install --downloadonly --downloaddir=./ ${VPP_RPM_PACKAGES} || true
 fi
-
-for FILE in ${FILES}; do
-    echo " "${FILE} >> ${FILE}.md5
-done
-for MD5FILE in *.md5; do
-    md5sum -c ${MD5FILE} || exit
-    rm ${MD5FILE}
-done
+# TODO(CSIT-994): reenable NSH
+# NSH_GROUP="io.fd.nsh_sfc"
+# NSH_ARTIFACTS="vpp-nsh-plugin"
 
 # install vpp-api-java, this extracts jvpp .jar files into usr/share/java
-if [ "${OS}" == "centos7" ]; then
+if [[ "${OS}" == "centos7" ]]; then
     sudo rpm --nodeps --install vpp-api-java*
 else
-    sudo dpkg --ignore-depends=vpp --install vpp-api-java*
+    sudo dpkg --ignore-depends=${IGNORE_DEPS} --install vpp-api-java*
 fi
 
 # install jvpp jars into maven repo, so that maven picks them up when building hc2vpp
@@ -89,12 +102,19 @@ current_dir=`pwd`
 cd /usr/share/java
 
 for item in jvpp*.jar; do
-    # Example filename: jvpp-registry-17.01-20161206.125556-1.jar
+    # Example filename: jvpp-registry-19.04.jar
     # ArtifactId = jvpp-registry
-    # Version = 17.01
+    # Version = 19.04 or 19.04-SNAPSHOT
     basefile=$(basename -s .jar "$item")
     artifactId=$(echo "$basefile" | cut -d '-' -f 1-2)
     mvn install:install-file -Dfile=${item} -DgroupId=io.fd.vpp -DartifactId=${artifactId} -Dversion=${version} -Dpackaging=jar -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
 done
 
-cd ${current_dir}
\ No newline at end of file
+# vpp-api-package is no longer necessary, breaks the installation of other packages that follow in next steps
+if [[ "${OS}" == "centos7" ]]; then
+    sudo yum remove "*vpp-api-java*"
+else
+    sudo apt remove "*vpp-api-java*"
+fi
+
+cd ${current_dir}
diff --git a/resources/tools/scripts/download_hc_pkgs.sh b/resources/tools/scripts/download_hc_pkgs.sh
index caa92c555e..fb629da404 100755
--- a/resources/tools/scripts/download_hc_pkgs.sh
+++ b/resources/tools/scripts/download_hc_pkgs.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -18,34 +18,41 @@ set -x
 STREAM=$1
 OS=$2
 
-URL="https://nexus.fd.io/service/local/artifact/maven/content"
-VER="RELEASE"
-GROUP="io.fd.vpp"
-HC_GROUP="io.fd.hc2vpp"
-HC_ARTIFACTS="honeycomb"
 # TODO(CSIT-994): reenable NSH
 # NSH_GROUP="io.fd.nsh_sfc"
 # NSH_ARTIFACTS="vpp-nsh-plugin"
-VPP_ARTIFACTS="vpp vpp-lib vpp-plugins"
-
-if [ "${OS}" == "ubuntu1604" ]; then
-    OS="ubuntu.xenial.main"
-    PACKAGE="deb deb.md5"
-    CLASS="deb"
-elif [ "${OS}" == "centos7" ]; then
-    OS="centos7"
-    PACKAGE="rpm rpm.md5"
-    CLASS=""
+# Figure out what system we are running on
+if [[ -f /etc/os-release ]];then
+    source /etc/os-release
+else
+    echo "Cannot determine OS version"
+    exit 1
 fi
+echo "----- OS INFO -----"
+echo DISTRIB_ID: ${ID}
+echo DISTRIB_RELEASE: ${VERSION_ID}
+echo DISTRIB_CODENAME: ${VERSION_CODENAME}
+echo DISTRIB_DESCRIPTION: ${PRETTY_NAME}
 
-REPO="fd.io.${STREAM}.${OS}"
-
-# download latest honeycomb and nsh packages
-for ART in ${HC_ARTIFACTS}; do
-    for PAC in ${PACKAGE}; do
-        curl "${URL}?r=${REPO}&g=${HC_GROUP}&a=${ART}&p=${PAC}&v=${VER}&c=${CLASS}" -O -J || exit
-    done
-done
+VERSION="RELEASE"
+echo "----- DOWNLOADING HONEYCOMB AND JVPP PACKAGES -----"
+REPO_URL="https://packagecloud.io/fdio/${STREAM}"
+echo "REPO_URL: ${REPO_URL}"
+if [[ "$ID" == "ubuntu" ]]; then
+    if [[ -f /etc/apt/sources.list.d/99fd.io.list ]];then
+        echo "Deleting: /etc/apt/sources.list.d/99fd.io.list"
+        sudo rm /etc/apt/sources.list.d/99fd.io.list
+    fi
+    curl -s https://packagecloud.io/install/repositories/fdio/${STREAM}/script.deb.sh | sudo bash
+    apt-get download honeycomb vpp-api-java || true
+elif [[ "$ID" == "centos" ]]; then
+    if [[ -f /etc/yum.repos.d/fdio-master.repo ]]; then
+        echo "Deleting: /etc/yum.repos.d/fdio-master.repo"
+        sudo rm /etc/yum.repos.d/fdio-master.repo
+    fi
+    curl -s https://packagecloud.io/install/repositories/fdio/${STREAM}/script.rpm.sh | sudo bash
+    sudo yum -y install --downloadonly --downloaddir=./ honeycomb vpp-api-java || true
+fi
 
 # TODO(CSIT-994): reenable NSH
 # for ART in ${NSH_ARTIFACTS}; do
@@ -56,37 +63,51 @@ done
 
 # determine VPP dependency
 # use latest if honeycomb package does not depend on single VPP version, e.g. stable branches since HC2VPP-285
-VER="RELEASE"
-if [ "${OS}" == "centos7" ]; then
+VERSION="RELEASE"
+if [[ "${OS}" == "centos7" ]]; then
     HC_VPP_VER=`rpm -qpR honeycomb*.rpm | grep -oP 'vpp = \K.+'`
-    if [ "${HC_VPP_VER}" != "" ]; then
-        VER=${HC_VPP_VER}.x86_64
+    if [[ "${HC_VPP_VER}" != "" ]]; then
+        VERSION=${HC_VPP_VER}.x86_64
     fi
 else
     HC_VPP_VER=`dpkg -I honeycomb*.deb | grep -oP 'vpp \(= \K[^\)]+'`
-    if [ "${HC_VPP_VER}" != "" ]; then
-        VER=${HC_VPP_VER}_amd64
+    if [[ "${HC_VPP_VER}" != "" ]]; then
+        VERSION=${HC_VPP_VER}
     fi
 fi
 
-# download VPP packages
+VPP_DEB_NEW_ARTIFACTS="vpp libvppinfra vpp-plugin-core"
+VPP_DEB_ARTIFACTS="vpp vpp-lib vpp-plugins"
+VPP_RPM_ARTIFACTS="vpp vpp-lib vpp-plugins"
+# Check OS and stream to set correct packages
+if [[ "$ID" == "centos" ]]; then
+    VPP_ARTIFACTS=${VPP_RPM_ARTIFACTS}
+elif [[ "$ID" == "ubuntu" ]]; then
+    if [[ "1807 1810 1901" =~ .*$STREAM.* ]]; then
+        VPP_ARTIFACTS=${VPP_DEB_ARTIFACTS}
+    else
+        VPP_ARTIFACTS=${VPP_DEB_NEW_ARTIFACTS}
+    fi
+fi
+VPP_DEB_PACKAGES=""
+VPP_RPM_PACKAGES=""
+INSTALL_PACKAGES=""
 for ART in ${VPP_ARTIFACTS}; do
-    for PAC in ${PACKAGE}; do
-        curl "${URL}?r=${REPO}&g=${GROUP}&a=${ART}&p=${PAC}&v=${VER}&c=${CLASS}" -O -J || exit
-    done
+    INSTALL_PACKAGES="$INSTALL_PACKAGES $ART*"
+    if [[ "${VERSION}" != 'RELEASE' ]]; then
+            VPP_DEB_PACKAGES="$VPP_DEB_PACKAGES $ART=$VERSION"
+            VPP_RPM_PACKAGES="$VPP_RPM_PACKAGES $ART-$VERSION"
+    else
+        VPP_DEB_PACKAGES="$VPP_DEB_PACKAGES $ART"
+        VPP_RPM_PACKAGES="$VPP_RPM_PACKAGES $ART"
+    fi
 done
 
-# verify downloaded package
-if [ "${OS}" == "centos7" ]; then
-    FILES=*.rpm
-else
-    FILES=*.deb
+echo "----- DOWNLOADING VPP PACKAGES -----"
+REPO_URL="https://packagecloud.io/fdio/${STREAM}"
+echo "REPO_URL: ${REPO_URL}"
+if [[ "$ID" == "ubuntu" ]]; then
+    apt-get download ${VPP_DEB_PACKAGES} || true
+elif [[ "$ID" == "centos" ]]; then
+    sudo yum -y install --downloadonly --downloaddir=./ ${VPP_RPM_PACKAGES} || true
 fi
-
-for FILE in ${FILES}; do
-    echo " "${FILE} >> ${FILE}.md5
-done
-for MD5FILE in *.md5; do
-    md5sum -c ${MD5FILE} || exit
-    rm ${MD5FILE}
-done
\ No newline at end of file