Merge "Fix upload of vpp-ext-deps cached in executor"

5 files changed, 121 insertions, 128 deletions

diff --git a/jjb/scripts/packagecloud_push.sh b/jjb/scripts/packagecloud_push.sh
index a4ae9b166..80226aa04 100644..100755
--- a/jjb/scripts/packagecloud_push.sh
+++ b/jjb/scripts/packagecloud_push.sh
@@ -40,38 +40,53 @@ sleep 10
 FACTER_OS=$(/usr/bin/facter operatingsystem)
 push_cmd=""
 push_ext_deps_cmd=""
+ext_deps_pkg=""
+downloads_dir="/root/Downloads"
+
+create_deb_push_cmds()
+{
+    local distro="$1"
+
+    if [ "$distro" = "debian" ] || [ "$distro" = "ubuntu" ] ; then
+        FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
+        DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps | xargs || true)
+        push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/${distro}/${FACTER_LSBNAME}/main/ ${DEBS}"
+        ext_deps_ver="$(dpkg -l vpp-ext-deps | mawk '/vpp-ext-deps/{print $3}' || true)"
+        ext_deps_pkg="$(find . -type f -iname 'vpp-ext-deps*.deb' | grep $ext_deps_ver || find $downloads_dir -type f -iname 'vpp-ext-deps*.deb' | grep $ext_deps_ver || true)"
+        if [ -n "$ext_deps_pkg}" ] ; then
+            push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/${distro}/${FACTER_LSBNAME}/main/ ${ext_deps_pkg}"
+        fi
+    else
+        echo "ERROR: Unknown distro: '$distro'"
+        return 1
+    fi
+}
+
+create_rpm_push_cmds()
+{
+    FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease)
+    FACTER_ARCH=$(/usr/bin/facter architecture)
+    RPMS=$(find . -type f -iregex '.*/.*\.\(s\)?rpm' | grep -v vpp-ext-deps | xargs || true)
+    push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${RPMS}"
+    ext_deps_ver="$(dnf list vpp-ext-deps | mawk '/vpp-ext-deps/{print $2}' || true)"
+    ext_deps_pkg="$(find . -type f -iname 'vpp-ext-deps*.rpm' | grep $ext_deps_ver || find $downloads_dir -type f -iname 'vpp-ext-deps*.rpm' | grep $ext_deps_ver || true)"
+    if [ -n "$ext_deps_pkg" ] ; then
+        push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${ext_deps_pkg}"
+    fi
+}
 
 # PCIO_CO and SILO are Jenkins Global Environment variables defined in
 # .../ci-management/jenkins-config/global-vars-*.sh
 if [ -f ~/.packagecloud ]; then
     case "$FACTER_OS" in
         Debian)
-            FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
-            DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps)
-            push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/debian/${FACTER_LSBNAME}/main/ ${DEBS}"
-            EXT_DEPS_DEB=$(find . -type f -iname 'vpp-ext-deps*.deb')
-            if [ -n "$EXT_DEPS_DEB" ] ; then
-                push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/debian/${FACTER_LSBNAME}/main/ ${EXT_DEPS_DEB}"
-            fi
+            create_deb_push_cmds debian
             ;;
         Ubuntu)
-            FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
-            DEBS=$(find . -type f -iname '*.deb' | grep -v vpp-ext-deps)
-            push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ ${DEBS}"
-            EXT_DEPS_DEB=$(find . -type f -iname 'vpp-ext-deps*.deb')
-            if [ -n "$EXT_DEPS_DEB" ] ; then
-                push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ ${EXT_DEPS_DEB}"
-            fi
+            create_deb_push_cmds ubuntu
             ;;
         CentOS)
-            FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease)
-            FACTER_ARCH=$(/usr/bin/facter architecture)
-            RPMS=$(find . -type f -iregex '.*/.*\.\(s\)?rpm' | grep -v vpp-ext-deps)
-            push_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${RPMS}"
-            EXT_DEPS_RPM=$(find . -type f -iname 'vpp-ext-deps*.rpm')
-            if [ -n "$EXT_DEPS_RPM" ] ; then
-                push_ext_deps_cmd="package_cloud push ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ ${EXT_DEPS_RPM}"
-            fi
+            create_rpm_push_cmds
             ;;
         *)
             echo -e "\n$line\n* ERROR: Unsupported OS '$FACTER_OS'\n* PACKAGECLOUD PUSH FAILED!\n$line\n"
@@ -86,10 +101,7 @@ if [ -f ~/.packagecloud ]; then
     else
         $push_cmd
         if [ -n "$push_ext_deps_cmd" ] ; then
-            # Don't fail script if vpp-ext-deps push fails.
-            set +e
-            $push_ext_deps_cmd
-            set -e
+            $push_ext_deps_cmd || true
         fi
     fi
 else
diff --git a/jjb/scripts/setup_vpp_dpdk_dev_env.sh b/jjb/scripts/setup_vpp_dpdk_dev_env.sh
deleted file mode 100644
index 2c55f166b..000000000
--- a/jjb/scripts/setup_vpp_dpdk_dev_env.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2020 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "---> jjb/scripts/setup_vpp_dpdk_dev_env.sh"
-
-set -e -o pipefail
-
-OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-
-function setup {
-    if [ -n "$REPO_NAME" ] ; then
-        echo "Installing vpp-ext-deps..."
-        REPO_URL="https://packagecloud.io/fdio/${STREAM}"
-        echo "REPO_URL: $REPO_URL"
-        INSTALL_URL="https://packagecloud.io/install/repositories/fdio/${STREAM}"
-        echo "INSTALL_URL: $INSTALL_URL"
-        # Setup by installing vpp-dev and vpp-lib
-        if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then
-            if [ "${STREAM}" != "master" ]; then
-                echo "stream '${STREAM}' is not master: replacing packagecloud apt sources list with stream specific list"
-                sudo apt-get -y remove vpp-ext-deps || true
-                sudo rm -f /etc/apt/sources.list.d/fdio_master.list
-                curl -s $INSTALL_URL/script.deb.sh | sudo bash
-            fi
-            sudo apt-get update -qq || true
-            local vpp_ext_deps_version="$(apt-cache show vpp-ext-deps | mawk '/Version/ {print $2}' | head -1)"
-            local vpp_ext_deps_arch="$(apt-cache show vpp-ext-deps | mawk '/Architecture/ {print $2}' | head -1)"
-            local vpp_ext_deps_pkg="/root/Downloads/vpp-ext-deps_${vpp_ext_deps_version}_${vpp_ext_deps_arch}.deb"
-            if [ -f "$vpp_ext_deps_pkg" ] ; then
-                echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg"
-                sudo dpkg -i $vpp_ext_deps_pkg
-            else
-                echo "Installing vpp-ext-deps from packagecloud.io"
-                local force_opts="--allow-downgrades --allow-remove-essential"
-                force_opts="$force_opts --allow-change-held-packages"
-                sudo apt-get -y $force_opts install vpp-ext-deps || true
-            fi
-            echo "Removing packagecloud.io repository references and running apt-get update"
-            sudo rm -f /etc/apt/sources.list.d/fdio_*.list
-            sudo apt-get update -qq || true
-        elif [ "${OS_ID,,}" == "centos" ] ; then
-            if [ "${STREAM}" != "master" ] ; then
-                echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list"
-                sudo yum -y erase vpp-ext-deps || true
-                sudo yum clean all || true
-                sudo rm -f /etc/yum.repos.d/fdio_master.repo
-                curl -s $INSTALL_URL/script.rpm.sh | sudo bash
-            fi
-            local vpp_ext_deps_version="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $2}')"
-            local vpp_ext_deps_pkg="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $1}')"
-            vpp_ext_deps_pkg="/root/Downloads/${vpp_ext_deps_pkg/./-${vpp_ext_deps_version}.}.rpm"
-            if [ -f "$vpp_ext_deps_pkg" ] ; then
-                echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg"
-                sudo yum -y localinstall $vpp_ext_deps_pkg || true
-            else
-                echo "Installing vpp-ext-deps from packagecloud.io"
-                sudo yum -y install vpp-ext-deps || true
-            fi
-        else
-            echo "ERROR: Unsupported OS '$OS_ID'!"
-        fi
-    else
-        echo "ERROR: REPO_NAME not found!"
-    fi
-}
-
-setup
diff --git a/jjb/scripts/setup_vpp_ext_deps.sh b/jjb/scripts/setup_vpp_ext_deps.sh
new file mode 100755
index 000000000..1ca982d45
--- /dev/null
+++ b/jjb/scripts/setup_vpp_ext_deps.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/setup_vpp_ext_deps.sh"
+
+set -e -o pipefail
+
+OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+
+echo "Installing vpp-ext-deps..."
+REPO_URL="https://packagecloud.io/fdio/${STREAM}"
+echo "REPO_URL: $REPO_URL"
+INSTALL_URL="https://packagecloud.io/install/repositories/fdio/${STREAM}"
+echo "INSTALL_URL: $INSTALL_URL"
+
+downloads_dir="/root/Downloads"
+
+# Setup by installing vpp-dev and vpp-lib
+if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then
+    if [ "${STREAM}" != "master" ]; then
+        echo "stream '${STREAM}' is not master: replacing packagecloud apt sources list with stream specific list"
+        sudo apt-get -y remove vpp-ext-deps || true
+        sudo rm -f /etc/apt/sources.list.d/fdio_master.list
+        curl -s $INSTALL_URL/script.deb.sh | sudo bash || true
+    fi
+    sudo apt-get update -qq || true
+    vpp_ext_deps_version="$(apt-cache show vpp-ext-deps | mawk '/Version/ {print $2}' | head -1)"
+    vpp_ext_deps_arch="$(apt-cache show vpp-ext-deps | mawk '/Architecture/ {print $2}' | head -1)"
+    vpp_ext_deps_pkg="vpp-ext-deps_${vpp_ext_deps_version}_${vpp_ext_deps_arch}.deb"
+    if [ -f "$downloads_dir/$vpp_ext_deps_pkg" ] ; then
+        echo "Installing cached vpp-ext-deps pkg: $downloads_dir/$vpp_ext_deps_pkg"
+        sudo dpkg -i "$downloads_dir/$vpp_ext_deps_pkg" || true
+    else
+        echo "Installing vpp-ext-deps from packagecloud.io"
+        force_opts="--allow-downgrades --allow-remove-essential --allow-change-held-packages"
+        sudo apt-get -y $force_opts install vpp-ext-deps || true
+    fi
+    echo "Removing packagecloud.io repository references and running apt-get update"
+    sudo rm -f /etc/apt/sources.list.d/fdio_*.list
+    sudo apt-get update -qq || true
+#TODO: Remove centos when VPP 21.06 is no longer supported
+elif [ "${OS_ID,,}" == "centos" ] ; then
+    if [ "${STREAM}" != "master" ] ; then
+        echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list"
+        sudo yum -y erase vpp-ext-deps || true
+        sudo yum clean all || true
+        sudo rm -f /etc/yum.repos.d/fdio_master.repo
+        curl -s $INSTALL_URL/script.rpm.sh | sudo bash || true
+    fi
+    vpp_ext_deps_version="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $2}')"
+    vpp_ext_deps_pkg="$(yum -q list vpp-ext-deps 2> /dev/null | mawk '/vpp-ext-deps/{print $1}')"
+    vpp_ext_deps_pkg="${vpp_ext_deps_pkg/./-${vpp_ext_deps_version}.}.rpm"
+    if [ -f "$vpp_ext_deps_pkg" ] ; then
+        echo "Installing cached vpp-ext-deps pkg: $vpp_ext_deps_pkg"
+        sudo yum -y localinstall "$downloads_dir/$vpp_ext_deps_pkg" || true
+   else
+        echo "Installing vpp-ext-deps from packagecloud.io"
+        sudo yum -y install vpp-ext-deps || true
+    fi
+else
+    echo "ERROR: Unsupported OS '$OS_ID'!"
+fi
diff --git a/jjb/scripts/setup_vpp_ubuntu_docker_test.sh b/jjb/scripts/setup_vpp_ubuntu_docker_test.sh
index 8ed0acaee..548ac5660 100644..100755
--- a/jjb/scripts/setup_vpp_ubuntu_docker_test.sh
+++ b/jjb/scripts/setup_vpp_ubuntu_docker_test.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ##############################################################################
-# Copyright (c) 2018 The Linux Foundation and others.
+# Copyright (c) 2021 The Linux Foundation and others.
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Eclipse Public License v1.0
@@ -30,17 +30,3 @@ if [ -n ${DOCKER_TEST} ] ; then
 	sudo mount -o remount /dev/shm -o size=${MEM} || true
         echo "/dev/shm remounted with size='${MEM}'"
 fi
-
-# This will remove any previously installed external packages
-# for old branch builds
-if [ "${GERRIT_BRANCH}" != "master" ]; then
-    echo "Removing $OS_ID-$OS_VERSION_ID package 'vpp-ext-deps'"
-    if [ "${OS_ID,,}" == "ubuntu" ] || [ "${OS_ID,,}" == "debian" ] ; then
-        sudo apt-get -y remove vpp-ext-deps || true
-    elif [ "${OS_ID,,}" == "centos" ]; then
-        sudo yum -y erase vpp-ext-deps || true
-        sudo yum clean all || true
-    else
-        echo "ERROR: Unsupported OS '$OS_ID'!"
-    fi
-fi
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 880fcb215..ed470580d 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -475,7 +475,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/build.sh
@@ -539,7 +539,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/debug-build.sh
@@ -606,7 +606,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/build.sh
@@ -669,7 +669,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/gcc-build.sh
@@ -747,7 +747,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/csit-device.sh
@@ -824,7 +824,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/csit-device.sh
@@ -909,7 +909,7 @@
           - ../scripts/setup_vpp_ubuntu_docker_test.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/setup_vpp_dpdk_dev_env.sh
+          - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/csit-perf.sh