Add the core file decoding+cleanup for the verify jobs

- Add better failure reporting and dry run handling
  for build scripts.

Change-Id: Ia19bae15ff4880b07094f4f665e5e00030eda27c
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

3 files changed, 101 insertions, 30 deletions

diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh
index 148999cec..409061b4e 100644
--- a/jjb/scripts/vpp/build.sh
+++ b/jjb/scripts/vpp/build.sh
@@ -33,6 +33,9 @@ DRYRUN="${DRYRUN:-}"
 IS_CSIT_VPP_JOB="${IS_CSIT_VPP_JOB:-}"
 MAKE_PARALLEL_FLAGS="${MAKE_PARALLEL_FLAGS:-}"
 MAKE_PARALLEL_JOBS="${MAKE_PARALLEL_JOBS:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
 
 echo "sha1sum of this script: ${0}"
 sha1sum $0
@@ -74,10 +77,22 @@ then
              "See test logs for the exact number."
     fi
     echo "Building using \"make verify\""
-    [ "${DRYRUN,,}" = "true" ] || make UNATTENDED=yes verify
+    if [[ "${DRYRUN,,}" != "true" ]] ; then
+        if ! make UNATTENDED=yes verify ; then
+            BUILD_ERROR="FAILED 'make verify'"
+        fi
+    fi
 else
     echo "Building using \"make pkg-verify\""
-    [ "${DRYRUN,,}" = "true" ] || make UNATTENDED=yes pkg-verify
+    if [[ "${DRYRUN,,}" != "true" ]] ; then
+        if ! make UNATTENDED=yes pkg-verify ; then
+            BUILD_ERROR="FAILED 'make pkg-verify'"
+        fi
+    fi
 fi
-
-echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED\n$line\n"
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh
index a9887340d..dba984f21 100644
--- a/jjb/scripts/vpp/debug-build.sh
+++ b/jjb/scripts/vpp/debug-build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,11 +15,16 @@
 
 echo "---> jjb/scripts/vpp/debug-build.sh"
 
-set -xe -o pipefail
+set -euxo pipefail
 
+line="*************************************************************************"
 OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
 
 echo "sha1sum of this script: ${0}"
 sha1sum $0
@@ -27,15 +32,34 @@ sha1sum $0
 # run with ASAN on
 export VPP_EXTRA_CMAKE_ARGS='-DVPP_ENABLE_SANITIZE_ADDR=ON'
 
+make_build_test_debug() {
+    if ! make UNATTENDED=yes install-dep ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make UNATTENDED=yes build ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+    if ! make UNATTENDED=yes TEST_JOBS=auto test-debug ; then
+        BUILD_ERROR="FAILED 'make test-debug'"
+        return
+    fi
+}
+
 # clang is not working with ASAN right now - see change 27268
 # also, it does not work with gcc-7, we need gcc-8 at least
 # on ubuntu 20.04 executor the gcc is gcc9
-
-make UNATTENDED=yes install-dep
-make UNATTENDED=yes install-ext-deps
-make UNATTENDED=yes build
-make UNATTENDED=yes TEST_JOBS=auto test-debug
-
-echo "*******************************************************************"
-echo "* VPP debug/asan test BUILD on ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} SUCCESSFULLY COMPLETED"
-echo "*******************************************************************"
+if [ "${DRYRUN,,}" != "true" ] ; then
+    make_build_test_debug
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} DEBUG BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh
index 5cdbc2633..488e8d7d7 100644
--- a/jjb/scripts/vpp/gcc-build.sh
+++ b/jjb/scripts/vpp/gcc-build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,27 +15,59 @@
 
 echo "---> jjb/scripts/vpp/gcc-build.sh"
 
-set -xe -o pipefail
+set -euxo pipefail
 
+line="*************************************************************************"
 OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
 
 echo "sha1sum of this script: ${0}"
 sha1sum $0
 export CC=gcc
 
-make UNATTENDED=yes install-dep
-make UNATTENDED=yes install-ext-deps
-make UNATTENDED=yes build
-# TODO: Add 'smoke test' env var to select smoke test cases
-#       then update this accordingly.  For now pick a few basic suites...
-MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash"
-for suite in $MAKE_TEST_SUITES ; do
-    make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test
-    make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug
-done
+make_build_release_build_test_gcov_sanity() {
+    if ! make UNATTENDED=yes install-dep ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make UNATTENDED=yes build-release ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+    if ! make UNATTENDED=yes build ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+    # TODO: Add 'smoke test' env var to select smoke test cases
+    #       then update this accordingly.  For now pick a few basic suites...
+    MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash"
+    for suite in $MAKE_TEST_SUITES ; do
+        if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test ; then
+            BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test'!"
+            return
+        fi
+        if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug ; then
+            BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug'!"
+            return
+        fi
+    done
+}
 
-echo "*******************************************************************"
-echo "* VPP GCC on ${OS_ID^^}-${OS_VERSION}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED"
-echo "*******************************************************************"
+if [ "${DRYRUN,,}" != "true" ] ; then
+    make_build_release_build_test_gcov_sanity
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} GCC BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL