Add the core file decoding+cleanup for the verify jobs

- Add better failure reporting and dry run handling
  for build scripts.

Change-Id: Ia19bae15ff4880b07094f4f665e5e00030eda27c
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

5 files changed, 187 insertions, 46 deletions

diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index 88f50176..a3c7661d 100755
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -60,6 +60,7 @@ perl -i -p -e "s/$jenkins_uid\:/0\:/g" /etc/passwd
 cp $DOCKER_CIMAN_ROOT/global-jjb/jenkins-init-scripts/lf-env.sh /root
 chmod 644 /root/lf-env.sh
 
-# Install lftools for log / artifact upload.
+# Install lftools & boto3 for log / artifact upload.
 source /root/lf-env.sh
 lf-activate-venv lftools
+python3 -m pip install boto3
diff --git a/jjb/scripts/post_build_deploy_archives.sh b/jjb/scripts/post_build_deploy_archives.sh
index 3f68e842..0a47903e 100644..100755
--- a/jjb/scripts/post_build_deploy_archives.sh
+++ b/jjb/scripts/post_build_deploy_archives.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -19,34 +19,100 @@ set +e  # Do not affect the build result if some part of archiving fails.
 WS_ARCHIVES_DIR="$WORKSPACE/archives"
 BUILD_ENV_LOG="$WS_ARCHIVES_DIR/_build-enviroment-variables.log"
 
+# Generate gdb-command script to output vpp stack traceback from core files.
+gdb_cmdfile="/tmp/gdb-commands"
+cat >$gdb_cmdfile <<'__END__'
+# Usage:
+# gdb $BINFILE $CORE -ex 'source -v gdb-commands' -ex quit
+
+set pagination off
+thread apply all bt
+
+define printstack
+  set $i=0
+  while $i < 15
+      frame $i
+      x/i $pc
+      info locals
+      info reg
+      set $i = $i + 1
+  end
+end
+thread apply all printstack
+
+# info proc mappings
+
+__END__
+
+STACKTRACE=""
+# Returns stacktrace filename in STACKTRACE
+generate_vpp_stacktrace_and_delete_core() {
+    local corefile="$1"
+    if grep -qe 'debug' <<< "$WORKSPACE" ; then
+        local binfile="$WORKSPACE/build-root/install-vpp_debug-native/vpp/bin/vpp"
+    else
+        local binfile="$WORKSPACE/build-root/install-vpp-native/vpp/bin/vpp"
+    fi
+
+    echo "Generating stack trace from core file: $corefile"
+    STACKTRACE="${corefile}.stacktrace"
+    gdb "$binfile" $corefile -ex 'source -v /tmp/gdb-commands' -ex quit > $STACKTRACE
+    # remove the core to save space
+    echo "Removing core file: $corefile"
+    rm -f "$corefile"
+    # Dump stacktrace to console log
+    if [ -f $STACKTRACE ] ; then
+        echo -e "\n=====[ $STACKTRACE ]=====\n$(cat $STACKTRACE)\n=====[ $STACKTRACE ]=====\n"
+    else
+        echo "Stacktrace file not generated!"
+        STACKTRACE=""
+    fi
+}
+
+# Delete existing archives dir to ensure current artifact upload
+rm -rf "$WS_ARCHIVES_DIR"
+mkdir -p "$WS_ARCHIVES_DIR"
+
 # Log the build environment variables
 echo "Logging build environment variables in '$BUILD_ENV_LOG'..."
-mkdir -p $WS_ARCHIVES_DIR
 env > $BUILD_ENV_LOG
 
 echo "WS_ARCHIVE_ARTIFACTS = '$WS_ARCHIVE_ARTIFACTS'"
-if [ ! -z "${WS_ARCHIVE_ARTIFACTS}" ]; then
+if [ -n "${WS_ARCHIVE_ARTIFACTS}" ]; then
     pushd $WORKSPACE
     shopt -s globstar  # Enable globstar to copy archives
     archive_artifacts=$(echo ${WS_ARCHIVE_ARTIFACTS})
+    shopt -u globstar  # Disable globstar
     for file in $archive_artifacts; do
         if [ -f "$file" ] ; then
-            echo "Archiving '$file'..."
-            mkdir -p $WS_ARCHIVES_DIR/$(dirname $file)
-            mv $file $WS_ARCHIVES_DIR/$file
+            fname="$(basename $file)"
+            # Decompress core.gz file
+            if grep -qe '^core.*\.gz$' <<<"$fname" ; then
+                echo "Uncompressing core file $file"
+                gunzip "$file"
+                file="${file::(-3)}"
+            fi
+            # Convert core file to stacktrace
+            if [ "${fname::4}" = "core" ] ; then
+                generate_vpp_stacktrace_and_delete_core $file
+                [ -z "$STACKTRACE" ] && continue
+                file=$STACKTRACE
+            fi
+            # Set destination filename
+            if [ "${file::26}" = "/tmp/vpp-failed-unittests/" ] ; then
+                destfile=$WS_ARCHIVES_DIR${file:25}
+            else
+                destfile=$WS_ARCHIVE_DIR$file
+            fi
+            echo "Archiving '$file' to '$destfile'"
+            destdir="$(dirname $destfile)"
+            mkdir -p $destdir
+            mv $file $destfile
         else
-            echo "Skipping '$file' which is not a file:"
-            ls -ld $file
+            echo "Not archiving '$file'"
         fi
     done
-    shopt -u globstar  # Disable globstar once archives are copied
     popd
-    # Clean up failed 'make test' archive directories for better
-    # navigation and legibility of log directories.
-    if [ -d "$WS_ARCHIVES_DIR/tmp" ] ; then
-        mv $WS_ARCHIVES_DIR/tmp/* $WS_ARCHIVES_DIR
-        rmdir $WS_ARCHIVES_DIR/tmp
-    fi
 fi
 
 # find and gzip any 'text' files
@@ -55,3 +121,6 @@ find $WS_ARCHIVES_DIR -type f -print0 \
                 | egrep -e ':.*text.*' \
                 | cut -d: -f1 \
                 | xargs -d'\n' -r gzip
+
+echo "Workspace archived artifacts:"
+ls -alR $WS_ARCHIVES_DIR
diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh
index 148999ce..409061b4 100644
--- a/jjb/scripts/vpp/build.sh
+++ b/jjb/scripts/vpp/build.sh
@@ -33,6 +33,9 @@ DRYRUN="${DRYRUN:-}"
 IS_CSIT_VPP_JOB="${IS_CSIT_VPP_JOB:-}"
 MAKE_PARALLEL_FLAGS="${MAKE_PARALLEL_FLAGS:-}"
 MAKE_PARALLEL_JOBS="${MAKE_PARALLEL_JOBS:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
 
 echo "sha1sum of this script: ${0}"
 sha1sum $0
@@ -74,10 +77,22 @@ then
              "See test logs for the exact number."
     fi
     echo "Building using \"make verify\""
-    [ "${DRYRUN,,}" = "true" ] || make UNATTENDED=yes verify
+    if [[ "${DRYRUN,,}" != "true" ]] ; then
+        if ! make UNATTENDED=yes verify ; then
+            BUILD_ERROR="FAILED 'make verify'"
+        fi
+    fi
 else
     echo "Building using \"make pkg-verify\""
-    [ "${DRYRUN,,}" = "true" ] || make UNATTENDED=yes pkg-verify
+    if [[ "${DRYRUN,,}" != "true" ]] ; then
+        if ! make UNATTENDED=yes pkg-verify ; then
+            BUILD_ERROR="FAILED 'make pkg-verify'"
+        fi
+    fi
 fi
-
-echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED\n$line\n"
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh
index a9887340..dba984f2 100644
--- a/jjb/scripts/vpp/debug-build.sh
+++ b/jjb/scripts/vpp/debug-build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,11 +15,16 @@
 
 echo "---> jjb/scripts/vpp/debug-build.sh"
 
-set -xe -o pipefail
+set -euxo pipefail
 
+line="*************************************************************************"
 OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
 
 echo "sha1sum of this script: ${0}"
 sha1sum $0
@@ -27,15 +32,34 @@ sha1sum $0
 # run with ASAN on
 export VPP_EXTRA_CMAKE_ARGS='-DVPP_ENABLE_SANITIZE_ADDR=ON'
 
+make_build_test_debug() {
+    if ! make UNATTENDED=yes install-dep ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make UNATTENDED=yes build ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+    if ! make UNATTENDED=yes TEST_JOBS=auto test-debug ; then
+        BUILD_ERROR="FAILED 'make test-debug'"
+        return
+    fi
+}
+
 # clang is not working with ASAN right now - see change 27268
 # also, it does not work with gcc-7, we need gcc-8 at least
 # on ubuntu 20.04 executor the gcc is gcc9
-
-make UNATTENDED=yes install-dep
-make UNATTENDED=yes install-ext-deps
-make UNATTENDED=yes build
-make UNATTENDED=yes TEST_JOBS=auto test-debug
-
-echo "*******************************************************************"
-echo "* VPP debug/asan test BUILD on ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} SUCCESSFULLY COMPLETED"
-echo "*******************************************************************"
+if [ "${DRYRUN,,}" != "true" ] ; then
+    make_build_test_debug
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} DEBUG BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh
index 5cdbc263..488e8d7d 100644
--- a/jjb/scripts/vpp/gcc-build.sh
+++ b/jjb/scripts/vpp/gcc-build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,27 +15,59 @@
 
 echo "---> jjb/scripts/vpp/gcc-build.sh"
 
-set -xe -o pipefail
+set -euxo pipefail
 
+line="*************************************************************************"
 OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
 
 echo "sha1sum of this script: ${0}"
 sha1sum $0
 export CC=gcc
 
-make UNATTENDED=yes install-dep
-make UNATTENDED=yes install-ext-deps
-make UNATTENDED=yes build
-# TODO: Add 'smoke test' env var to select smoke test cases
-#       then update this accordingly.  For now pick a few basic suites...
-MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash"
-for suite in $MAKE_TEST_SUITES ; do
-    make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test
-    make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug
-done
+make_build_release_build_test_gcov_sanity() {
+    if ! make UNATTENDED=yes install-dep ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make UNATTENDED=yes build-release ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+    if ! make UNATTENDED=yes build ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+    # TODO: Add 'smoke test' env var to select smoke test cases
+    #       then update this accordingly.  For now pick a few basic suites...
+    MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash"
+    for suite in $MAKE_TEST_SUITES ; do
+        if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test ; then
+            BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test'!"
+            return
+        fi
+        if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug ; then
+            BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug'!"
+            return
+        fi
+    done
+}
 
-echo "*******************************************************************"
-echo "* VPP GCC on ${OS_ID^^}-${OS_VERSION}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED"
-echo "*******************************************************************"
+if [ "${DRYRUN,,}" != "true" ] ; then
+    make_build_release_build_test_gcov_sanity
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} GCC BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL