Flags for parallel test run and arm verify update

Added a script where we can look at the label (flavor) of the jenkins
job and set the number of cores used in build/test for that label.

When setting the number of cores, the label along with the corresponding
hosts must ensure (by properly configuring them in nomad) that the
resources used by jobs on the host won't exceed host's resources.

Change-Id: I138c8ad797df84ebf9a926074a1a7cb5037bd681
Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>

3 files changed, 53 insertions, 8 deletions

diff --git a/jjb/vpp/include-raw-vpp-arm-build.sh b/jjb/vpp/include-raw-vpp-arm-build.sh
index c527b9efd..34c131344 100644
--- a/jjb/vpp/include-raw-vpp-arm-build.sh
+++ b/jjb/vpp/include-raw-vpp-arm-build.sh
@@ -8,6 +8,16 @@ OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\
 echo OS_ID: $OS_ID
 echo OS_VERSION_ID: $OS_VERSION_ID
 
+# do nothing but print the current slave hostname
+hostname
+export CCACHE_DIR=/tmp/ccache
+if [ -d $CCACHE_DIR ];then
+    echo $CCACHE_DIR exists
+    du -sk $CCACHE_DIR
+else
+    echo $CCACHE_DIR does not exist.  This must be a new slave.
+fi
+
 echo "cat /etc/bootstrap.sha"
 if [ -f /etc/bootstrap.sha ];then
     cat /etc/bootstrap.sha
@@ -26,14 +36,21 @@ echo "sha1sum of this script: ${0}"
 sha1sum $0
 
 echo "CC=${CC}"
-
-make UNATTENDED=yes install-dep
-make UNATTENDED=yes dpdk-install-dev
-make UNATTENDED=yes -C build-root PLATFORM=vpp TAG=vpp wipe-all install-packages
-make UNATTENDED=yes -C build-root PLATFORM=vpp TAG=vpp sample-plugin-install
-make UNATTENDED=yes -C build-root PLATFORM=vpp TAG=vpp libmemif-install
-make UNATTENDED=yes pkg-deb
-
+echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}"
+# If and only if we are doing verify *after* make verify was made to work
+# and we are not a CSIT job just building packages, then use make verify,
+# else use the old build-root/vagrant/build.sh
+if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \
+        [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ]
+then
+    echo "Building using \"make verify\""
+    [ "x${DRYRUN}" == "xTrue" ] || make TEST_JOBS=$TEST_JOBS UNATTENDED=yes verify
+else
+    echo "Building using \"make build-root/vagrant/build.sh\""
+    [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes install-dep
+    [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes dpdk-install-dev
+    [ "x${DRYRUN}" == "xTrue" ] || build-root/vagrant/build.sh
+fi
 
 if [ "x${VPP_REPO}" == "x1" ]; then
     if [ "x${REBASE_NEEDED}" == "x1" ]; then
diff --git a/jjb/vpp/include-raw-vpp-parallel-vars.sh b/jjb/vpp/include-raw-vpp-parallel-vars.sh
new file mode 100644
index 000000000..b4d5f209c
--- /dev/null
+++ b/jjb/vpp/include-raw-vpp-parallel-vars.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+CORES=4
+if [[ -z ${NODE_NAME+x} || -z ${NODE_LABELS+x} ]]
+then
+    echo "NODE_NAME or NODE_LABELS is not set, using defaults for parallel build/test"
+else
+    for NODE_LABEL in $NODE_LABELS
+    do
+        # NODE_LABELS is jenkins-SOMEHEX and the label we're looking for
+        # NODE_NAME is jenkins-SOMEHEX
+        if [[ $NODE_NAME != $NODE_LABEL ]]
+        then
+            # Found label such as ubuntu1604arm-us
+            break
+        fi
+    done
+
+    if [[ $NODE_LABEL == 'ubuntu1604arm-us' ]]
+    then
+        CORES=16
+    fi
+fi
+
+echo "Using MAKE_PARALLEL_FLAGS='-j $CORES' TEST_JOBS=$CORES for parallel build/test"
+
+export MAKE_PARALLEL_FLAGS="-j $CORES"
+TEST_JOBS=$CORES
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index ce72e4506..c1164cd38 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -301,6 +301,7 @@
         - shell:
             !include-raw-escape:
                 - ../scripts/setup_vpp_dpdk_dev_env.sh
+                - include-raw-vpp-parallel-vars.sh
                 - include-raw-vpp-arm-build.sh
     publishers:
         - archive: