Replace vpp-gcc job with vpp-cov job per-patch

- the original gcc job was intended to ensure that gcc
  builds worked to generate test coverage reports.  The
  vpp-cov-verify job produces test coverage reports
  per patch and stores them in the S3 7-day docs
  retention bucket.

Change-Id: I28c7c323ea8d673cdee66b85189e20029e63336d
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

5 files changed, 223 insertions, 115 deletions

diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml
index 508a6217..25d2785c 100644
--- a/jjb/global-macros.yaml
+++ b/jjb/global-macros.yaml
@@ -801,6 +801,33 @@
           fail-build: false
 
 - publisher:
+    name: fdio-infra-publish-cov-7day
+    # macro to finish up a verify docs build.
+    #
+    # Handles the following:
+    #   - Mapping 7-day retention S3 bucket credentials for verify job docs upload
+    #   - Cleanup workspace
+    publishers:
+      - postbuildscript:
+          builders:
+            - role: BOTH
+              build-on:
+                - ABORTED
+                - FAILURE
+                - NOT_BUILT
+                - SUCCESS
+                - UNSTABLE
+              build-steps:
+                - fdio-infra-ship-cov-7day
+          mark-unstable-if-failed: true
+      - workspace-cleanup:
+          exclude:
+            # Do not clean up *.jenkins-trigger files for jobs that use a
+            # properties file as input for triggering another build.
+            - "**/*.jenkins-trigger"
+          fail-build: false
+
+- publisher:
     name: fdio-infra-publish-docs-7day
     # macro to finish up a verify docs build.
     #
@@ -912,6 +939,20 @@
           - ../global-jjb/shell/logs-clear-credentials.sh
 
 - builder:
+    name: fdio-infra-ship-cov-7day
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: "jenkins-s3-vpp-docs-ship"
+              target: $HOME/.aws/credentials
+      - shell: !include-raw:
+          - scripts/terraform_s3_docs_ship.sh
+      - shell: !include-raw:
+          - scripts/publish_cov.sh
+      - shell: !include-raw:
+          - ../global-jjb/shell/logs-clear-credentials.sh
+
+- builder:
     name: fdio-infra-ship-docs-7day
     builders:
       - config-file-provider:
diff --git a/jjb/scripts/publish_cov.sh b/jjb/scripts/publish_cov.sh
new file mode 100644
index 00000000..cdbbcdd8
--- /dev/null
+++ b/jjb/scripts/publish_cov.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> publish_cov.sh"
+
+set -exuo pipefail
+
+CDN_URL="s3-docs-7day.fd.io"
+bucket="vpp-docs-7day-retention"
+# Use the same bucket path as logs so that the code coverage report can be viewed by
+# s/s3-logs/s3-docs-7day/ in the URL after selecting the logs URL from
+# the jenkins job page.
+bucket_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/"
+
+if [[ ${JOB_NAME} == *verify* ]]; then
+    case "${JOB_NAME}" in
+        *"vpp-cov"*)
+            workspace_dir="${WORKSPACE}/build-root/test-coverage/html"
+            ;;
+        *)
+            die "Unknown job: ${JOB_NAME}"
+    esac
+else
+    die "Unknown job: ${JOB_NAME}"
+fi
+
+export TF_VAR_workspace_dir="$workspace_dir"
+export TF_VAR_bucket_path="$bucket_path"
+export TF_VAR_bucket="$bucket"
+export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/credentials
+export AWS_DEFAULT_REGION="us-east-1"
+
+echo "INFO: archiving test coverage to S3 bucket '$bucket'"
+pushd ..
+terraform init -no-color
+terraform apply -no-color -auto-approve
+popd
+
+echo "S3 Test Coverage: <a href=\"https://${CDN_URL}/${bucket_path}\">https://${CDN_URL}/${bucket_path}</a>"
diff --git a/jjb/scripts/vpp/cov-build.sh b/jjb/scripts/vpp/cov-build.sh
new file mode 100644
index 00000000..b5fdeb59
--- /dev/null
+++ b/jjb/scripts/vpp/cov-build.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/vpp/cov-build.sh"
+
+set -euxo pipefail
+
+line="*************************************************************************"
+# Don't build anything if this is a merge job.
+if [[ ${JOB_NAME} == *merge* ]] ; then
+    echo -e "\n$line\nSkipping build."
+    exit 0
+fi
+
+OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+IS_CSIT_VPP_JOB="${IS_CSIT_VPP_JOB:-}"
+MAKE_PARALLEL_FLAGS="${MAKE_PARALLEL_FLAGS:-}"
+MAKE_PARALLEL_JOBS="${MAKE_PARALLEL_JOBS:-}"
+MAKE_TEST_OS="${MAKE_TEST_OS:-ubuntu-22.04}"
+MAKE_TEST_MULTIWORKER_OS="${MAKE_TEST_MULTIWORKER_OS:-debian-11}"
+VPPAPIGEN_TEST_OS="${VPPAPIGEN_TEST_OS:-${MAKE_TEST_OS}}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
+
+if [ -n "${MAKE_PARALLEL_FLAGS}" ] ; then
+  echo "Building VPP. Number of cores for build set with" \
+       "MAKE_PARALLEL_FLAGS='${MAKE_PARALLEL_FLAGS}'."
+elif [ -n "${MAKE_PARALLEL_JOBS}" ] ; then
+  echo "Building VPP. Number of cores for build set with" \
+       "MAKE_PARALLEL_JOBS='${MAKE_PARALLEL_JOBS}'."
+else
+    echo "Building VPP. Number of cores not set," \
+         "using build default ($(grep -c ^processor /proc/cpuinfo))."
+fi
+
+make_test_coverage_report() {
+    if ! make UNATTENDED=yes install-dep ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make UNATTENDED=yes test-dep ; then
+        BUILD_ERROR="FAILED 'make test-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes pkg-verify ; then
+        BUILD_ERROR="FAILED 'make pkg-verify'"
+	    return
+    fi
+    if [ "${IS_CSIT_VPP_JOB,,}" == "true" ] ; then
+	    # CSIT jobs don't need to run make test
+	    return
+    fi
+    if [ -n "${MAKE_PARALLEL_JOBS}" ] ; then
+        TEST_JOBS="${MAKE_PARALLEL_JOBS}"
+        echo "Testing VPP with ${TEST_JOBS} cores."
+    else
+        TEST_JOBS="auto"
+        echo "Testing VPP with automatically calculated number of cores. " \
+             "See test logs for the exact number."
+    fi
+    if grep -q "${OS_ID}-${OS_VERSION_ID}" <<< "${VPPAPIGEN_TEST_OS}"; then
+        if ! src/tools/vppapigen/test_vppapigen.py ; then
+            BUILD_ERROR="FAILED src/tools/vppapigen/test_vppapigen.py"
+            return
+        fi
+    fi
+    if grep -q "${OS_ID}-${OS_VERSION_ID}" <<< "${MAKE_TEST_OS}"; then
+        if ! make COMPRESS_FAILED_TEST_LOGS=yes TEST_JOBS="$TEST_JOBS" RETRIES=3 test-cov ; then
+            BUILD_ERROR="FAILED 'make test-cov'"
+            return
+        fi
+    else
+        echo "Skip running 'make test-cov' on ${OS_ID}-${OS_VERSION_ID}"
+    fi
+}
+
+if [ "${DRYRUN,,}" != "true" ] ; then
+    make_test_coverage_report
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \
+        "TEST COVERAGE REPORT $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh
deleted file mode 100644
index 1d816004..00000000
--- a/jjb/scripts/vpp/gcc-build.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2022 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "---> jjb/scripts/vpp/gcc-build.sh"
-
-set -euxo pipefail
-
-line="*************************************************************************"
-OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
-OS_ARCH=$(uname -m)
-DRYRUN="${DRYRUN:-}"
-BUILD_RESULT="SUCCESSFULLY COMPLETED"
-BUILD_ERROR=""
-RETVAL="0"
-export CC=gcc
-
-make_build_release_build_test_gcov_sanity() {
-    if ! make UNATTENDED=yes install-dep ; then
-        BUILD_ERROR="FAILED 'make install-dep'"
-        return
-    fi
-    if ! make UNATTENDED=yes install-ext-deps ; then
-        BUILD_ERROR="FAILED 'make install-ext-deps'"
-        return
-    fi
-    if ! make UNATTENDED=yes build-release ; then
-        BUILD_ERROR="FAILED 'make build'"
-        return
-    fi
-    if ! make UNATTENDED=yes build ; then
-        BUILD_ERROR="FAILED 'make build'"
-        return
-    fi
-    if [ -n "${MAKE_PARALLEL_JOBS}" ] ; then
-        TEST_JOBS="${MAKE_PARALLEL_JOBS}"
-        echo "Testing VPP with ${TEST_JOBS} cores."
-    else
-        TEST_JOBS="auto"
-        echo "Testing VPP with automatically calculated number of cores. " \
-             "See test logs for the exact number."
-    fi
-    # TODO: Add 'smoke test' env var to select smoke test cases
-    #       then update this accordingly.  For now pick a few basic suites...
-    MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi cli bihash"
-    for suite in $MAKE_TEST_SUITES ; do
-        if ! make UNATTENDED=yes TESTS_GCOV=1 TEST_JOBS="$TEST_JOBS" TEST=$suite test ; then
-            BUILD_ERROR="FAILED 'make TESTS_GCOV=1 TEST_JOBS=$TEST_JOBS TEST=$suite test'!"
-            return
-        fi
-        if ! make UNATTENDED=yes TESTS_GCOV=1 TEST_JOBS="$TEST_JOBS" TEST=$suite test-debug ; then
-            BUILD_ERROR="FAILED 'make TESTS_GCOV=1 TEST_JOBS=$TEST_JOBS TEST=$suite test-debug'!"
-            return
-        fi
-    done
-}
-
-if [ "${DRYRUN,,}" != "true" ] ; then
-    make_build_release_build_test_gcov_sanity
-fi
-if [ -n "$BUILD_ERROR" ] ; then
-    BUILD_RESULT="$BUILD_ERROR"
-    RETVAL="1"
-fi
-echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} GCC BUILD $BUILD_RESULT\n$line\n"
-exit $RETVAL
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 55b0b1ad..28a48781 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -110,6 +110,8 @@
     executor-arch: 'x86_64'
 # [end] VPP-X86_64 PROJECT
 
+# [end] VPP-COV-X86_64 PROJECT
+
 # VPP-DEBUG-X86_64 PROJECT
 #
 # Currently a periodic job due to intermittent failures.
@@ -157,44 +159,27 @@
 
 # [end] VPP-DPDK-RDMA-VERSION-MISMATCH-X86_64 PROJECT
 
-# VPP-GCC-X86_64 PROJECT
+# VPP-COV-X86_64 PROJECT
+#
+# A per-patch job to produce code coverage.
+#
+# TODO: switch OS to ubuntu-22.04 once the skipping of intermittently failing
+#       tests are removed
+#
 - project:
-    name: vpp-gcc-x86_64
+    name: vpp-cov-x86_64
     jobs:
-      - 'vpp-gcc-verify-{stream}-{os}-{executor-arch}'
+      - 'vpp-cov-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
-    os:
-      - ubuntu2004
-      - ubuntu2204
-    executor-arch: 'x86_64'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04'
+    comment-trigger-value: 'cov-verify'
     stream:
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2210':
-          branch: 'stable/2210'
-          branch-refspec: ''
-      - '2302':
-          branch: 'stable/2302'
-          branch-refspec: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
-    exclude:
-      # os deprecated in 23.06
-      - stream: '2306'
-        os: 'ubuntu2004'
-      # os deprecated in 23.02
-      - stream: '2302'
-        os: 'ubuntu2004'
-      # os deprecated in 22.10
-      - stream: '2210'
-        os: 'ubuntu2004'
-      # os deprecated in master
-      - stream: 'master'
-        os: 'ubuntu2004'
-# [end] VPP-GCC-X86_64 PROJECT
+    os: 'ubuntu2004'
+    executor-arch: 'x86_64'
 
 # VPP-AARCH64 PROJECT
 #
@@ -593,7 +578,7 @@
 
     publishers:
       - fdio-infra-publish
-# [end] VPP-DEBUG-VERIFY-PERIODIC JOB TEMPLATE
+# [end] VPP-DEBUG-VERIFY JOB TEMPLATE
 
 # VPP-DPDK-RDMA-VERSION-MISMATCH PERIODIC JOB TEMPLATE
 - job-template:
@@ -731,9 +716,9 @@
       - fdio-infra-publish
 # [end] VPP-MERGE JOB TEMPLATE
 
-# VPP-GCC-VERIFY JOB TEMPLATE
+# VPP-COV-VERIFY JOB TEMPLATE
 - job-template:
-    name: 'vpp-gcc-verify-{stream}-{os}-{executor-arch}'
+    name: 'vpp-cov-verify-{stream}-{os}-{executor-arch}'
 
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
@@ -751,6 +736,8 @@
           refspec: '{branch-refspec}'
       - make-parallel-jobs-parameter:
           make-parallel-jobs: '{make-parallel-jobs}'
+      - make-test-os-parameter:
+          make-test-os: '{make-test-os}'
       - os-parameter:
           os: '{os}'
       - project-parameter:
@@ -766,10 +753,10 @@
 
     wrappers:
       - fdio-infra-wrappers:
-          build-timeout: 120
+          build-timeout: '{build-timeout}'
 
     triggers:
-      - gerrit-trigger-checkstyle:
+      - gerrit-trigger-checkstyle-nonvoting:
           name: '{project}'
           branch: '{branch}'
 
@@ -785,11 +772,12 @@
           - ../scripts/setup_vpp_ext_deps.sh
       - shell:
           !include-raw-escape:
-          - ../scripts/vpp/gcc-build.sh
+          - ../scripts/vpp/cov-build.sh
 
     publishers:
+      - fdio-infra-publish-cov-7day
       - fdio-infra-publish
-# [end] VPP-GCC-VERIFY JOB TEMPLATE
+# [end] VPP-COV-VERIFY JOB TEMPLATE
 
 # VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE
 - job-template: