vpp-hst: add VPP HostStack Test framework jobs

- add hst and hst-debug per-patch vpp verify jobs
- reduce idle termination time to 1 minute
- fix vpp script permissions to 755 so all are the same
- use docker-in-docker base image for ubuntu-22.04 containers

Change-Id: I10396c823fbeaa5e081e48332509e0c20b31b87c
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

18 files changed, 358 insertions, 20 deletions

diff --git a/docker/scripts/build_executor_docker_image.sh b/docker/scripts/build_executor_docker_image.sh
index 7a731956e..30cc6bdd5 100755
--- a/docker/scripts/build_executor_docker_image.sh
+++ b/docker/scripts/build_executor_docker_image.sh
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -105,7 +105,7 @@ docker_build_setup_ciman
 docker_build_setup_vpp
 docker_build_setup_csit
 for executor_os_name in $os_names ; do
-    docker_from_image="$(echo $executor_os_name | sed -e 's/-/:/')"
+    docker_from_image="${executor_os_name/-/:}"
     # Remove '-' and '.' from executor_os_name in Docker Hub repo name
     os_name="${executor_os_name//-}"
     repository="fdiotools/${EXECUTOR_CLASS}-${os_name//.}"
diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh
index 5e1e0f8f0..fecd823dd 100755
--- a/docker/scripts/dbld_vpp_install_packages.sh
+++ b/docker/scripts/dbld_vpp_install_packages.sh
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,7 +15,7 @@
 
 set -euxo pipefail
 
-export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
+export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname "${BASH_SOURCE[0]}")"}
 . "$CIMAN_DOCKER_SCRIPTS/lib_vpp.sh"
 . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh"
 
@@ -23,18 +23,23 @@ must_be_run_in_docker_build
 
 echo_log
 if ! vpp_supported_executor_class "$FDIOTOOLS_EXECUTOR_CLASS" ; then
-    echo_log "VPP is not supported on executor class '$FDIOTOOLS_EXECUTOR_CLASS'. Skipping $(basename $0)..."
+    echo_log "VPP is not supported on executor class '$FDIOTOOLS_EXECUTOR_CLASS'. Skipping $(basename "$0")..."
     exit 0
 else
-    echo_log "Starting  $(basename $0)"
+    echo_log "Starting  $(basename "$0")"
 fi
 
 do_git_config vpp
 for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
     do_git_branch "$branch"
 
+    # Install hs-test depndencies
+    if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then
+        install_hst_deps "$branch"
+    fi
+
     # Install OS packages
-    make_vpp "install-dep" "$branch"
+    make_vpp "install-deps" "$branch"
 
     # Download, build, and cache external deps packages
     make_vpp "install-ext-deps" "$branch"
@@ -47,7 +52,7 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
         exit 1
     fi
     if [ -f "$vpp_ext_deps_pkg" ] ; then
-        cp -f $vpp_ext_deps_pkg $DOCKER_DOWNLOADS_DIR
+        cp -f "$vpp_ext_deps_pkg" $DOCKER_DOWNLOADS_DIR
     else
         echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'"
         exit 1
@@ -76,4 +81,4 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
     esac
 done
 
-echo_log -e "Completed $(basename $0)!\n\n=========="
+echo_log -e "Completed $(basename "$0")!\n\n=========="
diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index 6cf37ae76..a1ee6db7c 100644
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -1,7 +1,7 @@
 # lib_apt.sh - Docker build script apt library.
 #              For import only.
 
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -20,6 +20,9 @@ if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
 fi
 alias lib_apt_imported=true
 
+DIND_FROM_IMAGE="cruizba/ubuntu-dind:latest"
+HST_FROM_IMAGE="ubuntu:22.04"
+
 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
 . "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
@@ -163,6 +166,7 @@ RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$d
 RUN apt-get update -qq \\
   && dbld_vpp_install_packages.sh \\
   && dbld_csit_install_packages.sh \\
+  && apt-get install -y pkg-config \\
   && rm -r /var/lib/apt/lists/*
 EOF
 }
@@ -172,19 +176,44 @@ generate_apt_dockerfile_clean() {
 
 # Clean up copy-in build tree
 RUN dbld_dump_build_logs.sh \\
+  && apt-get -y autoremove \\
   && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"
 EOF
 }
 
+generate_apt_dockerfile_install_golang() {
+    local GO_VERSION="$1"
+
+    cat <<EOF >>"$DOCKERFILE"
+
+# Install golang for HostStack Test (HST) jobs
+#
+ENV GOPATH /go
+ENV GOROOT /usr/local/go
+ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH
+RUN rm -rf /usr/local/go /usr/bin/go \\
+    && wget -P /tmp "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" \\
+    && tar -C /usr/local -xzf "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\
+    && rm "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\
+    && ln -s /usr/local/go/bin/go /usr/bin/go \\
+    && echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\
+    && mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH"
+EOF
+}
+
 # Generate 'builder' class apt dockerfile
 builder_generate_apt_dockerfile() {
     local executor_class="$1"
     local executor_os_name="$2"
     local executor_image="$3"
+    local install_golang="$4"
     local vpp_install_skip_sysctl_envvar="";
 
-    generate_apt_dockerfile_common $executor_class $executor_image
+    generate_apt_dockerfile_common "$executor_class" "$executor_image"
     csit_builder_generate_docker_build_files
+    if [ "$install_golang" = "true" ] ; then
+        generate_apt_dockerfile_install_golang "1.21.9"
+    fi
     cat <<EOF >>"$DOCKERFILE"
 
 # Install LF-IT requirements
@@ -336,6 +365,14 @@ generate_apt_dockerfile() {
     local executor_os_name="$2"
     local from_image="$3"
     local executor_image="$4"
+    local install_golang="false"
+    local is_dind_image="false"
+
+    if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then
+        from_image="$DIND_FROM_IMAGE"
+        install_golang="true"
+        is_dind_image="true"
+    fi
 
     cat <<EOF  >"$DOCKERIGNOREFILE"
 **/__pycache__
@@ -347,6 +384,13 @@ LABEL Description="FD.io CI '$executor_class' executor docker image for $executo
 LABEL Vendor="fd.io"
 LABEL Version="$DOCKER_TAG"
 EOF
-    ${executor_class}_generate_apt_dockerfile "$executor_class" \
-        "$executor_os_name" "$executor_image"
+    if [ "$is_dind_image" = "true" ] ; then
+        cat <<EOF >>"$DOCKERFILE"
+# Remove extraneous environment variable(s) created in "$from_image"
+ENV DEBUG=
+EOF
+    fi
+
+    "${executor_class}"_generate_apt_dockerfile "$executor_class" \
+        "$executor_os_name" "$executor_image" "$install_golang"
 }
diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index 39e2ca192..a4efbd119 100644
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -1,7 +1,7 @@
 # lib_vpp.sh - Docker build script VPP library.
 #              For import only.
 
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -20,8 +20,8 @@ if [ -n "$(alias lib_vpp_imported 2> /dev/null)" ] ; then
 fi
 alias lib_vpp_imported=true
 
-export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
-. $CIMAN_DOCKER_SCRIPTS/lib_common.sh
+export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname "${BASH_SOURCE[0]}")"}
+. "$CIMAN_DOCKER_SCRIPTS"/lib_common.sh
 
 
 VPP_SUPPORTED_EXECUTOR_CLASSES="builder"
@@ -32,10 +32,23 @@ vpp_supported_executor_class() {
     return 0
 }
 
+install_hst_deps() {
+    local branch=${1:-"master"}
+    local branchname=${branch/\//_}
+    local hst_dir="./extras/hs-test"
+    local bld_log="$DOCKER_BUILD_LOG_DIR"
+    bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
+    bld_log="${bld_log}-install_hst_deps_bld.log"
+
+    if [ -d "$hst_dir" ] ; then
+        make -C "$hst_dir" install-deps 2>&1 | tee -a "$bld_log"
+    fi
+}
+
 make_vpp() {
     local target=$1
     local branch=${2:-"master"}
-    local branchname="$(echo $branch | sed -e 's,/,_,')"
+    local branchname=${branch/\//_}
     local bld_log="$DOCKER_BUILD_LOG_DIR"
     bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
     bld_log="${bld_log}-make_vpp_${target}-bld.log"
@@ -48,7 +61,7 @@ make_vpp() {
     git clean -qfdx
     description="'make UNATTENDED=yes $target' in $(pwd) ($branch)"
     echo_log -e "    Starting  $description..."
-    make UNATTENDED=yes $target 2>&1 | tee -a "$bld_log"
+    make UNATTENDED=yes "$target" 2>&1 | tee -a "$bld_log"
     git checkout -q -- .
     echo_log "    Completed $description!"
 }
@@ -56,7 +69,7 @@ make_vpp() {
 make_vpp_test() {
     local target=$1
     local branch=${2:-"master"}
-    local branchname="$(echo $branch | sed -e 's,/,_,')"
+    local branchname=${branch/\//_}
     local bld_log="$DOCKER_BUILD_LOG_DIR"
     bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
     bld_log="${bld_log}-make_vpp_test_${target}-bld.log"
diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml
index e868998b1..25240c3ba 100644
--- a/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml
+++ b/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml
@@ -1,6 +1,6 @@
 ---
 templates:
-  - idleTerminationInMinutes: 10
+  - idleTerminationInMinutes: 1
     numExecutors: 1
     remoteFs: "/w"
     reusable: false
diff --git a/jjb/scripts/vpp/api-checkstyle.sh b/jjb/scripts/vpp/api-checkstyle.sh
index a1c2f84a1..a1c2f84a1 100644..100755
--- a/jjb/scripts/vpp/api-checkstyle.sh
+++ b/jjb/scripts/vpp/api-checkstyle.sh
diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh
index 850d61003..850d61003 100644..100755
--- a/jjb/scripts/vpp/build.sh
+++ b/jjb/scripts/vpp/build.sh
diff --git a/jjb/scripts/vpp/commitmsg.sh b/jjb/scripts/vpp/commitmsg.sh
index 479f35136..479f35136 100644..100755
--- a/jjb/scripts/vpp/commitmsg.sh
+++ b/jjb/scripts/vpp/commitmsg.sh
diff --git a/jjb/scripts/vpp/copy_archives.sh b/jjb/scripts/vpp/copy_archives.sh
index dafcd4aef..dafcd4aef 100644..100755
--- a/jjb/scripts/vpp/copy_archives.sh
+++ b/jjb/scripts/vpp/copy_archives.sh
diff --git a/jjb/scripts/vpp/cov-build.sh b/jjb/scripts/vpp/cov-build.sh
index 15f86c3bf..15f86c3bf 100644..100755
--- a/jjb/scripts/vpp/cov-build.sh
+++ b/jjb/scripts/vpp/cov-build.sh
diff --git a/jjb/scripts/vpp/csit-bisect.sh b/jjb/scripts/vpp/csit-bisect.sh
index 0a264dbc8..0a264dbc8 100644..100755
--- a/jjb/scripts/vpp/csit-bisect.sh
+++ b/jjb/scripts/vpp/csit-bisect.sh
diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh
index 68338df56..68338df56 100644..100755
--- a/jjb/scripts/vpp/debug-build.sh
+++ b/jjb/scripts/vpp/debug-build.sh
diff --git a/jjb/scripts/vpp/debug-hst.sh b/jjb/scripts/vpp/debug-hst.sh
new file mode 100755
index 000000000..e57ab4c22
--- /dev/null
+++ b/jjb/scripts/vpp/debug-hst.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright (c) 2024 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/vpp/debug-hst.sh"
+
+set -euxo pipefail
+
+line="*************************************************************************"
+OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
+HST_DIR="./extras/hs-test"
+
+vpp_make_build_debug() {
+    if ! make UNATTENDED=yes install-dep ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make UNATTENDED=yes build ; then
+        BUILD_ERROR="FAILED 'make build'"
+        return
+    fi
+}
+
+hst_test_debug() {
+    if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" build-debug test ; then
+        BUILD_ERROR="FAILED 'make -C $HST_DIR test' (debug)"
+        return
+    fi
+}
+
+if [ "${DRYRUN,,}" != "true" ] ; then
+    vpp_make_build_debug
+    hst_test_debug
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \
+        "BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh b/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh
index b2248064b..b2248064b 100644..100755
--- a/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh
+++ b/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh
diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh
index 9bd6d4ff7..9bd6d4ff7 100644..100755
--- a/jjb/scripts/vpp/gcc-build.sh
+++ b/jjb/scripts/vpp/gcc-build.sh
diff --git a/jjb/scripts/vpp/hst.sh b/jjb/scripts/vpp/hst.sh
new file mode 100755
index 000000000..08f075b68
--- /dev/null
+++ b/jjb/scripts/vpp/hst.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright (c) 2024 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/vpp/hst.sh"
+
+set -euxo pipefail
+
+line="*************************************************************************"
+OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
+OS_ARCH=$(uname -m)
+DRYRUN="${DRYRUN:-}"
+BUILD_RESULT="SUCCESSFULLY COMPLETED"
+BUILD_ERROR=""
+RETVAL="0"
+HST_DIR="./extras/hs-test"
+
+vpp_make_build() {
+    if ! make UNATTENDED=yes install-deps ; then
+        BUILD_ERROR="FAILED 'make install-dep'"
+        return
+    fi
+    if ! make UNATTENDED=yes install-ext-deps ; then
+        BUILD_ERROR="FAILED 'make install-ext-deps'"
+        return
+    fi
+    if ! make build-release; then
+        BUILD_ERROR="FAILED 'make build-release'"
+        return
+    fi
+    if ! make -C "$HST_DIR" install-deps build ; then
+        BUILD_ERROR="FAILED 'make -C $HST_DIR install-deps build'"
+        return
+    fi
+}
+
+hst_test() {
+    if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" build test ; then
+        BUILD_ERROR="FAILED 'make -C $HST_DIR test'"
+        return
+    fi
+}
+
+if [ "${DRYRUN,,}" != "true" ] ; then
+    vpp_make_build
+    hst_test
+fi
+if [ -n "$BUILD_ERROR" ] ; then
+    BUILD_RESULT="$BUILD_ERROR"
+    RETVAL="1"
+fi
+echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \
+        "BUILD $BUILD_RESULT\n$line\n"
+exit $RETVAL
diff --git a/jjb/scripts/vpp/test-checkstyle.sh b/jjb/scripts/vpp/test-checkstyle.sh
index 3f0bc9a06..3f0bc9a06 100644..100755
--- a/jjb/scripts/vpp/test-checkstyle.sh
+++ b/jjb/scripts/vpp/test-checkstyle.sh
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 62975f004..fc6d20889 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -49,6 +49,38 @@
 
 # [end] VPP-CHECKSTYLE-X86_64 PROJECT
 
+# VPP-HST-X86_64 PROJECT
+- project:
+    name: vpp-verify-hst-x86_64
+    jobs:
+      - 'vpp-verify-hst-{stream}-{os}-{executor-arch}'
+    project: 'vpp'
+    os:
+      - ubuntu2204
+    stream:
+      - 'master':
+          branch: 'master'
+          branch-refspec: ''
+    executor-arch: 'x86_64'
+
+# [end] VPP-HST-X86_64 PROJECT
+
+# VPP-HST-DEBUG-X86_64 PROJECT
+- project:
+    name: vpp-verify-hst-debug-x86_64
+    jobs:
+      - 'vpp-verify-hst-debug-{stream}-{os}-{executor-arch}'
+    project: 'vpp'
+    os:
+      - ubuntu2204
+    stream:
+      - 'master':
+          branch: 'master'
+          branch-refspec: ''
+    executor-arch: 'x86_64'
+
+# [end] VPP-HST-DEBUG-X86_64 PROJECT
+
 # VPP-X86_64 PROJECT
 #
 # NOTE: The list of supported branches for each OS are maintained
@@ -468,6 +500,122 @@
 
 # [end] VPP-CHECKSTYLE-VERIFY JOB TEMPLATE
 
+# VPP-VERIFY-HST JOB TEMPLATE
+- job-template:
+    name: 'vpp-verify-hst-{stream}-{os}-{executor-arch}'
+
+    project-type: freestyle
+    node: 'hst-{os}-prod-{executor-arch}'
+    concurrent: true
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    # Please keep parameters in alphabetical order
+    parameters:
+      - gerrit-parameter:
+          branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-refspec}'
+      - os-parameter:
+          os: '{os}'
+      - project-parameter:
+          project: '{project}'
+      - stream-parameter:
+          stream: '{stream}'
+
+    scm:
+      - gerrit-trigger-scm:
+          credentials-id: 'jenkins-gerrit-credentials'
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - gerrit-trigger-checkstyle:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_executor_env.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ubuntu_docker_test.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ext_deps.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/hst.sh
+
+    publishers:
+      - fdio-infra-publish
+# [end] VPP-VERIFY-HST JOB TEMPLATE
+
+# VPP-VERIFY-HST-DEBUG JOB TEMPLATE
+- job-template:
+    name: 'vpp-verify-hst-debug-{stream}-{os}-{executor-arch}'
+
+    project-type: freestyle
+    node: 'hst-{os}-prod-{executor-arch}'
+    concurrent: true
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    # Please keep parameters in alphabetical order
+    parameters:
+      - gerrit-parameter:
+          branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-refspec}'
+      - os-parameter:
+          os: '{os}'
+      - project-parameter:
+          project: '{project}'
+      - stream-parameter:
+          stream: '{stream}'
+
+    scm:
+      - gerrit-trigger-scm:
+          credentials-id: 'jenkins-gerrit-credentials'
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - gerrit-trigger-checkstyle:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_executor_env.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ubuntu_docker_test.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ext_deps.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/debug-hst.sh
+
+    publishers:
+      - fdio-infra-publish
+# [end] VPP-VERIFY-HST-DEBUG JOB TEMPLATE
+
 # VPP-VERIFY JOB TEMPLATE
 - job-template:
     name: 'vpp-verify-{stream}-{os}-{executor-arch}'