Merge "feat(docker builder): add ubuntu 24.04 support"

10 files changed, 516 insertions, 82 deletions

diff --git a/docker/scripts/dbld_csit_install_packages.sh b/docker/scripts/dbld_csit_install_packages.sh
index e303ce28e..7f48d0751 100755
--- a/docker/scripts/dbld_csit_install_packages.sh
+++ b/docker/scripts/dbld_csit_install_packages.sh
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,8 +16,7 @@
 set -euxo pipefail
 
 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
-. "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
-. "$CIMAN_DOCKER_SCRIPTS/lib_vpp.sh"
+source "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
 
 must_be_run_in_docker_build
 
@@ -33,8 +32,11 @@ else
     echo_log "Starting  $(basename $0)"
 fi
 
+python3 -m venv "$DOCKER_BUILD_VENV_DIR"
+source "$DOCKER_BUILD_VENV_DIR"/bin/activate
+
 do_git_config csit
-for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do
+for vpp_branch in ${CSIT_VPP_BRANCHES[$OS_NAME]} ; do
     # Returns checked out branch in csit_branch
     csit_checkout_branch_for_vpp "$vpp_branch"
 
@@ -44,8 +46,10 @@ for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do
     # Install/cache python packages
     csit_install_hugo "$csit_branch"
 
+    deactivate
     # Install/cache python packages
     csit_pip_cache "$csit_branch"
+    source "$DOCKER_BUILD_VENV_DIR"/bin/activate
 done
 
 # Install csit OS packages
@@ -57,4 +61,5 @@ csit_install_hugo "master"
 # Install/cache python packages
 csit_pip_cache "master"
 
+deactivate
 echo_log -e "Completed $(basename $0)!\n\n=========="
diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index ca54edd89..b28fc0e44 100755
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -48,7 +48,11 @@ lf-activate-venv() {
 EOF
 
 # Install lftools & boto3 for log / artifact upload.
-python3 -m pip install boto3
+pip_options=""
+if [ "$OS_CODENAME" = "noble" ] ; then
+    pip_options=" --break-system-packages"
+fi
+python3 -m pip install$pip_options boto3
 mkdir -p "$LF_VENV"
 OLD_PATH="$PATH"
 python3 -m venv "$LF_VENV"
diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index f8a6f67be..d5ec1589f 100644
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -41,7 +41,6 @@ apt_install_packages() {
 generate_apt_dockerfile_common() {
     local executor_class="$1"
     local executor_image="$2"
-    local install_golang="$3"
     local dpkg_arch="$(dpkg --print-architecture)"
 
     cat <<EOF >>"$DOCKERFILE"
@@ -139,6 +138,7 @@ RUN apt-get update -qq \\
              make \\
              python3-pip \\
              python3-venv \\
+             python3-virtualenv \\
              rsync \\
              ruby-dev \\
              software-properties-common \\
@@ -163,9 +163,7 @@ RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$d
   && rm -f terraform_1.7.3_linux_$dpkg_arch.zip
 EOF
 
-    if [ "$install_golang" = "true" ] ; then
-        generate_apt_dockerfile_install_golang "1.21.11"
-    fi
+    generate_apt_dockerfile_install_golang
 
     cat <<EOF >>"$DOCKERFILE"
 
@@ -190,7 +188,6 @@ EOF
 }
 
 generate_apt_dockerfile_install_golang() {
-    local go_version="$1"
     local go_tarball_arch="amd64"
 
     if [ "$OS_ARCH" = "aarch64" ] ; then
@@ -205,9 +202,9 @@ ENV GOPATH /go
 ENV GOROOT /usr/local/go
 ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH
 RUN rm -rf /usr/local/go /usr/bin/go \\
-    && wget -P /tmp "https://go.dev/dl/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\
-    && tar -C /usr/local -xzf "/tmp/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\
-    && rm "/tmp/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\
+    && wget -P /tmp "https://go.dev/dl/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\
+    && tar -C /usr/local -xzf "/tmp/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\
+    && rm "/tmp/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\
     && ln -s /usr/local/go/bin/go /usr/bin/go \\
     && echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\
     && mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH"
@@ -219,10 +216,9 @@ builder_generate_apt_dockerfile() {
     local executor_class="$1"
     local executor_os_name="$2"
     local executor_image="$3"
-    local install_golang="$4"
     local vpp_install_skip_sysctl_envvar="";
 
-    generate_apt_dockerfile_common "$executor_class" "$executor_image" "$install_golang"
+    generate_apt_dockerfile_common "$executor_class" "$executor_image"
     csit_builder_generate_docker_build_files
     cat <<EOF >>"$DOCKERFILE"
 
@@ -375,13 +371,11 @@ generate_apt_dockerfile() {
     local executor_os_name="$2"
     local from_image="$3"
     local executor_image="$4"
-    local install_golang="false"
     local is_dind_image="false"
 
     # TODO: Enable HST on AARCH64 when supported in vpp/extras/hs-test
     if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then
         from_image="$DIND_FROM_IMAGE"
-        install_golang="true"
         is_dind_image="true"
     fi
 
@@ -403,5 +397,5 @@ EOF
     fi
 
     "${executor_class}"_generate_apt_dockerfile "$executor_class" \
-        "$executor_os_name" "$executor_image" "$install_golang"
+        "$executor_os_name" "$executor_image"
 }
diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh
index 166b06fbf..54d291e3f 100644
--- a/docker/scripts/lib_common.sh
+++ b/docker/scripts/lib_common.sh
@@ -1,7 +1,7 @@
 # lib_common.sh - Docker build script common library.
 #                 For import only.
 
-# Copyright (c) 2022 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -153,8 +153,8 @@ export EXECUTOR_CLASS_ARCH="$EXECUTOR_DEFAULT_CLASS-$OS_ARCH"
 export EXECUTOR_CLASSES="$EXECUTOR_DEFAULT_CLASS csit_dut csit_shim"
 export EXECUTOR_ARCHS="aarch64 x86_64"
 declare -A EXECUTOR_CLASS_ARCH_OS_NAMES
-EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="ubuntu-20.04 ubuntu-22.04"
-EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="debian-11 ubuntu-20.04 ubuntu-22.04"
+EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="ubuntu-20.04 ubuntu-22.04 ubuntu-24.04"
+EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="debian-11 ubuntu-20.04 ubuntu-22.04 ubuntu-24.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit_dut-aarch64"]="ubuntu-22.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit_dut-x86_64"]="ubuntu-22.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit_shim-aarch64"]="ubuntu-22.04"
@@ -226,6 +226,7 @@ executor_verify_os_name() {
 # Docker variables
 export DOCKER_DATE=${DOCKER_DATE:-"$(date -u +%Y_%m_%d_%H%M%S_UTC)"}
 export DOCKER_BUILD_DIR="/scratch/docker-build"
+export DOCKER_BUILD_VENV_DIR="$DOCKER_BUILD_DIR"/venv
 export DOCKER_CIMAN_ROOT="$DOCKER_BUILD_DIR/ci-management"
 export DOCKERFILE="$DOCKER_BUILD_DIR/Dockerfile"
 export DOCKERIGNOREFILE="$DOCKER_BUILD_DIR/.dockerignore"
@@ -236,6 +237,7 @@ export DOCKER_VPP_DL_CACHE_DIR="$DOCKER_BUILD_DIR"/vpp_ext_deps_downloads
 export DOCKER_CSIT_DIR="$DOCKER_BUILD_DIR/csit"
 export DOCKER_DOWNLOADS_DIR="/root/Downloads"
 export DOCKER_BUILD_FILES_DIR="$DOCKER_BUILD_DIR/files"
+export DOCKER_GOLANG_VERSION="1.22.5"
 
 docker_build_setup_ciman() {
     if [ "$(dirname $CIMAN_ROOT)" != "$DOCKER_BUILD_DIR" ] ; then
diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh
index 0499cbfc4..a4d543666 100644
--- a/docker/scripts/lib_csit.sh
+++ b/docker/scripts/lib_csit.sh
@@ -1,7 +1,7 @@
 # lib_csit.sh - Docker build script CSIT library.
 #               For import only.
 
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -24,6 +24,20 @@ export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]}
 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
 . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh"
 
+# Branches must be listed in chronological order -- oldest stable branch
+# first and master last.
+#
+# Note: CI Jobs for each architecture are maintained in
+#       .../ci-management/jjb/vpp/vpp.yaml
+#       All OS's and branches are included in the 'os' and 'stream'
+#       definitions respectively, then the exclude list maintained
+#       to create an enumerated set of jobs jobs that match the
+#       definitions here.
+declare -A CSIT_VPP_BRANCHES
+CSIT_VPP_BRANCHES["ubuntu-22.04"]="stable/2406 master"
+CSIT_VPP_BRANCHES["ubuntu-24.04"]="master"
+export CSIT_VPP_BRANCHES
+
 CSIT_SUPPORTED_EXECUTOR_CLASSES="builder csit_dut"
 csit_supported_executor_class() {
     if ! grep -q "${1:-}" <<< "$CSIT_SUPPORTED_EXECUTOR_CLASSES" ; then
@@ -35,6 +49,7 @@ csit_supported_executor_class() {
 csit_supported_os() {
     case "$1" in
         ubuntu-22.04) return 0 ;;
+        ubuntu-24.04) return 0 ;;
                    *) ;;
     esac
     return 1
@@ -67,7 +82,7 @@ csit_install_packages() {
     bld_log="${bld_log}-$branchname-csit_install_packages-bld.log"
 
     git clean -qfdx
-    python3 -m pip install pyyaml
+    pip install pyyaml
 
     local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp"
     [ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf"
@@ -75,9 +90,9 @@ csit_install_packages() {
     # Not in double quotes to let bash remove newline characters
     local yaml_files
     yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)"
-    packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files) | grep -v "$OS_CODENAME")"
-    packages="${packages/jammy /}"
+    packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)"
     packages="${packages/focal /}"
+    packages="${packages/noble /}"
     packages="${packages/libmbedcrypto1/libmbedcrypto3}"
     packages="${packages/libmbedtls10/libmbedtls12}"
     packages="$(echo ${packages//python\-/python3\-} | tr ' ' '\n' | sort -u | xargs)"
@@ -117,7 +132,6 @@ csit_install_hugo() {
         git clean -qfdx
 
         source "$csit_bash_function_dir"/hugo.sh
-        go_install 2>&1 | tee -a "$bld_log"
         hugo_install 2>&1 | tee -a "$bld_log"
 
     else
@@ -150,14 +164,11 @@ csit_pip_cache() {
         rm -rf "$PYTHONPATH/env"
 
         # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt)
-        local common_sh="$csit_bash_function_dir/common.sh"
-        # shellcheck disable=1090
-        source "$common_sh"
-        activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log"
+        source "$csit_bash_function_dir"/common.sh
+        activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}"/requirements.txt 2>&1 | tee -a "$bld_log"
 
         # Install tox python requirements
-        activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\
-            tee -a "$bld_log"
+        activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}"/tox-requirements.txt 2>&1 | tee -a "$bld_log"
 
         # Clean up virtualenv directories
         git checkout -q -- .
diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index 4bf8dab88..ff4ebd47f 100644
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -125,4 +125,5 @@ declare -A VPP_BRANCHES
 VPP_BRANCHES["debian-11"]="stable/2402 stable/2406 master"
 VPP_BRANCHES["ubuntu-20.04"]="stable/2402 stable/2406 master"
 VPP_BRANCHES["ubuntu-22.04"]="stable/2402 stable/2406 master"
+VPP_BRANCHES["ubuntu-24.04"]="master"
 export VPP_BRANCHES
diff --git a/jjb/scripts/publish_library_py.sh b/jjb/scripts/publish_library_py.sh
index 1cbeb23c0..db60c1dbe 100644
--- a/jjb/scripts/publish_library_py.sh
+++ b/jjb/scripts/publish_library_py.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -19,7 +19,13 @@ set -exuo pipefail
 
 PYTHON_SCRIPT="/w/workspace/publish_library.py"
 
-pip3 install boto3
+OS_CODENAME="$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d= -f2)"
+pip_options=""
+if [ "$OS_CODENAME" = "noble" ] ; then
+    pip_options=" --break-system-packages"
+fi
+# shellcheck disable=SC2086
+pip3 install$pip_options boto3
 mkdir -p $(dirname "$PYTHON_SCRIPT")
 
 cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT'
diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml
index 545789edc..860625d1f 100644
--- a/jjb/vpp/docs.yaml
+++ b/jjb/vpp/docs.yaml
@@ -8,19 +8,22 @@
       - 'vpp-docsonly-merge-{stream}-{os}-{executor-arch}'
 
     project: 'vpp'
-    executor-arch: 'x86_64'
-    os:
-      - 'ubuntu2204'
+    os: 'ubuntu2404'
     stream:
       - 'master':
           branch: 'master'
           branch-refspec: ''
+          # TODO: Remove when ubuntu 24.04 is put into production.
+          os: 'ubuntu2204'
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
+          os: 'ubuntu2204'
       - '2406':
           branch: 'stable/2406'
           branch-refspec: ''
+          os: 'ubuntu2204'
+    executor-arch: 'x86_64'
 
 - job-template:
     name: 'vpp-docs-verify-{stream}-{os}-{executor-arch}'
diff --git a/jjb/vpp/vpp-preprod.yaml b/jjb/vpp/vpp-preprod.yaml
new file mode 100644
index 000000000..a4685211c
--- /dev/null
+++ b/jjb/vpp/vpp-preprod.yaml
@@ -0,0 +1,385 @@
+# Copyright (c) 2024 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# PROJECT DEFINITIONS
+#
+# Please keep projects grouped in this order:
+#     vpp-x86_64, vpp-aarch64, vpp-csit
+#
+# NOTES:
+#   - The 'branch-refspec' attribute is used for sandbox testing.
+#         Use the get_gerrit_refspec() bash function in
+#         .../ci-management/extras/bash/sandbox_test_functions.sh
+#         to query gerrit for the refspec of HEAD in a branch.
+#         TODO: see if there is a refspec that works without requiring
+#               use of get_gerrit_refspec and hand entering the result
+#               in the WebUI 'Build With Parameters' gerrit refspec field.
+# [end] NOTES
+
+# VPP-X86_64 PROJECT
+#
+# NOTE: The list of supported branches for each OS are maintained
+#       in .../ci-management/docker/scripts/lib_vpp.sh
+#       The job configuration must be derived from that list, thus
+#       modifications to both should be made at the same time.
+#       Since x86_64 is the primary development architecture, this
+#       project definition should be identical to the docker image
+#       build scripts.  Other architectures may be a subset.
+- project:
+    name: vpp-preprod-x86_64
+    jobs:
+      - 'vpp-preprod-verify-{stream}-{os}-{executor-arch}'
+      - 'vpp-preprod-merge-{stream}-{os}-{executor-arch}'
+    project: 'vpp'
+    os:
+      - 'ubuntu2404'
+    make-parallel-jobs: '4'
+    make-test-multiworker-os: 'debian-11'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
+    stream:
+      - 'master':
+          branch: 'master'
+          branch-refspec: ''
+    executor-arch: 'x86_64'
+# [end] VPP-X86_64 PROJECT
+
+# VPP-GCC-X86_64 PROJECT
+- project:
+    name: vpp-preprod-gcc-x86_64
+    jobs:
+      - 'vpp-preprod-gcc-verify-{stream}-{os}-{executor-arch}'
+    project: 'vpp'
+    os:
+      - 'ubuntu2404'
+    make-parallel-jobs: '4'
+    executor-arch: 'x86_64'
+    stream:
+      - 'master':
+          branch: 'master'
+          branch-refspec: ''
+# [end] VPP-GCC-X86_64 PROJECT
+
+# VPP-DEBUG-X86_64 PROJECT
+#
+# Currently a periodic job due to intermittent failures.
+# Limiting branches to master, since other branches
+# may not get fixes for the intermittent failures.
+#
+- project:
+    name: vpp-preprod-debug-x86_64
+    jobs:
+      - 'vpp-preprod-debug-verify-{stream}-{os}-{executor-arch}'
+    project: 'vpp'
+    os: 'ubuntu2404'
+    make-parallel-jobs: '4'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
+    comment-trigger-value: 'debug-verify'
+    stream:
+      - 'master':
+          branch: 'master'
+          branch-refspec: ''
+    executor-arch: 'x86_64'
+
+# VPP-AARCH64 PROJECT
+#
+# NOTE: The list of supported branches for each OS are maintained
+#       in .../ci-management/docker/scripts/lib_vpp.sh
+#       The job configuration must be derived from that list, thus
+#       modifications to both should be made at the same time.
+#       The AARCH64 architecture is in the process of catching up
+#       with X86_64 in terms of OS coverage.  Thus the jobs are
+#       a subset of those defined in the docker image build script.
+- project:
+    name: vpp-preprod-aarch64
+    jobs:
+      - 'vpp-preprod-verify-{stream}-{os}-{executor-arch}'
+      - 'vpp-preprod-merge-{stream}-{os}-{executor-arch}'
+    project: 'vpp'
+    os:
+      - 'ubuntu2404'
+    make-parallel-jobs: '4'
+    make-test-multiworker-os: 'debian-11'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
+    stream:
+      - 'master':
+          branch: 'master'
+          branch-refspec: ''
+    executor-arch: 'aarch64'
+# [end] VPP-AARCH64 PROJECT
+
+# JOB TEMPLATE DEFINITIONS
+#
+# Please keep job templates in the same order as they are referenced
+# in the project definitions above.
+
+# VPP-VERIFY JOB TEMPLATE
+- job-template:
+    name: 'vpp-preprod-verify-{stream}-{os}-{executor-arch}'
+
+    project-type: freestyle
+    node: 'builder-{os}-prod-{executor-arch}'
+    concurrent: true
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    # Please keep parameters in alphabetical order
+    parameters:
+      - gerrit-parameter:
+          branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
+      - make-test-os-parameter:
+          make-test-os: '{make-test-os}'
+      - make-test-multiworker-os-parameter:
+          make-test-multiworker-os: '{make-test-multiworker-os}'
+      - os-parameter:
+          os: '{os}'
+      - project-parameter:
+          project: '{project}'
+      - stream-parameter:
+          stream: '{stream}'
+
+    scm:
+      - gerrit-trigger-scm:
+          credentials-id: 'jenkins-gerrit-credentials'
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - gerrit-trigger-checkstyle-nonvoting:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_executor_env.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ubuntu_docker_test.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ext_deps.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/build.sh
+
+    publishers:
+      - fdio-infra-publish
+# [end] VPP-VERIFY JOB TEMPLATE
+
+# VPP-DEBUG-VERIFY JOB TEMPLATE
+- job-template:
+    name: 'vpp-preprod-debug-verify-{stream}-{os}-{executor-arch}'
+
+    project-type: freestyle
+    node: 'builder-{os}-prod-{executor-arch}'
+    concurrent: true
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    # Please keep parameters in alphabetical order
+    parameters:
+      - gerrit-parameter:
+          branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
+      - make-test-os-parameter:
+          make-test-os: '{make-test-os}'
+      - os-parameter:
+          os: '{os}'
+      - project-parameter:
+          project: '{project}'
+      - stream-parameter:
+          stream: '{stream}'
+
+    scm:
+      - gerrit-trigger-scm:
+          credentials-id: 'jenkins-gerrit-credentials'
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-checkstyle-nonvoting:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_executor_env.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ubuntu_docker_test.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ext_deps.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/debug-build.sh
+
+    publishers:
+      - fdio-infra-publish
+# [end] VPP-DEBUG-VERIFY JOB TEMPLATE
+
+# VPP-MERGE JOB TEMPLATE
+- job-template:
+    name: 'vpp-preprod-merge-{stream}-{os}-{executor-arch}'
+
+    project-type: freestyle
+    node: 'builder-{os}-prod-{executor-arch}'
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    # Please keep parameters in alphabetical order
+    parameters:
+      - gerrit-parameter:
+          branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
+      - make-test-os-parameter:
+          make-test-os: '{make-test-os}'
+      - make-test-multiworker-os-parameter:
+          make-test-multiworker-os: '{make-test-multiworker-os}'
+      - os-parameter:
+          os: '{os}'
+      - project-parameter:
+          project: '{project}'
+      - stream-parameter:
+          stream: '{stream}'
+
+    scm:
+      - gerrit-trigger-scm:
+          credentials-id: 'jenkins-gerrit-credentials'
+          refspec: ''
+          choosing-strategy: 'default'
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - gerrit-trigger-patch-merged:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: '.packagecloud'
+              target: '/root/.packagecloud'
+      - config-file-provider:
+          files:
+            - file-id: 'packagecloud_api'
+              target: '/root/packagecloud_api'
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_executor_env.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/commitmsg.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ubuntu_docker_test.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ext_deps.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/build.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/packagecloud_push.sh
+
+    publishers:
+      - fdio-infra-publish
+# [end] VPP-MERGE JOB TEMPLATE
+
+# VPP-GCC-VERIFY JOB TEMPLATE
+- job-template:
+    name: 'vpp-preprod-gcc-verify-{stream}-{os}-{executor-arch}'
+
+    project-type: freestyle
+    node: 'builder-{os}-prod-{executor-arch}'
+    concurrent: true
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+
+    # Please keep parameters in alphabetical order
+    parameters:
+      - gerrit-parameter:
+          branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
+      - os-parameter:
+          os: '{os}'
+      - project-parameter:
+          project: '{project}'
+      - stream-parameter:
+          stream: '{stream}'
+
+    scm:
+      - gerrit-trigger-scm:
+          credentials-id: 'jenkins-gerrit-credentials'
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - gerrit-trigger-checkstyle-nonvoting:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_executor_env.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ubuntu_docker_test.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/setup_vpp_ext_deps.sh
+      - shell:
+          !include-raw-escape:
+          - ../scripts/vpp/gcc-build.sh
+
+    publishers:
+      - fdio-infra-publish
+# [end] VPP-GCC-VERIFY JOB TEMPLATE
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 3ec3e0f40..e8832d0c1 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -34,15 +34,19 @@
       - 'vpp-checkstyle-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     os:
-      - ubuntu2204
+      - ubuntu2404
     stream:
       - 'master':
+          # TODO: Remove when ubuntu2404 is put into production
+          os: 'ubuntu2204'
           branch: 'master'
           branch-refspec: ''
       - '2402':
+          os: 'ubuntu2204'
           branch: 'stable/2402'
           branch-refspec: ''
       - '2406':
+          os: 'ubuntu2204'
           branch: 'stable/2406'
           branch-refspec: ''
     executor-arch: 'x86_64'
@@ -75,6 +79,8 @@
       - ubuntu2204
     stream:
       - 'master':
+          # TODO: Remove when ubuntu2404 is put into production
+          os: 'ubuntu2204'
           branch: 'master'
           branch-refspec: ''
     executor-arch: 'x86_64'
@@ -96,28 +102,34 @@
       - 'vpp-verify-{stream}-{os}-{executor-arch}'
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
+    os:
+      - 'debian11'
+      - 'ubuntu2004'
+      - 'ubuntu2204'
+      # TODO: Uncomment when ubuntu 24.04 is put into production
+      #- 'ubuntu2404'
     make-parallel-jobs: '4'
+    make-test-multiworker-os: 'debian-11'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
     stream:
       - 'master':
           branch: 'master'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: 'debian-11'
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: 'debian-11'
       - '2406':
           branch: 'stable/2406'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: 'debian-11'
-    os:
-      - debian11
-      - ubuntu2004
-      - ubuntu2204
     executor-arch: 'x86_64'
+    exclude:
+        # ubuntu-24.04 introduced in VPP 24.10
+        - stream: '2402'
+          os: 'ubuntu2404'
+        # ubuntu-24.04 introduced in VPP 24.10
+        - stream: '2406'
+          os: 'ubuntu2404'
+
 # [end] VPP-X86_64 PROJECT
 
 # VPP-GCC-X86_64 PROJECT
@@ -126,9 +138,11 @@
     jobs:
       - 'vpp-gcc-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
-    make-parallel-jobs: '4'
     os:
-      - ubuntu2204
+      - 'ubuntu2004'
+      # TODO: Uncomment when ubuntu 24.04 is put into production
+      #- 'ubuntu2404'
+    make-parallel-jobs: '4'
     executor-arch: 'x86_64'
     stream:
       - 'master':
@@ -140,6 +154,13 @@
       - '2406':
           branch: 'stable/2406'
           branch-refspec: ''
+    exclude:
+        # ubuntu-24.04 introduced in VPP 24.10
+        - stream: '2402'
+          os: 'ubuntu2404'
+        # ubuntu-24.04 introduced in VPP 24.10
+        - stream: '2406'
+          os: 'ubuntu2404'
 
 # [end] VPP-GCC-X86_64 PROJECT
 
@@ -154,15 +175,16 @@
     jobs:
       - 'vpp-debug-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
+    os: 'ubuntu2404'
     make-parallel-jobs: '4'
-    make-test-os: 'ubuntu-20.04 ubuntu-22.04'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
     comment-trigger-value: 'debug-verify'
     stream:
       - 'master':
+          # TODO: Remove when ubuntu2404 is put into production
+          os: 'ubuntu2204'
           branch: 'master'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-    os: 'ubuntu2204'
     executor-arch: 'x86_64'
 
 # [end] VPP-DEBUG-X86_64 PROJECT
@@ -178,14 +200,15 @@
     jobs:
       - 'vpp-dpdk-rdma-version-mismatch-{stream}-{os}-{executor-arch}'
     project: 'vpp'
-    make-test-os: 'ubuntu-20.04 ubuntu-22.04'
+    os: 'ubuntu2404'
     comment-trigger-value: 'dpdk-rdma-version-mismatch'
     stream:
       - 'master':
+          # TODO: Remove when ubuntu2404 is put into production
+          os: 'ubuntu2204'
           branch: 'master'
           # Once per day at 00:15
           periodicity: '15 0 * * 0-6'
-    os: 'ubuntu2204'
     executor-arch: 'x86_64'
 
 # [end] VPP-DPDK-RDMA-VERSION-MISMATCH-X86_64 PROJECT
@@ -202,16 +225,18 @@
     jobs:
       - 'vpp-cov-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
+    os: 'ubuntu2404'
     make-parallel-jobs: '4'
-    make-test-os: 'ubuntu-20.04 ubuntu-22.04'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
     comment-trigger-value: 'gcov-report'
     stream:
       - 'master':
+          # TODO: Remove when ubuntu2404 is put into production
+          os: 'ubuntu2204'
           branch: 'master'
           branch-refspec: ''
           # Once per day at 00:25
           periodicity: '25 0 * * 0-6'
-    os: 'ubuntu2004'
     executor-arch: 'x86_64'
 
 # [end] VPP-COV-X86_64 PROJECT
@@ -231,27 +256,32 @@
       - 'vpp-verify-{stream}-{os}-{executor-arch}'
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
+    os:
+      - ubuntu2004
+      - ubuntu2204
+      # TODO: Uncomment when ubuntu 24.04 is put into production
+      #- 'ubuntu2404'
     make-parallel-jobs: '4'
+    make-test-multiworker-os: 'debian-11'
+    make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04'
     stream:
       - 'master':
           branch: 'master'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: ''
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: ''
       - '2406':
           branch: 'stable/2406'
           branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: ''
-    os:
-      - ubuntu2004
-      - ubuntu2204
     executor-arch: 'aarch64'
+    exclude:
+        # ubuntu-24.04 introduced in VPP 24.10
+        - stream: '2402'
+          os: 'ubuntu2404'
+        # ubuntu-24.04 introduced in VPP 24.10
+        - stream: '2406'
+          os: 'ubuntu2404'
 # [end] VPP-AARCH64 PROJECT
 
 # VPP-CSIT-VERIFY-DEVICE PROJECT
@@ -261,29 +291,23 @@
       - 'vpp-csit-verify-device-{stream}-{os}-{executor-arch}-{testbed-arch}':
           make-parallel-jobs: '32'
     project: 'vpp'
+    os: 'ubuntu2404'
     skip-vote: 'false'
     stream:
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2402':
-          branch: 'stable/2402'
-          branch-refspec: ''
-      - '2406':
-          branch: 'stable/2406'
-          branch-refspec: ''
     testbed-arch:
       - 1n-spr:
+          # TODO: Remove when ubuntu2404 is put into production
           os: 'ubuntu2204'
           executor-arch: 'x86_64'
           skip-vote: 'false'
       - 1n-alt:
+          # TODO: Remove when ubuntu2404 is put into production
           os: 'ubuntu2204'
           executor-arch: 'aarch64'
           skip-vote: 'false'
-    exclude:
-      - testbed-arch: 1n-spr
-        stream: '2402'
 # [end] VPP-CSIT-VERIFY-DEVICE PROJECT
 
 # VPP-CSIT-VERIFY-PERF PROJECT
@@ -338,20 +362,23 @@
     jobs:
       - 'vpp-csit-verify-api-crc-{stream}-{os}-{executor-arch}'
     project: 'vpp'
+    os: ubuntu2404
     executor-arch: 'x86_64'
     skip-vote: 'false'
     stream:
       - 'master':
           branch: 'master'
           branch-refspec: ''
+          # TODO: Remove when ubuntu2404 is put into production
+          os: 'ubuntu2204'
       - '2402':
           branch: 'stable/2402'
           branch-refspec: ''
+          os: 'ubuntu2204'
       - '2406':
           branch: 'stable/2406'
           branch-refspec: ''
-    os:
-      - 'ubuntu2204'
+          os: 'ubuntu2204'
 # [end] VPP-CSIT-API-CRC PROJECT
 
 # LF RELENG VPP PROJECTS
@@ -745,10 +772,6 @@
     parameters:
       - gerrit-parameter:
           branch: '{branch}'
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - make-test-os-parameter:
-          make-test-os: '{make-test-os}'
       - os-parameter:
           os: '{os}'
       - project-parameter: