diff options
73 files changed, 1092 insertions, 827 deletions
diff --git a/docker/scripts/build_executor_docker_image.sh b/docker/scripts/build_executor_docker_image.sh index 7a731956e..30cc6bdd5 100755 --- a/docker/scripts/build_executor_docker_image.sh +++ b/docker/scripts/build_executor_docker_image.sh @@ -1,6 +1,6 @@ #! /bin/bash -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -105,7 +105,7 @@ docker_build_setup_ciman docker_build_setup_vpp docker_build_setup_csit for executor_os_name in $os_names ; do - docker_from_image="$(echo $executor_os_name | sed -e 's/-/:/')" + docker_from_image="${executor_os_name/-/:}" # Remove '-' and '.' from executor_os_name in Docker Hub repo name os_name="${executor_os_name//-}" repository="fdiotools/${EXECUTOR_CLASS}-${os_name//.}" diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py index 52ed4c3b0..50b7d8478 100755 --- a/docker/scripts/dbld_csit_find_ansible_packages.py +++ b/docker/scripts/dbld_csit_find_ansible_packages.py @@ -37,6 +37,7 @@ class CsitAnsibleYamlStruct: self.__dict__.update(entries) def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list: + codename = os.getenv("OS_CODENAME") with open(yamlfile) as yf: csit_ansible_yaml = yaml.safe_load(yf) if csit_ansible_yaml is None: @@ -50,7 +51,11 @@ def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list: packages += [pkg for pkg in cays.packages_by_arch[arch] if type(pkg) is str] if distro in [*cays.packages_by_distro]: - packages += [pkg for pkg in cays.packages_by_distro[distro] + if codename in [*cays.packages_by_distro[distro]]: + packages += [pkg for pkg in cays.packages_by_distro[distro][codename] + if type(pkg) is str] + else: + packages += [pkg for pkg in cays.packages_by_distro[distro] if type(pkg) is str] return packages diff --git a/docker/scripts/dbld_csit_install_packages.sh b/docker/scripts/dbld_csit_install_packages.sh index e303ce28e..7f48d0751 100755 --- a/docker/scripts/dbld_csit_install_packages.sh +++ b/docker/scripts/dbld_csit_install_packages.sh @@ -1,6 +1,6 @@ #! /bin/bash -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,8 +16,7 @@ set -euxo pipefail export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"} -. "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh" -. "$CIMAN_DOCKER_SCRIPTS/lib_vpp.sh" +source "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh" must_be_run_in_docker_build @@ -33,8 +32,11 @@ else echo_log "Starting $(basename $0)" fi +python3 -m venv "$DOCKER_BUILD_VENV_DIR" +source "$DOCKER_BUILD_VENV_DIR"/bin/activate + do_git_config csit -for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do +for vpp_branch in ${CSIT_VPP_BRANCHES[$OS_NAME]} ; do # Returns checked out branch in csit_branch csit_checkout_branch_for_vpp "$vpp_branch" @@ -44,8 +46,10 @@ for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do # Install/cache python packages csit_install_hugo "$csit_branch" + deactivate # Install/cache python packages csit_pip_cache "$csit_branch" + source "$DOCKER_BUILD_VENV_DIR"/bin/activate done # Install csit OS packages @@ -57,4 +61,5 @@ csit_install_hugo "master" # Install/cache python packages csit_pip_cache "master" +deactivate echo_log -e "Completed $(basename $0)!\n\n==========" diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh index ca54edd89..c67c2e579 100755 --- a/docker/scripts/dbld_lfit_requirements.sh +++ b/docker/scripts/dbld_lfit_requirements.sh @@ -1,6 +1,6 @@ #! /bin/bash -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -48,7 +48,11 @@ lf-activate-venv() { EOF # Install lftools & boto3 for log / artifact upload. -python3 -m pip install boto3 +pip_options="" +if [ "$OS_CODENAME" = "noble" ] || [ "$OS_CODENAME" = "bookworm" ] ; then + pip_options=" --break-system-packages" +fi +python3 -m pip install$pip_options boto3 mkdir -p "$LF_VENV" OLD_PATH="$PATH" python3 -m venv "$LF_VENV" diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh index 5e1e0f8f0..be8b34cdc 100755 --- a/docker/scripts/dbld_vpp_install_packages.sh +++ b/docker/scripts/dbld_vpp_install_packages.sh @@ -1,6 +1,6 @@ #! /bin/bash -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,7 +15,7 @@ set -euxo pipefail -export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"} +export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname "${BASH_SOURCE[0]}")"} . "$CIMAN_DOCKER_SCRIPTS/lib_vpp.sh" . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh" @@ -23,10 +23,10 @@ must_be_run_in_docker_build echo_log if ! vpp_supported_executor_class "$FDIOTOOLS_EXECUTOR_CLASS" ; then - echo_log "VPP is not supported on executor class '$FDIOTOOLS_EXECUTOR_CLASS'. Skipping $(basename $0)..." + echo_log "VPP is not supported on executor class '$FDIOTOOLS_EXECUTOR_CLASS'. Skipping $(basename "$0")..." exit 0 else - echo_log "Starting $(basename $0)" + echo_log "Starting $(basename "$0")" fi do_git_config vpp @@ -34,7 +34,7 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do do_git_branch "$branch" # Install OS packages - make_vpp "install-dep" "$branch" + make_vpp "install-deps" "$branch" # Download, build, and cache external deps packages make_vpp "install-ext-deps" "$branch" @@ -47,11 +47,21 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do exit 1 fi if [ -f "$vpp_ext_deps_pkg" ] ; then - cp -f $vpp_ext_deps_pkg $DOCKER_DOWNLOADS_DIR + cp -f "$vpp_ext_deps_pkg" $DOCKER_DOWNLOADS_DIR else echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'" exit 1 fi + + # Install hs-test depndencies + if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then + make_vpp build "$branch" "false" + make_vpp build-release "$branch" "false" + make_vpp build-vpp-gcov "$branch" "false" + install_hst_deps "$branch" + git clean -qfdx + fi + # TODO: remove this after all supported VPP branches have removed # python3-virtualenv & virtualenv from install-deps which are no longer # used in vpp repo. These packages can mess up csit virtualenv @@ -76,4 +86,4 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do esac done -echo_log -e "Completed $(basename $0)!\n\n==========" +echo_log -e "Completed $(basename "$0")!\n\n==========" diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh index 6cf37ae76..d5ec1589f 100644 --- a/docker/scripts/lib_apt.sh +++ b/docker/scripts/lib_apt.sh @@ -1,7 +1,7 @@ # lib_apt.sh - Docker build script apt library. # For import only. -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -20,6 +20,9 @@ if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then fi alias lib_apt_imported=true +DIND_FROM_IMAGE="cruizba/ubuntu-dind:jammy-26.1.3-r2" +HST_FROM_IMAGE="ubuntu:22.04" + export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"} . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh" . "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh" @@ -135,6 +138,7 @@ RUN apt-get update -qq \\ make \\ python3-pip \\ python3-venv \\ + python3-virtualenv \\ rsync \\ ruby-dev \\ software-properties-common \\ @@ -157,12 +161,18 @@ RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$d && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\ && mv terraform /usr/bin \\ && rm -f terraform_1.7.3_linux_$dpkg_arch.zip +EOF + + generate_apt_dockerfile_install_golang + + cat <<EOF >>"$DOCKERFILE" # Install packages for all project branches # RUN apt-get update -qq \\ && dbld_vpp_install_packages.sh \\ && dbld_csit_install_packages.sh \\ + && apt-get install -y pkg-config \\ && rm -r /var/lib/apt/lists/* EOF } @@ -172,10 +182,35 @@ generate_apt_dockerfile_clean() { # Clean up copy-in build tree RUN dbld_dump_build_logs.sh \\ + && apt-get -y autoremove \\ && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache" EOF } +generate_apt_dockerfile_install_golang() { + local go_tarball_arch="amd64" + + if [ "$OS_ARCH" = "aarch64" ] ; then + go_tarball_arch="arm64" + fi + + cat <<EOF >>"$DOCKERFILE" + +# Install golang for HostStack Test (HST) jobs +# +ENV GOPATH /go +ENV GOROOT /usr/local/go +ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH +RUN rm -rf /usr/local/go /usr/bin/go \\ + && wget -P /tmp "https://go.dev/dl/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\ + && tar -C /usr/local -xzf "/tmp/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\ + && rm "/tmp/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\ + && ln -s /usr/local/go/bin/go /usr/bin/go \\ + && echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\ + && mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH" +EOF +} + # Generate 'builder' class apt dockerfile builder_generate_apt_dockerfile() { local executor_class="$1" @@ -183,7 +218,7 @@ builder_generate_apt_dockerfile() { local executor_image="$3" local vpp_install_skip_sysctl_envvar=""; - generate_apt_dockerfile_common $executor_class $executor_image + generate_apt_dockerfile_common "$executor_class" "$executor_image" csit_builder_generate_docker_build_files cat <<EOF >>"$DOCKERFILE" @@ -336,6 +371,13 @@ generate_apt_dockerfile() { local executor_os_name="$2" local from_image="$3" local executor_image="$4" + local is_dind_image="false" + + # TODO: Enable HST on AARCH64 when supported in vpp/extras/hs-test + if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then + from_image="$DIND_FROM_IMAGE" + is_dind_image="true" + fi cat <<EOF >"$DOCKERIGNOREFILE" **/__pycache__ @@ -347,6 +389,13 @@ LABEL Description="FD.io CI '$executor_class' executor docker image for $executo LABEL Vendor="fd.io" LABEL Version="$DOCKER_TAG" EOF - ${executor_class}_generate_apt_dockerfile "$executor_class" \ + if [ "$is_dind_image" = "true" ] ; then + cat <<EOF >>"$DOCKERFILE" +# Remove extraneous environment variable(s) created in "$from_image" +ENV DEBUG= +EOF + fi + + "${executor_class}"_generate_apt_dockerfile "$executor_class" \ "$executor_os_name" "$executor_image" } diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh index 150441bb7..916191b80 100644 --- a/docker/scripts/lib_common.sh +++ b/docker/scripts/lib_common.sh @@ -1,7 +1,7 @@ # lib_common.sh - Docker build script common library. # For import only. -# Copyright (c) 2022 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -153,8 +153,8 @@ export EXECUTOR_CLASS_ARCH="$EXECUTOR_DEFAULT_CLASS-$OS_ARCH" export EXECUTOR_CLASSES="$EXECUTOR_DEFAULT_CLASS csit_dut csit_shim" export EXECUTOR_ARCHS="aarch64 x86_64" declare -A EXECUTOR_CLASS_ARCH_OS_NAMES -EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="ubuntu-20.04 ubuntu-22.04" -EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="debian-11 ubuntu-20.04 ubuntu-22.04" +EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="ubuntu-20.04 ubuntu-22.04 ubuntu-24.04" +EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="debian-11 debian-12 ubuntu-20.04 ubuntu-22.04 ubuntu-24.04" EXECUTOR_CLASS_ARCH_OS_NAMES["csit_dut-aarch64"]="ubuntu-22.04" EXECUTOR_CLASS_ARCH_OS_NAMES["csit_dut-x86_64"]="ubuntu-22.04" EXECUTOR_CLASS_ARCH_OS_NAMES["csit_shim-aarch64"]="ubuntu-22.04" @@ -226,15 +226,18 @@ executor_verify_os_name() { # Docker variables export DOCKER_DATE=${DOCKER_DATE:-"$(date -u +%Y_%m_%d_%H%M%S_UTC)"} export DOCKER_BUILD_DIR="/scratch/docker-build" +export DOCKER_BUILD_VENV_DIR="$DOCKER_BUILD_DIR"/venv export DOCKER_CIMAN_ROOT="$DOCKER_BUILD_DIR/ci-management" export DOCKERFILE="$DOCKER_BUILD_DIR/Dockerfile" export DOCKERIGNOREFILE="$DOCKER_BUILD_DIR/.dockerignore" export DOCKERFILE_FROM=${DOCKERFILE_FROM:="${OS_ID}:${OS_VERSION_ID}"} export DOCKER_TAG="$DOCKER_DATE-$OS_ARCH" export DOCKER_VPP_DIR="$DOCKER_BUILD_DIR/vpp" +export DOCKER_VPP_DL_CACHE_DIR="$DOCKER_BUILD_DIR"/vpp_ext_deps_downloads export DOCKER_CSIT_DIR="$DOCKER_BUILD_DIR/csit" export DOCKER_DOWNLOADS_DIR="/root/Downloads" export DOCKER_BUILD_FILES_DIR="$DOCKER_BUILD_DIR/files" +export DOCKER_GOLANG_VERSION="1.22.5" docker_build_setup_ciman() { if [ "$(dirname $CIMAN_ROOT)" != "$DOCKER_BUILD_DIR" ] ; then diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh index 2e877f803..06cfb471b 100644 --- a/docker/scripts/lib_csit.sh +++ b/docker/scripts/lib_csit.sh @@ -1,7 +1,7 @@ # lib_csit.sh - Docker build script CSIT library. # For import only. -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -24,6 +24,20 @@ export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]} . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh" . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh" +# Branches must be listed in chronological order -- oldest stable branch +# first and master last. +# +# Note: CI Jobs for each architecture are maintained in +# .../ci-management/jjb/vpp/vpp.yaml +# All OS's and branches are included in the 'os' and 'stream' +# definitions respectively, then the exclude list maintained +# to create an enumerated set of jobs jobs that match the +# definitions here. +declare -A CSIT_VPP_BRANCHES +CSIT_VPP_BRANCHES["ubuntu-22.04"]="stable/2406 master" +CSIT_VPP_BRANCHES["ubuntu-24.04"]="master" +export CSIT_VPP_BRANCHES + CSIT_SUPPORTED_EXECUTOR_CLASSES="builder csit_dut" csit_supported_executor_class() { if ! grep -q "${1:-}" <<< "$CSIT_SUPPORTED_EXECUTOR_CLASSES" ; then @@ -35,6 +49,7 @@ csit_supported_executor_class() { csit_supported_os() { case "$1" in ubuntu-22.04) return 0 ;; + ubuntu-24.04) return 0 ;; *) ;; esac return 1 @@ -67,7 +82,7 @@ csit_install_packages() { bld_log="${bld_log}-$branchname-csit_install_packages-bld.log" git clean -qfdx - python3 -m pip install pyyaml + pip install pyyaml local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp" [ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf" @@ -78,6 +93,7 @@ csit_install_packages() { packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)" packages="${packages/jammy /}" packages="${packages/focal /}" + packages="${packages/noble /}" packages="${packages/libmbedcrypto1/libmbedcrypto3}" packages="${packages/libmbedtls10/libmbedtls12}" packages="$(echo ${packages//python\-/python3\-} | tr ' ' '\n' | sort -u | xargs)" @@ -117,7 +133,6 @@ csit_install_hugo() { git clean -qfdx source "$csit_bash_function_dir"/hugo.sh - go_install 2>&1 | tee -a "$bld_log" hugo_install 2>&1 | tee -a "$bld_log" else @@ -150,14 +165,11 @@ csit_pip_cache() { rm -rf "$PYTHONPATH/env" # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt) - local common_sh="$csit_bash_function_dir/common.sh" - # shellcheck disable=1090 - source "$common_sh" - activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log" + source "$csit_bash_function_dir"/common.sh + activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}"/requirements.txt 2>&1 | tee -a "$bld_log" # Install tox python requirements - activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\ - tee -a "$bld_log" + activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}"/tox-requirements.txt 2>&1 | tee -a "$bld_log" # Clean up virtualenv directories git checkout -q -- . diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh index 39e2ca192..54aa4bee0 100644 --- a/docker/scripts/lib_vpp.sh +++ b/docker/scripts/lib_vpp.sh @@ -1,7 +1,7 @@ # lib_vpp.sh - Docker build script VPP library. # For import only. -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -20,8 +20,8 @@ if [ -n "$(alias lib_vpp_imported 2> /dev/null)" ] ; then fi alias lib_vpp_imported=true -export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"} -. $CIMAN_DOCKER_SCRIPTS/lib_common.sh +export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname "${BASH_SOURCE[0]}")"} +. "$CIMAN_DOCKER_SCRIPTS"/lib_common.sh VPP_SUPPORTED_EXECUTOR_CLASSES="builder" @@ -32,23 +32,45 @@ vpp_supported_executor_class() { return 0 } +install_hst_deps() { + local branch=${1:-"master"} + local branchname=${branch/\//_} + local hst_dir="./extras/hs-test" + local bld_log="$DOCKER_BUILD_LOG_DIR" + bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname" + bld_log="${bld_log}-install_hst_deps_bld.log" + + if [ -d "$hst_dir" ] ; then + make -C "$hst_dir" install-deps 2>&1 | tee -a "$bld_log" + fi +} + make_vpp() { local target=$1 local branch=${2:-"master"} - local branchname="$(echo $branch | sed -e 's,/,_,')" + local clean=${3:-"true"} + local branchname=${branch/\//_} local bld_log="$DOCKER_BUILD_LOG_DIR" + if [ "$target" = "install-ext-deps" ] ; then + if [ -d "$DOCKER_VPP_DL_CACHE_DIR" ] ; then + mkdir -p "$DOCKER_DOWNLOADS_DIR" + cp -a "$DOCKER_VPP_DL_CACHE_DIR"/* "$DOCKER_DOWNLOADS_DIR" + fi + fi bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname" bld_log="${bld_log}-make_vpp_${target}-bld.log" makefile_target="^${target}:" - if [ -z "$(grep $makefile_target Makefile)" ] ; then + if ! grep "$makefile_target" Makefile ; then echo "Make target '$target' does not exist in VPP branch '$branch'!" return fi - git clean -qfdx + if [ "$clean" = "true" ] ; then + git clean -qfdx + fi description="'make UNATTENDED=yes $target' in $(pwd) ($branch)" echo_log -e " Starting $description..." - make UNATTENDED=yes $target 2>&1 | tee -a "$bld_log" + make UNATTENDED=yes "$target" 2>&1 | tee -a "$bld_log" git checkout -q -- . echo_log " Completed $description!" } @@ -56,13 +78,13 @@ make_vpp() { make_vpp_test() { local target=$1 local branch=${2:-"master"} - local branchname="$(echo $branch | sed -e 's,/,_,')" + local branchname=${branch/\//_} local bld_log="$DOCKER_BUILD_LOG_DIR" bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname" bld_log="${bld_log}-make_vpp_test_${target}-bld.log" makefile_target="^${target}:" - if [ -z "$(grep -e $makefile_target test/Makefile)" ] ; then + if ! grep -e "$makefile_target" test/Makefile ; then echo "Make test target '$target' does not exist in VPP branch '$branch'!" return fi @@ -70,7 +92,7 @@ make_vpp_test() { description="'make -C test $target' in $(pwd) ($branch)" echo_log " Starting $description..." make WS_ROOT="$DOCKER_VPP_DIR" BR="$DOCKER_VPP_DIR/build-root" \ - TEST_DIR="$DOCKER_VPP_DIR/test" -C test $target 2>&1 | tee -a $bld_log + TEST_DIR="$DOCKER_VPP_DIR"/test -C test "$target" 2>&1 | tee -a "$bld_log" remove_pyc_files_and_pycache_dirs git checkout -q -- . echo_log " Completed $description!" @@ -81,6 +103,10 @@ docker_build_setup_vpp() { if [ ! -d "$DOCKER_VPP_DIR" ] ; then echo_log "Cloning VPP into $DOCKER_VPP_DIR..." git clone -q https://gerrit.fd.io/r/vpp $DOCKER_VPP_DIR + if [ -d "$DOCKER_DOWNLOADS_DIR" ] ; then + mkdir -p "$DOCKER_VPP_DL_CACHE_DIR" + cp -a "$DOCKER_DOWNLOADS_DIR"/* "$DOCKER_VPP_DL_CACHE_DIR" + fi fi clean_git_repo $DOCKER_VPP_DIR fi @@ -96,7 +122,9 @@ docker_build_setup_vpp() { # to create an enumerated set of jobs jobs that match the # definitions here. declare -A VPP_BRANCHES -VPP_BRANCHES["debian-11"]="stable/2310 stable/2402 master" -VPP_BRANCHES["ubuntu-20.04"]="stable/2310 stable/2402 master" -VPP_BRANCHES["ubuntu-22.04"]="stable/2310 stable/2402 master" +VPP_BRANCHES["debian-11"]="stable/2402 stable/2406 master" +VPP_BRANCHES["debian-12"]="master" +VPP_BRANCHES["ubuntu-20.04"]="stable/2402 stable/2406 master" +VPP_BRANCHES["ubuntu-22.04"]="stable/2402 stable/2406 master" +VPP_BRANCHES["ubuntu-24.04"]="master" export VPP_BRANCHES diff --git a/extras/bash/sandbox_test_functions.sh b/extras/bash/sandbox_test_functions.sh index ae37d3a19..f952dd335 100644 --- a/extras/bash/sandbox_test_functions.sh +++ b/extras/bash/sandbox_test_functions.sh @@ -29,7 +29,7 @@ jjb-sandbox-env() { - local jjb_version=${JJB_VERSION:-"6.0.0"} + local jjb_version=${JJB_VERSION:-"6.4.1"} if [ -z "$WS_ROOT" ] ; then echo "ERROR: WS_ROOT is not set!" @@ -73,7 +73,7 @@ jjb-sandbox-env() && python3 -m venv $VENV_DIR \ && source $VENV_DIR/bin/activate \ && pip3 install --upgrade pip \ - && pip3 install wheel \ + && pip3 install --upgrade setuptools \ && pip3 install jenkins-job-builder==$jjb_version alias jjsb='jenkins-jobs --conf $JENKINS_INI' diff --git a/global-jjb b/global-jjb -Subproject b9d610a9e522d71a89d2daeb6f71b2bc2a71cdf +Subproject 395a21cbac34ed0d3cb6d0bd8cfea41032aae8b diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-prod-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-prod-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..211295406 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-prod-x86_64-jobtemplate.yaml @@ -0,0 +1,18 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-debian12:prod-x86_64 + volumes: + - /tmp/ccache.debian12:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-prod-x86_64.yaml new file mode 100644 index 000000000..549f2c634 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-prod-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "builder-debian12-prod-x86_64" + labels: "builder-debian12-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-sandbox-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-sandbox-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..cb5dabc2f --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-sandbox-x86_64-jobtemplate.yaml @@ -0,0 +1,31 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-debian12:sandbox-x86_64 + volumes: + - /tmp/ccache.debian12:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-sandbox-x86_64.yaml new file mode 100644 index 000000000..5b342f765 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian12-sandbox-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "builder-debian12-sandbox-x86_64" + labels: "builder-debian12-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..0ccf24752 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-aarch64-jobtemplate.yaml @@ -0,0 +1,11 @@ +--- +Job: + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-aarch64.yaml new file mode 100644 index 000000000..6faf8e64a --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "builder-ubuntu2404-prod-aarch64" + labels: "builder-ubuntu2404-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..5be920ef2 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-x86_64-jobtemplate.yaml @@ -0,0 +1,18 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-x86_64.yaml new file mode 100644 index 000000000..2059a6c42 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-prod-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "builder-ubuntu2404-prod-x86_64" + labels: "builder-ubuntu2404-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..28c9916d0 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml @@ -0,0 +1,24 @@ +--- +Job: + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-aarch64.yaml new file mode 100644 index 000000000..767f97e42 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "builder-ubuntu2404-sandbox-aarch64" + labels: "builder-ubuntu2404-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..f52b30dcf --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml @@ -0,0 +1,31 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-x86_64.yaml new file mode 100644 index 000000000..4ef466bb0 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2404-sandbox-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "builder-ubuntu2404-sandbox-x86_64" + labels: "builder-ubuntu2404-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..514305bd9 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml @@ -0,0 +1,18 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: arm64 + - LTarget: ^${node.class} + Operand: = + RTarget: csitarm + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-aarch64.yaml new file mode 100644 index 000000000..a3fba0ad6 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "csit-builder-ubuntu2404-prod-aarch64" + labels: "csit-builder-ubuntu2404-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..fb46d169a --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml @@ -0,0 +1,18 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: csit + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-x86_64.yaml new file mode 100644 index 000000000..5a5e6918b --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-prod-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "csit-builder-ubuntu2404-prod-x86_64" + labels: "csit-builder-ubuntu2404-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..9f34823e3 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml @@ -0,0 +1,31 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: arm64 + - LTarget: ^${node.class} + Operand: = + RTarget: csitarm + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-aarch64.yaml new file mode 100644 index 000000000..56a04489f --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "csit-builder-ubuntu2404-sandbox-aarch64" + labels: "csit-builder-ubuntu2404-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..e92976e06 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml @@ -0,0 +1,31 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: csit + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-x86_64.yaml new file mode 100644 index 000000000..46b7dae6a --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit-builder-ubuntu2404-sandbox-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "csit-builder-ubuntu2404-sandbox-x86_64" + labels: "csit-builder-ubuntu2404-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml index e868998b1..25240c3ba 100644 --- a/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml +++ b/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml @@ -1,6 +1,6 @@ --- templates: - - idleTerminationInMinutes: 10 + - idleTerminationInMinutes: 1 numExecutors: 1 remoteFs: "/w" reusable: false diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..9aaf453cb --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml @@ -0,0 +1,14 @@ +--- +Job: + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 8000 + MemoryMB: 8192 diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-aarch64.yaml new file mode 100644 index 000000000..083a2a467 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "m-builder-ubuntu2404-prod-aarch64" + labels: "m-builder-ubuntu2404-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..e68c8de8e --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml @@ -0,0 +1,21 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 8000 + MemoryMB: 8192 diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-x86_64.yaml new file mode 100644 index 000000000..71c2c8a46 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-prod-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "m-builder-ubuntu2404-prod-x86_64" + labels: "m-builder-ubuntu2404-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..1ab2dfe73 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml @@ -0,0 +1,27 @@ +--- +Job: + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 8000 + MemoryMB: 8192 + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-aarch64.yaml new file mode 100644 index 000000000..3906913d3 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "m-builder-ubuntu2404-sandbox-aarch64" + labels: "m-builder-ubuntu2404-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..8c7cc93cc --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml @@ -0,0 +1,34 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 8000 + MemoryMB: 8192 + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-x86_64.yaml new file mode 100644 index 000000000..70d1c5d11 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/m-builder-ubuntu2404-sandbox-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "m-builder-ubuntu2404-sandbox-x86_64" + labels: "m-builder-ubuntu2404-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml index 65434f3ce..8c7c75dc1 100644 --- a/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml +++ b/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml @@ -3,7 +3,7 @@ jenkins: clouds: - nomad: name: "FDIONOMAD" - nomadUrl: "http://10.30.51.23:4646" + nomadUrl: "http://10.30.51.26:4646" prune: "false" workerTimeout: 5 tlsEnabled: false diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..0de9cb656 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-aarch64-jobtemplate.yaml @@ -0,0 +1,14 @@ +--- +Job: + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 4000 + MemoryMB: 4096 diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-aarch64.yaml new file mode 100644 index 000000000..afba2c53d --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "s-builder-ubuntu2404-prod-aarch64" + labels: "s-builder-ubuntu2404-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..8e57a90a7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-x86_64-jobtemplate.yaml @@ -0,0 +1,21 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + image: fdiotools/builder-ubuntu2404:prod-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 4000 + MemoryMB: 4096 diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-x86_64.yaml new file mode 100644 index 000000000..9a80fbd3e --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-prod-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "s-builder-ubuntu2404-prod-x86_64" + labels: "s-builder-ubuntu2404-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml new file mode 100644 index 000000000..f7072f832 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-aarch64-jobtemplate.yaml @@ -0,0 +1,27 @@ +--- +Job: + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-aarch64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 4000 + MemoryMB: 4096 + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-aarch64.yaml new file mode 100644 index 000000000..662f10f5a --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-aarch64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "s-builder-ubuntu2404-sandbox-aarch64" + labels: "s-builder-ubuntu2404-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml new file mode 100644 index 000000000..16311e877 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-x86_64-jobtemplate.yaml @@ -0,0 +1,34 @@ +--- +Job: + Constraints: + - LTarget: ^${attr.cpu.arch} + Operand: = + RTarget: amd64 + - LTarget: ^${node.class} + Operand: = + RTarget: builder + TaskGroups: + - Name: jenkins-worker-taskgroup + Tasks: + - Name: jenkins-worker + Config: + args: + - -cp + - /local/slave.jar + - hudson.remoting.jnlp.Main + - -headless + - -url + - https://jenkins.fd.io/sandbox + - -workDir + - /w/ + - '%WORKER_SECRET%' + - '%WORKER_NAME%' + image: fdiotools/builder-ubuntu2404:sandbox-x86_64 + volumes: + - /tmp/ccache.ubuntu2404:/scratch/ccache + - /scratch/nomad:/scratch/nomad:ro + Resources: + CPU: 4000 + MemoryMB: 4096 + Artifacts: + - GetterSource: https://jenkins.fd.io/sandbox/jnlpJars/slave.jar diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-x86_64.yaml new file mode 100644 index 000000000..2176918b6 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/s-builder-ubuntu2404-sandbox-x86_64.yaml @@ -0,0 +1,4 @@ +--- +templates: + - prefix: "s-builder-ubuntu2404-sandbox-x86_64" + labels: "s-builder-ubuntu2404-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml index 65434f3ce..8c7c75dc1 100644 --- a/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml +++ b/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml @@ -3,7 +3,7 @@ jenkins: clouds: - nomad: name: "FDIONOMAD" - nomadUrl: "http://10.30.51.23:4646" + nomadUrl: "http://10.30.51.26:4646" prune: "false" workerTimeout: 5 tlsEnabled: false diff --git a/jjb/csit/csit-perf.yaml b/jjb/csit/csit-perf.yaml index 20630caea..4bcd9897b 100644 --- a/jjb/csit/csit-perf.yaml +++ b/jjb/csit/csit-perf.yaml @@ -20,27 +20,21 @@ jobs: - 'csit-dpdk-perf-verify-{stream}-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -58,27 +52,21 @@ executor-arch: 'x86_64' - 'csit-dpdk-perf-mrr-weekly-master-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -108,27 +96,21 @@ executor-arch: 'x86_64' - 'csit-vpp-perf-verify-{stream}-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -148,91 +130,83 @@ - master: branch: 'master' branch-id: 'oper' - - '2402': - branch: 'rls2402' - branch-id: 'oper-rls2402' + - '2410': + branch: 'rls2410' + branch-id: 'oper-rls2410' - 'csit-vpp-perf-mrr-daily-master-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' + periodicity: '30 22 * * 1,4' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3n-snr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3n-icxd: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3na-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3nb-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 'csit-vpp-perf-hoststack-daily-master-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' + periodicity: '30 22 * * 1,4' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3na-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 3nb-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - periodicity: '30 22 * * 0-4' + periodicity: '30 22 * * 1,4' - 'csit-vpp-perf-mrr-weekly-master-{node-arch}': node-arch: - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' periodicity: '0 0 * * 6' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - periodicity: '0 0 * * 6' - 2n-aws: os: 'ubuntu2204' executor-arch: 'x86_64' @@ -250,9 +224,6 @@ - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' @@ -279,27 +250,21 @@ periodicity: '0 0 * * 6' - 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -318,27 +283,21 @@ periodicity: '0 0 * * 6' - 'csit-vpp-perf-report-iterative-{stream_report}-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -368,27 +327,21 @@ executor-arch: 'x86_64' - 'csit-dpdk-perf-report-iterative-{stream_report}-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -423,27 +376,21 @@ executor-arch: 'x86_64' - 'csit-vpp-perf-report-coverage-{stream_report}-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -473,27 +420,21 @@ executor-arch: 'x86_64' - 'csit-dpdk-perf-report-coverage-{stream_report}-{node-arch}': node-arch: + - 2n-grc: + os: 'ubuntu2404' + executor-arch: 'aarch64' - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' - 2n-zn2: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' @@ -542,13 +483,13 @@ - master: branch: 'master' branch-id: 'oper' - - '2402': - branch: 'rls2402' - branch-id: 'oper-rls2402' + - '2410': + branch: 'rls2410' + branch-id: 'oper-rls2410' stream_report: - - '2402': - branch: 'rls2402' - branch-id: 'oper-rls2402' + - '2410': + branch: 'rls2410' + branch-id: 'oper-rls2410' build-artifact-num-to-keep: 10 diff --git a/jjb/csit/csit-tox.yaml b/jjb/csit/csit-tox.yaml index 0e9d7ffc2..2f0d7ecb0 100644 --- a/jjb/csit/csit-tox.yaml +++ b/jjb/csit/csit-tox.yaml @@ -25,8 +25,8 @@ stream: - master: branch: 'master' - - '2402': - branch: 'rls2402' + - '2410': + branch: 'rls2410' - job-template: name: 'csit-verify-tox-{stream}-{os}-{executor-arch}' @@ -68,7 +68,7 @@ exclude-trivial-rebase: 'false' exclude-no-code-change: 'false' # Not enabling on draft published event. - # Frequently, people publish for public access, not fr verify. + # Frequently, people publish for public access, not for verify. # Use recheck comment to trigger verify. - comment-added-contains-event: comment-contains-value: 'recheck' @@ -83,10 +83,8 @@ override-votes: true gerrit-build-notbuilt-verified-value: 0 gerrit-build-successful-verified-value: 1 - failed-message: 'Tox failed. No further verify jobs will be started.' - successful-message: 'tox_success' skip-vote: - successful: true + successful: false failed: false unstable: false notbuilt: false diff --git a/jjb/csit/csit-vpp-device.yaml b/jjb/csit/csit-vpp-device.yaml deleted file mode 100644 index dbf3ef931..000000000 --- a/jjb/csit/csit-vpp-device.yaml +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- project: - name: csit-vpp-device - description: 'CSIT jenkins jobs.' - jobs: - - 'csit-vpp-device-{stream}-{os}-{executor-arch}-{node-arch}' - - 'csit-vpp-device-semiweekly-{stream}-{os}-{executor-arch}-{node-arch}' - - 'csit-vpp-device-weekly-{stream}-{os}-{executor-arch}-{node-arch}' - - 'csit-vpp-device-report-coverage-{stream}-{node-arch}': - stream: - - '2402': - branch: 'rls2402' - branch-id: 'oper-rls2402' - node-arch: - - '1n-alt': - os: 'ubuntu2204' - executor-arch: 'aarch64' - make-parallel-jobs: '32' - skip-vote: 'false' - - '1n-spr': - os: 'ubuntu2204' - executor-arch: 'x86_64' - make-parallel-jobs: '32' - skip-vote: 'true' - project: 'csit' - branch: 'master' - branch-id: 'oper' - stream: - - master: - branch: 'master' - branch-id: 'oper' - - '2402': - branch: 'rls2402' - branch-id: 'oper-rls2402' - -- job-template: - name: 'csit-vpp-device-report-coverage-{stream}-{node-arch}' - - project-type: freestyle - node: 'csit-builder-{os}-prod-{executor-arch}' - concurrent: true - latest-only: false - - properties: - - lf-infra-properties: - build-days-to-keep: "{build-days-to-keep}" - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - os-parameter: - os: '{os}' - - gerrit-refspec-parameter: - refspec: '$GERRIT_REFSPEC' - - gerrit-event-type-parameter - - gerrit-event-comment-text-parameter - - csit-fail-on-crc-mismatch-parameter: - fail-on-crc-mismatch: 'True' - - scm: - - gerrit-trigger-scm: - credentials-id: 'jenkins-gerrit-credentials' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - wrappers: - - fdio-infra-wrappers: - build-timeout: '{build-timeout}' - - triggers: - - gerrit-trigger-csit-perftest: - project: '{project}' - branch: '{branch}' - trigger-word: 'csit-vpp-report-dev-{node-arch}' - skip-vote: false - - builders: - - shell: - !include-raw-escape: - - ../scripts/setup_executor_env.sh - - shell: - !include-raw-escape: - - ../scripts/csit/device-verify.sh - - publishers: - - fdio-infra-publish - -- job-template: - name: 'csit-vpp-device-{stream}-{os}-{executor-arch}-{node-arch}' - - project-type: freestyle - node: 'csit-builder-{os}-prod-{executor-arch}' - concurrent: true - latest-only: false - - properties: - - lf-infra-properties: - build-days-to-keep: "{build-days-to-keep}" - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - os-parameter: - os: '{os}' - - gerrit-refspec-parameter: - refspec: '$GERRIT_REFSPEC' - - gerrit-event-type-parameter - - gerrit-event-comment-text-parameter - # This is an automatic verify job, part of API coverage. - # CRC checking is needed to ensure vpp crc job breaks only when intended. - # Unlikely to be removed, as this job is primary source of API coverage. - - csit-fail-on-crc-mismatch-parameter: - fail-on-crc-mismatch: 'True' - - scm: - - gerrit-trigger-scm: - credentials-id: 'jenkins-gerrit-credentials' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - wrappers: - - fdio-infra-wrappers: - build-timeout: '{build-timeout}' - - triggers: - - gerrit-trigger-csit-vpp-devicetest-full: - project: '{project}' - branch: '{branch}' - skip-vote: '{skip-vote}' - - builders: - - shell: - !include-raw-escape: - - ../scripts/setup_executor_env.sh - - shell: - !include-raw-escape: - - ../scripts/csit/device-verify.sh - - publishers: - - fdio-infra-publish - -- job-template: - name: 'csit-vpp-device-semiweekly-{stream}-{os}-{executor-arch}-{node-arch}' - - project-type: freestyle - node: 'csit-builder-{os}-prod-{executor-arch}' - concurrent: false - latest-only: false - - properties: - - lf-infra-properties: - build-days-to-keep: "{build-days-to-keep}" - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - os-parameter: - os: '{os}' - - wrappers: - - fdio-infra-wrappers-non-activity-timeout: - build-timeout: '{build-timeout}' - - triggers: - - timed: '0 4 * * 2,6' - - - gerrit-trigger-manually-triggered: - comment-trigger-value: 'csit-verify-vpp' - name: '{project}' - branch: '{branch}' - - builders: - - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop' - - inject: - properties-file: variables.prop - - shell: - !include-raw-escape: - - ../scripts/setup_executor_env.sh - - shell: - !include-raw-escape: - - ../scripts/csit/device-semiweekly.sh - - publishers: - - fdio-infra-publish - -- job-template: - name: 'csit-vpp-device-weekly-{stream}-{os}-{executor-arch}-{node-arch}' - - project-type: freestyle - node: 'csit-builder-{os}-prod-{executor-arch}' - concurrent: false - latest-only: false - - properties: - - lf-infra-properties: - build-days-to-keep: "{build-days-to-keep}" - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - os-parameter: - os: '{os}' - - scm: - - git-scm: - credentials-id: 'jenkins-gerrit-credentials' - branch: '{branch}' - - wrappers: - - fdio-infra-wrappers-non-activity-timeout: - build-timeout: '{build-timeout}' - - triggers: - - timed: '0 4 * * 0' - - - gerrit-trigger-manually-triggered: - comment-trigger-value: 'csit-vpp-ref-verify' - name: '{project}' - branch: '{branch}' - - builders: - - shell: - !include-raw-escape: - - ../scripts/setup_executor_env.sh - - shell: - !include-raw-escape: - - ../scripts/csit/device-verify.sh - - publishers: - - fdio-infra-publish diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 2de57c726..4a1fdf424 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -390,56 +390,6 @@ notbuilt: '{skip-vote}' - trigger: - name: gerrit-trigger-csit-vpp-devicetest-full - # Comes from CSIT, for automatic jobs running device tests with VPP builds. - # The job acts as an automatic verify job for CSIT contributions. - triggers: - - gerrit: - server-name: 'Primary' - trigger-on: - # The main way the job is triggered automatically on patch upload. - # Gated by tox job, so we do not spend resources on WiP changes. - - comment-added-contains-event: - comment-contains-value: 'tox_success' - # Manual trigger to allow testing WiP changes with selected tags. - - comment-added-contains-event: - comment-contains-value: 'csit-devicetest' - # A comment with "recheck" restarts tox job, so no condition here. - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - skip-vote: - successful: '{skip-vote}' - failed: '{skip-vote}' - unstable: '{skip-vote}' - notbuilt: '{skip-vote}' - -- trigger: - name: gerrit-trigger-csit-vpp-devicetest-manual - # Comes from CSIT, for manual jobs running "device" tests with VPP builds. - # The job acts as a manual (optional) verify job for CSIT contributions. - triggers: - - gerrit: - server-name: 'Primary' - trigger-on: - - comment-added-contains-event: - comment-contains-value: 'csit-devicetest' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - skip-vote: - successful: '{skip-vote}' - failed: '{skip-vote}' - unstable: '{skip-vote}' - notbuilt: '{skip-vote}' - -- trigger: name: gerrit-trigger-manually-triggered triggers: - gerrit: @@ -598,52 +548,6 @@ failed: true unstable: true notbuilt: true - -- trigger: - name: gerrit-trigger-vpp-csit-devicetest-full - # Comes from VPP, for automated jobs running "device" tests with CSIT code. - # The job acts as an automatic verify job for VPP contributions. - triggers: - - gerrit: - server-name: 'Primary' - trigger-on: - - comment-added-contains-event: - comment-contains-value: 'checkstyle_success' - - comment-added-contains-event: - comment-contains-value: 'devicetest' - projects: - - project-compare-type: 'ANT' - project-pattern: '{name}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - skip-vote: - successful: '{skip-vote}' - failed: '{skip-vote}' - unstable: '{skip-vote}' - notbuilt: '{skip-vote}' - -- trigger: - name: gerrit-trigger-vpp-csit-devicetest-manual - # Comes from VPP, for manual jobs running "device" tests with CSIT code. - # The job acts as a manual (optional) verify job for VPP contributions. - triggers: - - gerrit: - server-name: 'Primary' - trigger-on: - - comment-added-contains-event: - comment-contains-value: 'devicetest' - projects: - - project-compare-type: 'ANT' - project-pattern: '{name}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - skip-vote: - successful: '{skip-vote}' - failed: '{skip-vote}' - unstable: '{skip-vote}' - notbuilt: '{skip-vote}' # [end] Keep 'trigger' entries ordered alphabetically, please. #### PUBLISHERS diff --git a/jjb/scripts/csit/device-semiweekly.sh b/jjb/scripts/csit/device-semiweekly.sh deleted file mode 100755 index 1086b5463..000000000 --- a/jjb/scripts/csit/device-semiweekly.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2020 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "---> jjb/scripts/csit/device-semiweekly.sh" - -set -exuo pipefail - -# Clone CSIT git repository and proceed with entry script located there. -# -# Variables read: -# - WORKSPACE - Jenkins workspace to create csit subdirectory in. -# - GIT_URL - Git clone URL -# - BRANCH_ID - CSIT operational branch to be used for test. -# Directories updated: -# - ${WORKSPACE}/csit - Created, holding a checked out CSIT repository. -# - Multiple other side effects by entry script(s), see CSIT repository. - -cd "${WORKSPACE}" -git clone "${GIT_URL}/csit" --depth=1 --no-single-branch --no-checkout -# Check BRANCH_ID value. -if [[ -z "${BRANCH_ID-}" ]]; then - echo "BRANCH_ID not provided => 'oper' belonging to master will be used." - BRANCH_ID="oper" -fi -pushd "${WORKSPACE}/csit" -# Get the latest verified version of the required branch. -BRANCH_NAME=$(echo $(git branch -r | grep -E "${BRANCH_ID}-[0-9]+" | tail -n 1)) -if [[ -z "${BRANCH_NAME-}" ]]; then - echo "No verified CSIT branch found - exiting!" - exit 1 -fi -# Remove 'origin/' from the branch name. -BRANCH_NAME=$(echo ${BRANCH_NAME#origin/}) -# Checkout the required csit branch. -git checkout "${BRANCH_NAME}" -popd -csit_entry_dir="${WORKSPACE}/csit/resources/libraries/bash/entry" -source "${csit_entry_dir}/bootstrap_vpp_device.sh" -cp -R "${WORKSPACE}/csit/archives" "${WORKSPACE}/archives" || true diff --git a/jjb/scripts/csit/device-verify.sh b/jjb/scripts/csit/device-verify.sh deleted file mode 100644 index 5ccd2a580..000000000 --- a/jjb/scripts/csit/device-verify.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2020 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "---> jjb/scripts/csit/device-verify.sh" - -set -exuo pipefail - -csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry" -source "${csit_entry_dir}/bootstrap_vpp_device.sh" diff --git a/jjb/scripts/publish_library_py.sh b/jjb/scripts/publish_library_py.sh index 1cbeb23c0..af4f22a08 100644 --- a/jjb/scripts/publish_library_py.sh +++ b/jjb/scripts/publish_library_py.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -19,7 +19,13 @@ set -exuo pipefail PYTHON_SCRIPT="/w/workspace/publish_library.py" -pip3 install boto3 +OS_CODENAME="$(grep 'VERSION_CODENAME=' /etc/os-release | cut -d= -f2)" +pip_options="" +if [ "$OS_CODENAME" = "noble" ] || [ "$OS_CODENAME" = "bookworm" ] ; then + pip_options=" --break-system-packages" +fi +# shellcheck disable=SC2086 +pip3 install$pip_options boto3 mkdir -p $(dirname "$PYTHON_SCRIPT") cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT' diff --git a/jjb/scripts/vpp/api-checkstyle.sh b/jjb/scripts/vpp/api-checkstyle.sh index a1c2f84a1..a1c2f84a1 100644..100755 --- a/jjb/scripts/vpp/api-checkstyle.sh +++ b/jjb/scripts/vpp/api-checkstyle.sh diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh index 850d61003..850d61003 100644..100755 --- a/jjb/scripts/vpp/build.sh +++ b/jjb/scripts/vpp/build.sh diff --git a/jjb/scripts/vpp/checkstyle.sh b/jjb/scripts/vpp/checkstyle.sh index 00fc8bdb8..7feae3980 100755 --- a/jjb/scripts/vpp/checkstyle.sh +++ b/jjb/scripts/vpp/checkstyle.sh @@ -34,9 +34,9 @@ if git show --stat | grep -q "$MATRIX_FILE" ; then fi fi -if [ -n "$(grep -E '^checkstyle:' Makefile)" ] +if grep -qE '^checkstyle:' Makefile then - make checkstyle + make UNATTENDED=yes install-deps checkstyle else - echo "Can't find checkstyle target in Makefile - skipping checkstyle" + echo "Can't find checkstyle target in Makefile - skipping checkstyle" fi diff --git a/jjb/scripts/vpp/commitmsg.sh b/jjb/scripts/vpp/commitmsg.sh index 479f35136..479f35136 100644..100755 --- a/jjb/scripts/vpp/commitmsg.sh +++ b/jjb/scripts/vpp/commitmsg.sh diff --git a/jjb/scripts/vpp/copy_archives.sh b/jjb/scripts/vpp/copy_archives.sh index dafcd4aef..dafcd4aef 100644..100755 --- a/jjb/scripts/vpp/copy_archives.sh +++ b/jjb/scripts/vpp/copy_archives.sh diff --git a/jjb/scripts/vpp/cov-build.sh b/jjb/scripts/vpp/cov-build.sh index 15f86c3bf..15f86c3bf 100644..100755 --- a/jjb/scripts/vpp/cov-build.sh +++ b/jjb/scripts/vpp/cov-build.sh diff --git a/jjb/scripts/vpp/csit-bisect.sh b/jjb/scripts/vpp/csit-bisect.sh index 0a264dbc8..0a264dbc8 100644..100755 --- a/jjb/scripts/vpp/csit-bisect.sh +++ b/jjb/scripts/vpp/csit-bisect.sh diff --git a/jjb/scripts/vpp/csit-device.sh b/jjb/scripts/vpp/csit-device.sh deleted file mode 100755 index 989193001..000000000 --- a/jjb/scripts/vpp/csit-device.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2020 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "---> jjb/scripts/vpp/csit-device.sh" - -set -exuo pipefail - -# Clone CSIT git repository and proceed with entry script located there. -# -# Variables read: -# - WORKSPACE - Jenkins workspace to create csit subdirectory in. -# - GIT_URL - Git clone URL -# - CSIT_REF - Override ref of CSIT git repository to checkout. -# Directories updated: -# - ${WORKSPACE}/csit - Created, holding a checked out CSIT repository. -# - Multiple other side effects by entry script(s), see CSIT repository. - -cd "${WORKSPACE}" -git clone "${GIT_URL}/csit" --depth=1 --no-single-branch --no-checkout -pushd "${WORKSPACE}/csit" -if [[ -n "${CSIT_REF-}" ]]; then - git fetch --depth=1 "${GIT_URL}/csit" "${CSIT_REF}" - git checkout FETCH_HEAD -else - git checkout HEAD -fi -popd -csit_entry_dir="${WORKSPACE}/csit/resources/libraries/bash/entry" -source "${csit_entry_dir}/with_oper_for_vpp.sh" "per_patch_device.sh" diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh index 68338df56..68338df56 100644..100755 --- a/jjb/scripts/vpp/debug-build.sh +++ b/jjb/scripts/vpp/debug-build.sh diff --git a/jjb/scripts/vpp/debug-hst.sh b/jjb/scripts/vpp/debug-hst.sh new file mode 100755 index 000000000..1761d0949 --- /dev/null +++ b/jjb/scripts/vpp/debug-hst.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/debug-hst.sh" + +set -euxo pipefail + +line="*************************************************************************" +OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" +HST_DIR="./extras/hs-test" + +vpp_make_build_debug() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" build-debug ; then + BUILD_ERROR="FAILED 'make -C $HST_DIR build-debug'" + return + fi +} + +hst_test_debug() { + if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" test-debug ; then + BUILD_ERROR="FAILED 'make -C $HST_DIR test-debug'" + return + fi +} + +if [ "${DRYRUN,,}" != "true" ] ; then + vpp_make_build_debug + hst_test_debug +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \ + "DEBUG HostStack Test Suite $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh b/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh index b2248064b..b2248064b 100644..100755 --- a/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh +++ b/jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh index 9bd6d4ff7..9bd6d4ff7 100644..100755 --- a/jjb/scripts/vpp/gcc-build.sh +++ b/jjb/scripts/vpp/gcc-build.sh diff --git a/jjb/scripts/vpp/hst.sh b/jjb/scripts/vpp/hst.sh new file mode 100755 index 000000000..1a5f3cad8 --- /dev/null +++ b/jjb/scripts/vpp/hst.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/vpp/hst.sh" + +set -euxo pipefail + +line="*************************************************************************" +OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') +OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" +HST_DIR="./extras/hs-test" + +vpp_make_build() { + if ! make UNATTENDED=yes install-deps ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make build-release; then + BUILD_ERROR="FAILED 'make build-release'" + return + fi + if ! make -C "$HST_DIR" install-deps build ; then + BUILD_ERROR="FAILED 'make -C $HST_DIR install-deps build'" + return + fi +} + +hst_test() { + if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" build test ; then + BUILD_ERROR="FAILED 'make -C $HST_DIR test'" + return + fi +} + +if [ "${DRYRUN,,}" != "true" ] ; then + vpp_make_build + hst_test +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \ + "HostStack Test Suite $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/test-checkstyle.sh b/jjb/scripts/vpp/test-checkstyle.sh index 3f0bc9a06..3f0bc9a06 100644..100755 --- a/jjb/scripts/vpp/test-checkstyle.sh +++ b/jjb/scripts/vpp/test-checkstyle.sh diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml index 1ee98ba8e..91f7fa687 100644 --- a/jjb/vpp/docs.yaml +++ b/jjb/vpp/docs.yaml @@ -8,22 +8,24 @@ - 'vpp-docsonly-merge-{stream}-{os}-{executor-arch}' project: 'vpp' - executor-arch: 'x86_64' - os: - - 'ubuntu2204' + os: 'ubuntu2404' stream: - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' + os: 'ubuntu2204' - '2406': branch: 'stable/2406' branch-refspec: '' + os: 'ubuntu2204' + - '2410': + branch: 'stable/2410' + branch-refspec: '' + os: 'ubuntu2204' + executor-arch: 'x86_64' - job-template: name: 'vpp-docs-verify-{stream}-{os}-{executor-arch}' diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index 6e9b9c8f7..7e9bee63d 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2024 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -34,24 +34,61 @@ - 'vpp-checkstyle-verify-{stream}-{os}-{executor-arch}' project: 'vpp' os: - - ubuntu2204 + - ubuntu2404 stream: - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': + os: 'ubuntu2204' branch: 'stable/2402' branch-refspec: '' - '2406': + os: 'ubuntu2204' branch: 'stable/2406' branch-refspec: '' + - '2410': + os: 'ubuntu2204' + branch: 'stable/2410' + branch-refspec: '' executor-arch: 'x86_64' # [end] VPP-CHECKSTYLE-X86_64 PROJECT +# VPP-HST-X86_64 PROJECT +# - project: +# name: vpp-verify-hst-x86_64 +# jobs: +# - 'vpp-verify-hst-{stream}-{os}-{executor-arch}' +# project: 'vpp' +# os: +# - ubuntu2204 +# stream: +# - 'master': +# branch: 'master' +# branch-refspec: '' +# executor-arch: 'x86_64' + +# [end] VPP-HST-X86_64 PROJECT + +# VPP-HST-DEBUG-X86_64 PROJECT +- project: + name: vpp-verify-hst-debug-x86_64 + jobs: + - 'vpp-verify-hst-debug-{stream}-{os}-{executor-arch}' + project: 'vpp' + os: + - ubuntu2204 + stream: + - 'master': + # TODO: Remove when ubuntu2404 is put into production + os: 'ubuntu2204' + branch: 'master' + branch-refspec: '' + executor-arch: 'x86_64' + +# [end] VPP-HST-DEBUG-X86_64 PROJECT + # VPP-X86_64 PROJECT # # NOTE: The list of supported branches for each OS are maintained @@ -67,33 +104,43 @@ - 'vpp-verify-{stream}-{os}-{executor-arch}' - 'vpp-merge-{stream}-{os}-{executor-arch}' project: 'vpp' + os: + - 'debian11' + - 'debian12' + - 'ubuntu2004' + - 'ubuntu2204' + - 'ubuntu2404' make-parallel-jobs: '4' + make-test-multiworker-os: 'debian-11 debian-12' + make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04' stream: - 'master': branch: 'master' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: 'debian-11' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: 'debian-11' - '2402': branch: 'stable/2402' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: 'debian-11' - '2406': branch: 'stable/2406' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: 'debian-11' - os: - - debian11 - - ubuntu2004 - - ubuntu2204 + - '2410': + branch: 'stable/2410' + branch-refspec: '' executor-arch: 'x86_64' + exclude: + # ubuntu-24.04 introduced in VPP 24.10 + - stream: '2402' + os: 'ubuntu2404' + # ubuntu-24.04 introduced in VPP 24.10 + - stream: '2406' + os: 'ubuntu2404' + # debian-12 introduced in VPP 24.10 + - stream: '2402' + os: 'debian12' + # debian-12 introduced in VPP 24.10 + - stream: '2406' + os: 'debian12' + # [end] VPP-X86_64 PROJECT # VPP-GCC-X86_64 PROJECT @@ -102,23 +149,31 @@ jobs: - 'vpp-gcc-verify-{stream}-{os}-{executor-arch}' project: 'vpp' - make-parallel-jobs: '4' os: - - ubuntu2204 + - 'ubuntu2004' + - 'ubuntu2404' + make-parallel-jobs: '4' executor-arch: 'x86_64' stream: - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' - '2406': branch: 'stable/2406' branch-refspec: '' + - '2410': + branch: 'stable/2410' + branch-refspec: '' + exclude: + # ubuntu-24.04 introduced in VPP 24.10 + - stream: '2402' + os: 'ubuntu2404' + # ubuntu-24.04 introduced in VPP 24.10 + - stream: '2406' + os: 'ubuntu2404' # [end] VPP-GCC-X86_64 PROJECT @@ -133,15 +188,16 @@ jobs: - 'vpp-debug-verify-{stream}-{os}-{executor-arch}' project: 'vpp' + os: 'ubuntu2404' make-parallel-jobs: '4' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' + make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04' comment-trigger-value: 'debug-verify' stream: - 'master': + # TODO: Remove when ubuntu2404 is put into production + os: 'ubuntu2204' branch: 'master' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - os: 'ubuntu2204' executor-arch: 'x86_64' # [end] VPP-DEBUG-X86_64 PROJECT @@ -157,14 +213,13 @@ jobs: - 'vpp-dpdk-rdma-version-mismatch-{stream}-{os}-{executor-arch}' project: 'vpp' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' + os: 'ubuntu2404' comment-trigger-value: 'dpdk-rdma-version-mismatch' stream: - 'master': branch: 'master' # Once per day at 00:15 periodicity: '15 0 * * 0-6' - os: 'ubuntu2204' executor-arch: 'x86_64' # [end] VPP-DPDK-RDMA-VERSION-MISMATCH-X86_64 PROJECT @@ -181,8 +236,9 @@ jobs: - 'vpp-cov-verify-{stream}-{os}-{executor-arch}' project: 'vpp' + os: 'ubuntu2004' make-parallel-jobs: '4' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' + make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04' comment-trigger-value: 'gcov-report' stream: - 'master': @@ -190,7 +246,6 @@ branch-refspec: '' # Once per day at 00:25 periodicity: '25 0 * * 0-6' - os: 'ubuntu2004' executor-arch: 'x86_64' # [end] VPP-COV-X86_64 PROJECT @@ -210,70 +265,35 @@ - 'vpp-verify-{stream}-{os}-{executor-arch}' - 'vpp-merge-{stream}-{os}-{executor-arch}' project: 'vpp' + os: + - 'ubuntu2004' + - 'ubuntu2204' + - 'ubuntu2404' make-parallel-jobs: '4' + make-test-multiworker-os: 'debian-11' + make-test-os: 'ubuntu-20.04 ubuntu-22.04 ubuntu-24.04' stream: - 'master': branch: 'master' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: '' - '2402': branch: 'stable/2402' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: '' - '2406': branch: 'stable/2406' branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: '' - os: - - ubuntu2004 - - ubuntu2204 - executor-arch: 'aarch64' -# [end] VPP-AARCH64 PROJECT - -# VPP-CSIT-VERIFY-DEVICE PROJECT -- project: - name: vpp-csit-verify-device - jobs: - - 'vpp-csit-verify-device-{stream}-{os}-{executor-arch}-{testbed-arch}': - make-parallel-jobs: '32' - project: 'vpp' - skip-vote: 'false' - stream: - - 'master': - branch: 'master' - branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - - '2402': - branch: 'stable/2402' - branch-refspec: '' - - '2406': - branch: 'stable/2406' + - '2410': + branch: 'stable/2410' branch-refspec: '' - testbed-arch: - - 1n-spr: - os: 'ubuntu2204' - executor-arch: 'x86_64' - skip-vote: 'true' - - 1n-alt: - os: 'ubuntu2204' - executor-arch: 'aarch64' - skip-vote: 'true' + executor-arch: 'aarch64' exclude: - # 1n-alt testbed not supported - # in CSIT prior to 2402 - - testbed-arch: 1n-alt - stream: '2310' -# [end] VPP-CSIT-VERIFY-DEVICE PROJECT + # ubuntu-24.04 introduced in VPP 24.10 + - stream: '2402' + os: 'ubuntu2404' + # ubuntu-24.04 introduced in VPP 24.10 + - stream: '2406' + os: 'ubuntu2404' +# [end] VPP-AARCH64 PROJECT # VPP-CSIT-VERIFY-PERF PROJECT - project: @@ -291,9 +311,6 @@ - 2n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 2n-clx: - os: 'ubuntu2204' - executor-arch: 'x86_64' - 2n-spr: os: 'ubuntu2204' executor-arch: 'x86_64' @@ -303,18 +320,10 @@ - 3n-icx: os: 'ubuntu2204' executor-arch: 'x86_64' - - 3n-tsh: - os: 'ubuntu2204' - executor-arch: 'aarch64' - make-parallel-jobs: '16' - 3n-alt: os: 'ubuntu2204' executor-arch: 'aarch64' make-parallel-jobs: '16' - - 2n-tx2: - os: 'ubuntu2204' - executor-arch: 'aarch64' - make-parallel-jobs: '16' - 3n-snr: os: 'ubuntu2204' executor-arch: 'x86_64' @@ -338,23 +347,25 @@ jobs: - 'vpp-csit-verify-api-crc-{stream}-{os}-{executor-arch}' project: 'vpp' + os: ubuntu2404 executor-arch: 'x86_64' skip-vote: 'false' stream: - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' + os: 'ubuntu2204' - '2406': branch: 'stable/2406' branch-refspec: '' - os: - - 'ubuntu2204' + os: 'ubuntu2204' + - '2410': + branch: 'stable/2410' + branch-refspec: '' + os: 'ubuntu2204' # [end] VPP-CSIT-API-CRC PROJECT # LF RELENG VPP PROJECTS @@ -490,6 +501,122 @@ # [end] VPP-CHECKSTYLE-VERIFY JOB TEMPLATE +# VPP-VERIFY-HST JOB TEMPLATE +- job-template: + name: 'vpp-verify-hst-{stream}-{os}-{executor-arch}' + + project-type: freestyle + node: 'hst-{os}-prod-{executor-arch}' + concurrent: true + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + # Please keep parameters in alphabetical order + parameters: + - gerrit-parameter: + branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - os-parameter: + os: '{os}' + - project-parameter: + project: '{project}' + - stream-parameter: + stream: '{stream}' + + scm: + - gerrit-trigger-scm: + credentials-id: 'jenkins-gerrit-credentials' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - fdio-infra-wrappers: + build-timeout: 120 + + triggers: + - gerrit-trigger-checkstyle: + name: '{project}' + branch: '{branch}' + + builders: + - shell: + !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: + !include-raw-escape: + - ../scripts/setup_vpp_ubuntu_docker_test.sh + - shell: + !include-raw-escape: + - ../scripts/setup_vpp_ext_deps.sh + - shell: + !include-raw-escape: + - ../scripts/vpp/hst.sh + + publishers: + - fdio-infra-publish +# [end] VPP-VERIFY-HST JOB TEMPLATE + +# VPP-VERIFY-HST-DEBUG JOB TEMPLATE +- job-template: + name: 'vpp-verify-hst-debug-{stream}-{os}-{executor-arch}' + + project-type: freestyle + node: 'hst-{os}-prod-{executor-arch}' + concurrent: true + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + # Please keep parameters in alphabetical order + parameters: + - gerrit-parameter: + branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - os-parameter: + os: '{os}' + - project-parameter: + project: '{project}' + - stream-parameter: + stream: '{stream}' + + scm: + - gerrit-trigger-scm: + credentials-id: 'jenkins-gerrit-credentials' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - fdio-infra-wrappers: + build-timeout: 120 + + triggers: + - gerrit-trigger-checkstyle: + name: '{project}' + branch: '{branch}' + + builders: + - shell: + !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: + !include-raw-escape: + - ../scripts/setup_vpp_ubuntu_docker_test.sh + - shell: + !include-raw-escape: + - ../scripts/setup_vpp_ext_deps.sh + - shell: + !include-raw-escape: + - ../scripts/vpp/debug-hst.sh + + publishers: + - fdio-infra-publish +# [end] VPP-VERIFY-HST-DEBUG JOB TEMPLATE + # VPP-VERIFY JOB TEMPLATE - job-template: name: 'vpp-verify-{stream}-{os}-{executor-arch}' @@ -632,10 +759,6 @@ parameters: - gerrit-parameter: branch: '{branch}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - make-test-os-parameter: - make-test-os: '{make-test-os}' - os-parameter: os: '{os}' - project-parameter: @@ -875,82 +998,6 @@ - fdio-infra-publish # [end] VPP-COV-VERIFY JOB TEMPLATE -# VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE -- job-template: - name: 'vpp-csit-verify-device-{stream}-{os}-{executor-arch}-{testbed-arch}' - description: | - <ul> - <li>executor - <ul> - <li>csit-builder-{os}-prod-{executor-arch} - </ul> - <li>testbed architecture - <ul> - <li>{testbed-arch} - </ul> - </ul> - - project-type: freestyle - node: 'csit-builder-{os}-prod-{executor-arch}' - concurrent: true - latest-only: false - - properties: - - lf-infra-properties: - build-days-to-keep: "{build-days-to-keep}" - - # Please keep parameters in alphabetical order - parameters: - - gerrit-csit-refspec-parameter - - gerrit-event-comment-text-parameter - - gerrit-event-type-parameter - - gerrit-parameter: - branch: '{branch}' - - gerrit-refspec-parameter: - refspec: '{branch-refspec}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - os-parameter: - os: '{os}' - - project-parameter: - project: '{project}' - - stream-parameter: - stream: '{stream}' - - scm: - - gerrit-trigger-scm: - credentials-id: 'jenkins-gerrit-credentials' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - wrappers: - - fdio-infra-wrappers: - build-timeout: '90' - - triggers: - - 'gerrit-trigger-vpp-csit-devicetest-full': - name: '{project}' - branch: '{branch}' - skip-vote: '{skip-vote}' - - builders: - - shell: - !include-raw-escape: - - ../scripts/setup_executor_env.sh - - shell: - !include-raw-escape: - - ../scripts/setup_vpp_ubuntu_docker_test.sh - - shell: - !include-raw-escape: - - ../scripts/setup_vpp_ext_deps.sh - - shell: - !include-raw-escape: - - ../scripts/vpp/csit-device.sh - - publishers: - - fdio-infra-publish -# [end] VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE - # VPP-CSIT-VERIFY-PERF JOB TEMPLATE - job-template: name: 'vpp-csit-verify-perf-{stream}-{os}-{executor-arch}-{testbed-arch}' |