diff options
| -rwxr-xr-x | docker/scripts/dbld_csit_find_ansible_packages.py | 7 | ||||
| -rwxr-xr-x | docker/scripts/dbld_vpp_install_packages.sh | 15 | ||||
| -rw-r--r-- | docker/scripts/lib_apt.sh | 29 | ||||
| -rw-r--r-- | docker/scripts/lib_common.sh | 1 | ||||
| -rw-r--r-- | docker/scripts/lib_csit.sh | 2 | ||||
| -rw-r--r-- | docker/scripts/lib_vpp.sh | 27 | ||||
| -rw-r--r-- | jjb/vpp/docs.yaml | 3 | ||||
| -rw-r--r-- | jjb/vpp/vpp.yaml | 19 |
8 files changed, 58 insertions, 45 deletions
diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py index 52ed4c3b0..50b7d8478 100755 --- a/docker/scripts/dbld_csit_find_ansible_packages.py +++ b/docker/scripts/dbld_csit_find_ansible_packages.py @@ -37,6 +37,7 @@ class CsitAnsibleYamlStruct: self.__dict__.update(entries) def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list: + codename = os.getenv("OS_CODENAME") with open(yamlfile) as yf: csit_ansible_yaml = yaml.safe_load(yf) if csit_ansible_yaml is None: @@ -50,7 +51,11 @@ def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list: packages += [pkg for pkg in cays.packages_by_arch[arch] if type(pkg) is str] if distro in [*cays.packages_by_distro]: - packages += [pkg for pkg in cays.packages_by_distro[distro] + if codename in [*cays.packages_by_distro[distro]]: + packages += [pkg for pkg in cays.packages_by_distro[distro][codename] + if type(pkg) is str] + else: + packages += [pkg for pkg in cays.packages_by_distro[distro] if type(pkg) is str] return packages diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh index fecd823dd..be8b34cdc 100755 --- a/docker/scripts/dbld_vpp_install_packages.sh +++ b/docker/scripts/dbld_vpp_install_packages.sh @@ -33,11 +33,6 @@ do_git_config vpp for branch in ${VPP_BRANCHES[$OS_NAME]} ; do do_git_branch "$branch" - # Install hs-test depndencies - if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then - install_hst_deps "$branch" - fi - # Install OS packages make_vpp "install-deps" "$branch" @@ -57,6 +52,16 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'" exit 1 fi + + # Install hs-test depndencies + if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then + make_vpp build "$branch" "false" + make_vpp build-release "$branch" "false" + make_vpp build-vpp-gcov "$branch" "false" + install_hst_deps "$branch" + git clean -qfdx + fi + # TODO: remove this after all supported VPP branches have removed # python3-virtualenv & virtualenv from install-deps which are no longer # used in vpp repo. These packages can mess up csit virtualenv diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh index a1ee6db7c..f8a6f67be 100644 --- a/docker/scripts/lib_apt.sh +++ b/docker/scripts/lib_apt.sh @@ -20,7 +20,7 @@ if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then fi alias lib_apt_imported=true -DIND_FROM_IMAGE="cruizba/ubuntu-dind:latest" +DIND_FROM_IMAGE="cruizba/ubuntu-dind:jammy-26.1.3-r2" HST_FROM_IMAGE="ubuntu:22.04" export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"} @@ -41,6 +41,7 @@ apt_install_packages() { generate_apt_dockerfile_common() { local executor_class="$1" local executor_image="$2" + local install_golang="$3" local dpkg_arch="$(dpkg --print-architecture)" cat <<EOF >>"$DOCKERFILE" @@ -160,6 +161,13 @@ RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$d && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\ && mv terraform /usr/bin \\ && rm -f terraform_1.7.3_linux_$dpkg_arch.zip +EOF + + if [ "$install_golang" = "true" ] ; then + generate_apt_dockerfile_install_golang "1.21.11" + fi + + cat <<EOF >>"$DOCKERFILE" # Install packages for all project branches # @@ -182,7 +190,12 @@ EOF } generate_apt_dockerfile_install_golang() { - local GO_VERSION="$1" + local go_version="$1" + local go_tarball_arch="amd64" + + if [ "$OS_ARCH" = "aarch64" ] ; then + go_tarball_arch="arm64" + fi cat <<EOF >>"$DOCKERFILE" @@ -192,9 +205,9 @@ ENV GOPATH /go ENV GOROOT /usr/local/go ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH RUN rm -rf /usr/local/go /usr/bin/go \\ - && wget -P /tmp "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" \\ - && tar -C /usr/local -xzf "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\ - && rm "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\ + && wget -P /tmp "https://go.dev/dl/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\ + && tar -C /usr/local -xzf "/tmp/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\ + && rm "/tmp/go${go_version}.linux-${go_tarball_arch}.tar.gz" \\ && ln -s /usr/local/go/bin/go /usr/bin/go \\ && echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\ && mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH" @@ -209,11 +222,8 @@ builder_generate_apt_dockerfile() { local install_golang="$4" local vpp_install_skip_sysctl_envvar=""; - generate_apt_dockerfile_common "$executor_class" "$executor_image" + generate_apt_dockerfile_common "$executor_class" "$executor_image" "$install_golang" csit_builder_generate_docker_build_files - if [ "$install_golang" = "true" ] ; then - generate_apt_dockerfile_install_golang "1.21.9" - fi cat <<EOF >>"$DOCKERFILE" # Install LF-IT requirements @@ -368,6 +378,7 @@ generate_apt_dockerfile() { local install_golang="false" local is_dind_image="false" + # TODO: Enable HST on AARCH64 when supported in vpp/extras/hs-test if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then from_image="$DIND_FROM_IMAGE" install_golang="true" diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh index 150441bb7..166b06fbf 100644 --- a/docker/scripts/lib_common.sh +++ b/docker/scripts/lib_common.sh @@ -232,6 +232,7 @@ export DOCKERIGNOREFILE="$DOCKER_BUILD_DIR/.dockerignore" export DOCKERFILE_FROM=${DOCKERFILE_FROM:="${OS_ID}:${OS_VERSION_ID}"} export DOCKER_TAG="$DOCKER_DATE-$OS_ARCH" export DOCKER_VPP_DIR="$DOCKER_BUILD_DIR/vpp" +export DOCKER_VPP_DL_CACHE_DIR="$DOCKER_BUILD_DIR"/vpp_ext_deps_downloads export DOCKER_CSIT_DIR="$DOCKER_BUILD_DIR/csit" export DOCKER_DOWNLOADS_DIR="/root/Downloads" export DOCKER_BUILD_FILES_DIR="$DOCKER_BUILD_DIR/files" diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh index 2e877f803..0499cbfc4 100644 --- a/docker/scripts/lib_csit.sh +++ b/docker/scripts/lib_csit.sh @@ -75,7 +75,7 @@ csit_install_packages() { # Not in double quotes to let bash remove newline characters local yaml_files yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)" - packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)" + packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files) | grep -v "$OS_CODENAME")" packages="${packages/jammy /}" packages="${packages/focal /}" packages="${packages/libmbedcrypto1/libmbedcrypto3}" diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh index a4efbd119..4bf8dab88 100644 --- a/docker/scripts/lib_vpp.sh +++ b/docker/scripts/lib_vpp.sh @@ -48,17 +48,26 @@ install_hst_deps() { make_vpp() { local target=$1 local branch=${2:-"master"} + local clean=${3:-"true"} local branchname=${branch/\//_} local bld_log="$DOCKER_BUILD_LOG_DIR" + if [ "$target" = "install-ext-deps" ] ; then + if [ -d "$DOCKER_VPP_DL_CACHE_DIR" ] ; then + mkdir -p "$DOCKER_DOWNLOADS_DIR" + cp -a "$DOCKER_VPP_DL_CACHE_DIR"/* "$DOCKER_DOWNLOADS_DIR" + fi + fi bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname" bld_log="${bld_log}-make_vpp_${target}-bld.log" makefile_target="^${target}:" - if [ -z "$(grep $makefile_target Makefile)" ] ; then + if ! grep "$makefile_target" Makefile ; then echo "Make target '$target' does not exist in VPP branch '$branch'!" return fi - git clean -qfdx + if [ "$clean" = "true" ] ; then + git clean -qfdx + fi description="'make UNATTENDED=yes $target' in $(pwd) ($branch)" echo_log -e " Starting $description..." make UNATTENDED=yes "$target" 2>&1 | tee -a "$bld_log" @@ -75,7 +84,7 @@ make_vpp_test() { bld_log="${bld_log}-make_vpp_test_${target}-bld.log" makefile_target="^${target}:" - if [ -z "$(grep -e $makefile_target test/Makefile)" ] ; then + if ! grep -e "$makefile_target" test/Makefile ; then echo "Make test target '$target' does not exist in VPP branch '$branch'!" return fi @@ -83,7 +92,7 @@ make_vpp_test() { description="'make -C test $target' in $(pwd) ($branch)" echo_log " Starting $description..." make WS_ROOT="$DOCKER_VPP_DIR" BR="$DOCKER_VPP_DIR/build-root" \ - TEST_DIR="$DOCKER_VPP_DIR/test" -C test $target 2>&1 | tee -a $bld_log + TEST_DIR="$DOCKER_VPP_DIR"/test -C test "$target" 2>&1 | tee -a "$bld_log" remove_pyc_files_and_pycache_dirs git checkout -q -- . echo_log " Completed $description!" @@ -94,6 +103,10 @@ docker_build_setup_vpp() { if [ ! -d "$DOCKER_VPP_DIR" ] ; then echo_log "Cloning VPP into $DOCKER_VPP_DIR..." git clone -q https://gerrit.fd.io/r/vpp $DOCKER_VPP_DIR + if [ -d "$DOCKER_DOWNLOADS_DIR" ] ; then + mkdir -p "$DOCKER_VPP_DL_CACHE_DIR" + cp -a "$DOCKER_DOWNLOADS_DIR"/* "$DOCKER_VPP_DL_CACHE_DIR" + fi fi clean_git_repo $DOCKER_VPP_DIR fi @@ -109,7 +122,7 @@ docker_build_setup_vpp() { # to create an enumerated set of jobs jobs that match the # definitions here. declare -A VPP_BRANCHES -VPP_BRANCHES["debian-11"]="stable/2310 stable/2402 master" -VPP_BRANCHES["ubuntu-20.04"]="stable/2310 stable/2402 master" -VPP_BRANCHES["ubuntu-22.04"]="stable/2310 stable/2402 master" +VPP_BRANCHES["debian-11"]="stable/2402 stable/2406 master" +VPP_BRANCHES["ubuntu-20.04"]="stable/2402 stable/2406 master" +VPP_BRANCHES["ubuntu-22.04"]="stable/2402 stable/2406 master" export VPP_BRANCHES diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml index 1ee98ba8e..545789edc 100644 --- a/jjb/vpp/docs.yaml +++ b/jjb/vpp/docs.yaml @@ -15,9 +15,6 @@ - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index c421b2931..49c6ce8f3 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -39,9 +39,6 @@ - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' @@ -106,11 +103,6 @@ branch-refspec: '' make-test-os: 'ubuntu-20.04 ubuntu-22.04' make-test-multiworker-os: 'debian-11' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: 'debian-11' - '2402': branch: 'stable/2402' branch-refspec: '' @@ -142,9 +134,6 @@ - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' @@ -249,11 +238,6 @@ branch-refspec: '' make-test-os: 'ubuntu-20.04 ubuntu-22.04' make-test-multiworker-os: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - make-test-os: 'ubuntu-20.04 ubuntu-22.04' - make-test-multiworker-os: '' - '2402': branch: 'stable/2402' branch-refspec: '' @@ -371,9 +355,6 @@ - 'master': branch: 'master' branch-refspec: '' - - '2310': - branch: 'stable/2310' - branch-refspec: '' - '2402': branch: 'stable/2402' branch-refspec: '' |