diff options
Diffstat (limited to 'docker/scripts')
| -rwxr-xr-x | docker/scripts/dbld_csit_find_ansible_packages.py | 2 | ||||
| -rwxr-xr-x | docker/scripts/dbld_csit_install_packages.sh | 12 | ||||
| -rwxr-xr-x | docker/scripts/dbld_lfit_requirements.sh | 2 | ||||
| -rwxr-xr-x | docker/scripts/dbld_vpp_install_packages.sh | 11 | ||||
| -rw-r--r-- | docker/scripts/lib_apt.sh | 47 | ||||
| -rw-r--r-- | docker/scripts/lib_csit.sh | 90 | ||||
| -rw-r--r-- | docker/scripts/lib_vpp.sh | 6 |
7 files changed, 69 insertions, 101 deletions
diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py index 9da578d25..52ed4c3b0 100755 --- a/docker/scripts/dbld_csit_find_ansible_packages.py +++ b/docker/scripts/dbld_csit_find_ansible_packages.py @@ -58,7 +58,7 @@ def is_csit_ansible_yaml_file(filename: str): (root,ext) = os.path.splitext(filename) if ext == '.yaml' \ and filename.find('csit/') >= 0 \ - and filename.find('/ansible/') > 0 \ + and filename.find('ansible/') > 0 \ and os.path.isfile(filename): return True else: diff --git a/docker/scripts/dbld_csit_install_packages.sh b/docker/scripts/dbld_csit_install_packages.sh index d8ce597fb..e303ce28e 100755 --- a/docker/scripts/dbld_csit_install_packages.sh +++ b/docker/scripts/dbld_csit_install_packages.sh @@ -42,7 +42,19 @@ for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do csit_install_packages "$csit_branch" # Install/cache python packages + csit_install_hugo "$csit_branch" + + # Install/cache python packages csit_pip_cache "$csit_branch" done +# Install csit OS packages +csit_install_packages "master" + +# Install/cache python packages +csit_install_hugo "master" + +# Install/cache python packages +csit_pip_cache "master" + echo_log -e "Completed $(basename $0)!\n\n==========" diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh index b60e24dfd..ca54edd89 100755 --- a/docker/scripts/dbld_lfit_requirements.sh +++ b/docker/scripts/dbld_lfit_requirements.sh @@ -53,6 +53,6 @@ mkdir -p "$LF_VENV" OLD_PATH="$PATH" python3 -m venv "$LF_VENV" PATH="$LF_VENV/bin:$PATH" -python3 -m pip install --upgrade pip==21.2.4 +python3 -m pip install pip --upgrade python3 -m pip install --upgrade --upgrade-strategy eager lftools PATH="$OLD_PATH" diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh index c30fb25b5..5e1e0f8f0 100755 --- a/docker/scripts/dbld_vpp_install_packages.sh +++ b/docker/scripts/dbld_vpp_install_packages.sh @@ -35,11 +35,6 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do # Install OS packages make_vpp "install-dep" "$branch" - if [ "$OS_ID" = "ubuntu" ] && [ "$OS_ARCH" = "x86_64" ] ; then - # 'Make docs jobs are only run on ubuntu x86_64 executors - # so only run for ubuntu build executors. - make_vpp "docs-venv" "$branch" - fi # Download, build, and cache external deps packages make_vpp "install-ext-deps" "$branch" @@ -57,6 +52,12 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'" exit 1 fi + # TODO: remove this after all supported VPP branches have removed + # python3-virtualenv & virtualenv from install-deps which are no longer + # used in vpp repo. These packages can mess up csit virtualenv + # installation which uses pip3 + sudo apt remove -y --purge --autoremove python3-virtualenv virtualenv || true + # Install/cache python packages make_vpp_test "test-dep" "$branch" if [ "$OS_ID" = "ubuntu" ] ; then diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh index eae97b43b..6cf37ae76 100644 --- a/docker/scripts/lib_apt.sh +++ b/docker/scripts/lib_apt.sh @@ -104,22 +104,6 @@ ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8" # libpcap-dev for python pypcap install (CSIT) # sshpass for CSIT jobs # -# From .../csit/resources/tools/presentation/run_report_*.sh: // TO BE REMOVED IN RLS2306 -# libxml2 -# libxml2-dev -# libxslt-dev -# build-essential -# zlib1g-dev -# unzip -# xvrb -# texlive-latex-recommended -# texlive-fonts-recommended -# texlive-fonts-extra -# texlive-latex-extra -# latexmk -# wkhtmltopdf -# inkscape -# RUN apt-get update -qq \\ && apt-get install -y \\ apt-transport-https \\ @@ -138,21 +122,16 @@ RUN apt-get update -qq \\ git-review \\ gnupg-agent \\ graphviz \\ - inkscape \\ iproute2 \\ iputils-clockdiff \\ iputils-ping \\ iputils-tracepath \\ jq \\ - latexmk \\ libffi-dev \\ liblapack-dev \\ libopenblas-dev \\ libpcap-dev \\ - libxml2 \\ - libxml2-dev \\ libxml-xpath-perl \\ - libxslt-dev \\ make \\ python3-pip \\ python3-venv \\ @@ -162,42 +141,22 @@ RUN apt-get update -qq \\ sshpass \\ sysstat \\ sudo \\ - texlive-fonts-extra \\ - texlive-fonts-recommended \\ - texlive-latex-extra \\ - texlive-latex-recommended \\ traceroute \\ tree \\ - unzip \\ vim \\ wget \\ - wkhtmltopdf \\ xmlstarlet \\ - xvfb \\ xz-utils \\ - zlib1g-dev \\ && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\ && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\ -EOF - - cat <<EOF >>"$DOCKERFILE" && rm -r /var/lib/apt/lists/* # Install terraform for CSIT # -RUN wget https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_$dpkg_arch.zip \\ - && unzip terraform_1.4.2_linux_$dpkg_arch.zip \\ +RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$dpkg_arch.zip \\ + && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\ && mv terraform /usr/bin \\ - && rm -f terraform_1.4.2_linux_$dpkg_arch.zip - -# Install Hugo and Go for CSIT -RUN wget https://github.com/gohugoio/hugo/releases/download/v0.111.3/hugo_extended_0.111.3_linux-$dpkg_arch.deb \\ - && dpkg -i hugo_extended_0.111.3_linux-$dpkg_arch.deb \\ - && rm -f hugo_extended_0.111.3_linux-$dpkg_arch.deb \\ - && wget https://go.dev/dl/go1.20.2.linux-$dpkg_arch.tar.gz \\ - && rm -rf /usr/local/go \\ - && tar -C /usr/local -xzf go1.20.2.linux-$dpkg_arch.tar.gz \\ - && rm -f go1.20.2.linux-$dpkg_arch.tar.gz + && rm -f terraform_1.7.3_linux_$dpkg_arch.zip # Install packages for all project branches # diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh index 36ab0b27d..2e877f803 100644 --- a/docker/scripts/lib_csit.sh +++ b/docker/scripts/lib_csit.sh @@ -20,7 +20,7 @@ if [ -n "$(alias lib_csit_imported 2> /dev/null)" ] ; then fi alias lib_csit_imported=true -export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname $BASH_SOURCE)}" +export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]})}" . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh" . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh" @@ -50,7 +50,7 @@ csit_checkout_branch_for_vpp() { && source "$csit_bash_function_dir/branch.sh" CSIT_DIR="$csit_dir" checkout_csit_for_vpp "$vpp_branch" - # shellcheck disable=SC2034 + # shellcheck disable=SC2034,SC2063 csit_branch="$(git branch | grep -e '^*' | mawk '{print $2}')" } @@ -69,14 +69,14 @@ csit_install_packages() { git clean -qfdx python3 -m pip install pyyaml - local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul" + local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp" [ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf" # Not in double quotes to let bash remove newline characters local yaml_files yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)" packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)" - packages="${packages/bionic /}" + packages="${packages/jammy /}" packages="${packages/focal /}" packages="${packages/libmbedcrypto1/libmbedcrypto3}" packages="${packages/libmbedtls10/libmbedtls12}" @@ -85,10 +85,10 @@ csit_install_packages() { if [ -n "$packages" ] ; then case "$OS_NAME" in ubuntu*) - apt_install_packages $packages + apt_install_packages "$packages" 2>&1 | tee -a "$bld_log" ;; debian*) - apt_install_packages $packages + apt_install_packages "$packages" 2>&1 | tee -a "$bld_log" ;; *) echo "Unsupported OS ($OS_ID): CSIT packages NOT INSTALLED!" @@ -97,6 +97,35 @@ csit_install_packages() { fi } +csit_install_hugo() { + local branch="$1" + CSIT_DIR="$DOCKER_CSIT_DIR" + + if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \ + && [ -f "$CSIT_DIR/requirements.txt" ]; then + + local branchname + # use bash variable substitution to replace '/' with '_' to convert from + # vpp to csit branch name nomenclature + branchname="${branch////_}" + local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function" + local bld_log="$DOCKER_BUILD_LOG_DIR" + bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_install_hugo-bld.log" + + description="Install CSIT hugo packages from $branch branch" + echo_log " Starting $description..." + git clean -qfdx + + source "$csit_bash_function_dir"/hugo.sh + go_install 2>&1 | tee -a "$bld_log" + hugo_install 2>&1 | tee -a "$bld_log" + + else + echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!" + return 1 + fi +} + csit_pip_cache() { local branch="$1" # ensure PS1 is defined (used by virtualenv activate script) @@ -113,60 +142,27 @@ csit_pip_cache() { local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function" local bld_log="$DOCKER_BUILD_LOG_DIR" bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_pip_cache-bld.log" - local pip_cmd="python3 -m pip --disable-pip-version-check" export PYTHONPATH=$CSIT_DIR description="Install CSIT python packages from $branch branch" - echo_log " Starting $description..." - git clean -qfdx + echo_log " Starting $description..." 2>&1 | tee -a "$bld_log" + git clean -qfdx 2>&1 | tee -a "$bld_log" rm -rf "$PYTHONPATH/env" - # Virtualenv version is pinned in common.sh in newer csit branches. - # (note: xargs removes leading/trailing spaces) + # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt) local common_sh="$csit_bash_function_dir/common.sh" - install_virtualenv="$(grep 'virtualenv' $common_sh | grep pip | grep install | cut -d'|' -f1 | xargs)" - $install_virtualenv - virtualenv --no-download --python="$(which python3)" "$CSIT_DIR/env" - source "$CSIT_DIR/env/bin/activate" - - if [ "$OS_ARCH" = "aarch64" ] ; then - local numpy_ver - numpy_ver="$(grep numpy $PYTHONPATH/requirements.txt)" - [ -n "$numpy_ver" ] && $pip_cmd install $numpy_ver 2>&1 | \ - tee -a $bld_log - fi - - # TODO: Update CSIT release branches to PyYAML==6.0.1 or greater - # then remove this workaround: https://github.com/yaml/pyyaml/issues/736 - if grep -q 'PyYAML==5.4.1' "$PYTHONPATH/requirements.txt" ; then - local constraintfile - constraintfile="/tmp/constraint.txt" - - # create a constraint file that limits the Cython version to one that should work - echo 'Cython < 3.0' > "$constraintfile" + # shellcheck disable=1090 + source "$common_sh" + activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log" - # seed pip's local wheel cache with a PyYAML wheel - PIP_CONSTRAINT="$constraintfile" $pip_cmd wheel PyYAML==5.4.1 - - rm -f "$constraintfile" - fi - - # Install csit python requirements - $pip_cmd install -r "$CSIT_DIR/requirements.txt" 2>&1 | \ - tee -a "$bld_log" # Install tox python requirements - $pip_cmd install -r "$CSIT_DIR/tox-requirements.txt" 2>&1 | \ + activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\ tee -a "$bld_log" - # Run tox which installs pylint requirments - pushd "$CSIT_DIR" >& /dev/null || exit 1 - tox || true - popd >& /dev/null || exit 1 # Clean up virtualenv directories - deactivate git checkout -q -- . git clean -qfdx - echo_log " Completed $description!" + echo_log " Completed $description!" 2>&1 | tee -a "$bld_log" else echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!" return 1 diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh index 312bbf760..39e2ca192 100644 --- a/docker/scripts/lib_vpp.sh +++ b/docker/scripts/lib_vpp.sh @@ -96,7 +96,7 @@ docker_build_setup_vpp() { # to create an enumerated set of jobs jobs that match the # definitions here. declare -A VPP_BRANCHES -VPP_BRANCHES["debian-11"]="stable/2306 stable/2310 master" -VPP_BRANCHES["ubuntu-20.04"]="stable/2306 stable/2310 master" -VPP_BRANCHES["ubuntu-22.04"]="stable/2306 stable/2310 master" +VPP_BRANCHES["debian-11"]="stable/2310 stable/2402 master" +VPP_BRANCHES["ubuntu-20.04"]="stable/2310 stable/2402 master" +VPP_BRANCHES["ubuntu-22.04"]="stable/2310 stable/2402 master" export VPP_BRANCHES |