summaryrefslogtreecommitdiffstats
path: root/docker/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'docker/scripts')
-rwxr-xr-xdocker/scripts/dbld_csit_find_ansible_packages.py2
-rwxr-xr-xdocker/scripts/dbld_csit_install_packages.sh12
-rwxr-xr-xdocker/scripts/dbld_lfit_requirements.sh2
-rwxr-xr-xdocker/scripts/dbld_vpp_install_packages.sh11
-rw-r--r--docker/scripts/lib_apt.sh47
-rw-r--r--docker/scripts/lib_csit.sh90
-rw-r--r--docker/scripts/lib_vpp.sh6
7 files changed, 69 insertions, 101 deletions
diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py
index 9da578d25..52ed4c3b0 100755
--- a/docker/scripts/dbld_csit_find_ansible_packages.py
+++ b/docker/scripts/dbld_csit_find_ansible_packages.py
@@ -58,7 +58,7 @@ def is_csit_ansible_yaml_file(filename: str):
(root,ext) = os.path.splitext(filename)
if ext == '.yaml' \
and filename.find('csit/') >= 0 \
- and filename.find('/ansible/') > 0 \
+ and filename.find('ansible/') > 0 \
and os.path.isfile(filename):
return True
else:
diff --git a/docker/scripts/dbld_csit_install_packages.sh b/docker/scripts/dbld_csit_install_packages.sh
index d8ce597fb..e303ce28e 100755
--- a/docker/scripts/dbld_csit_install_packages.sh
+++ b/docker/scripts/dbld_csit_install_packages.sh
@@ -42,7 +42,19 @@ for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do
csit_install_packages "$csit_branch"
# Install/cache python packages
+ csit_install_hugo "$csit_branch"
+
+ # Install/cache python packages
csit_pip_cache "$csit_branch"
done
+# Install csit OS packages
+csit_install_packages "master"
+
+# Install/cache python packages
+csit_install_hugo "master"
+
+# Install/cache python packages
+csit_pip_cache "master"
+
echo_log -e "Completed $(basename $0)!\n\n=========="
diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index b60e24dfd..ca54edd89 100755
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -53,6 +53,6 @@ mkdir -p "$LF_VENV"
OLD_PATH="$PATH"
python3 -m venv "$LF_VENV"
PATH="$LF_VENV/bin:$PATH"
-python3 -m pip install --upgrade pip==21.2.4
+python3 -m pip install pip --upgrade
python3 -m pip install --upgrade --upgrade-strategy eager lftools
PATH="$OLD_PATH"
diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh
index c30fb25b5..5e1e0f8f0 100755
--- a/docker/scripts/dbld_vpp_install_packages.sh
+++ b/docker/scripts/dbld_vpp_install_packages.sh
@@ -35,11 +35,6 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
# Install OS packages
make_vpp "install-dep" "$branch"
- if [ "$OS_ID" = "ubuntu" ] && [ "$OS_ARCH" = "x86_64" ] ; then
- # 'Make docs jobs are only run on ubuntu x86_64 executors
- # so only run for ubuntu build executors.
- make_vpp "docs-venv" "$branch"
- fi
# Download, build, and cache external deps packages
make_vpp "install-ext-deps" "$branch"
@@ -57,6 +52,12 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'"
exit 1
fi
+ # TODO: remove this after all supported VPP branches have removed
+ # python3-virtualenv & virtualenv from install-deps which are no longer
+ # used in vpp repo. These packages can mess up csit virtualenv
+ # installation which uses pip3
+ sudo apt remove -y --purge --autoremove python3-virtualenv virtualenv || true
+
# Install/cache python packages
make_vpp_test "test-dep" "$branch"
if [ "$OS_ID" = "ubuntu" ] ; then
diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index eae97b43b..6cf37ae76 100644
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -104,22 +104,6 @@ ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"
# libpcap-dev for python pypcap install (CSIT)
# sshpass for CSIT jobs
#
-# From .../csit/resources/tools/presentation/run_report_*.sh: // TO BE REMOVED IN RLS2306
-# libxml2
-# libxml2-dev
-# libxslt-dev
-# build-essential
-# zlib1g-dev
-# unzip
-# xvrb
-# texlive-latex-recommended
-# texlive-fonts-recommended
-# texlive-fonts-extra
-# texlive-latex-extra
-# latexmk
-# wkhtmltopdf
-# inkscape
-#
RUN apt-get update -qq \\
&& apt-get install -y \\
apt-transport-https \\
@@ -138,21 +122,16 @@ RUN apt-get update -qq \\
git-review \\
gnupg-agent \\
graphviz \\
- inkscape \\
iproute2 \\
iputils-clockdiff \\
iputils-ping \\
iputils-tracepath \\
jq \\
- latexmk \\
libffi-dev \\
liblapack-dev \\
libopenblas-dev \\
libpcap-dev \\
- libxml2 \\
- libxml2-dev \\
libxml-xpath-perl \\
- libxslt-dev \\
make \\
python3-pip \\
python3-venv \\
@@ -162,42 +141,22 @@ RUN apt-get update -qq \\
sshpass \\
sysstat \\
sudo \\
- texlive-fonts-extra \\
- texlive-fonts-recommended \\
- texlive-latex-extra \\
- texlive-latex-recommended \\
traceroute \\
tree \\
- unzip \\
vim \\
wget \\
- wkhtmltopdf \\
xmlstarlet \\
- xvfb \\
xz-utils \\
- zlib1g-dev \\
&& curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\
&& curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\
-EOF
-
- cat <<EOF >>"$DOCKERFILE"
&& rm -r /var/lib/apt/lists/*
# Install terraform for CSIT
#
-RUN wget https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_$dpkg_arch.zip \\
- && unzip terraform_1.4.2_linux_$dpkg_arch.zip \\
+RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$dpkg_arch.zip \\
+ && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\
&& mv terraform /usr/bin \\
- && rm -f terraform_1.4.2_linux_$dpkg_arch.zip
-
-# Install Hugo and Go for CSIT
-RUN wget https://github.com/gohugoio/hugo/releases/download/v0.111.3/hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
- && dpkg -i hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
- && rm -f hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
- && wget https://go.dev/dl/go1.20.2.linux-$dpkg_arch.tar.gz \\
- && rm -rf /usr/local/go \\
- && tar -C /usr/local -xzf go1.20.2.linux-$dpkg_arch.tar.gz \\
- && rm -f go1.20.2.linux-$dpkg_arch.tar.gz
+ && rm -f terraform_1.7.3_linux_$dpkg_arch.zip
# Install packages for all project branches
#
diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh
index 36ab0b27d..2e877f803 100644
--- a/docker/scripts/lib_csit.sh
+++ b/docker/scripts/lib_csit.sh
@@ -20,7 +20,7 @@ if [ -n "$(alias lib_csit_imported 2> /dev/null)" ] ; then
fi
alias lib_csit_imported=true
-export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname $BASH_SOURCE)}"
+export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]})}"
. "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
. "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh"
@@ -50,7 +50,7 @@ csit_checkout_branch_for_vpp() {
&& source "$csit_bash_function_dir/branch.sh"
CSIT_DIR="$csit_dir" checkout_csit_for_vpp "$vpp_branch"
- # shellcheck disable=SC2034
+ # shellcheck disable=SC2034,SC2063
csit_branch="$(git branch | grep -e '^*' | mawk '{print $2}')"
}
@@ -69,14 +69,14 @@ csit_install_packages() {
git clean -qfdx
python3 -m pip install pyyaml
- local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul"
+ local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp"
[ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf"
# Not in double quotes to let bash remove newline characters
local yaml_files
yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)"
packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)"
- packages="${packages/bionic /}"
+ packages="${packages/jammy /}"
packages="${packages/focal /}"
packages="${packages/libmbedcrypto1/libmbedcrypto3}"
packages="${packages/libmbedtls10/libmbedtls12}"
@@ -85,10 +85,10 @@ csit_install_packages() {
if [ -n "$packages" ] ; then
case "$OS_NAME" in
ubuntu*)
- apt_install_packages $packages
+ apt_install_packages "$packages" 2>&1 | tee -a "$bld_log"
;;
debian*)
- apt_install_packages $packages
+ apt_install_packages "$packages" 2>&1 | tee -a "$bld_log"
;;
*)
echo "Unsupported OS ($OS_ID): CSIT packages NOT INSTALLED!"
@@ -97,6 +97,35 @@ csit_install_packages() {
fi
}
+csit_install_hugo() {
+ local branch="$1"
+ CSIT_DIR="$DOCKER_CSIT_DIR"
+
+ if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \
+ && [ -f "$CSIT_DIR/requirements.txt" ]; then
+
+ local branchname
+ # use bash variable substitution to replace '/' with '_' to convert from
+ # vpp to csit branch name nomenclature
+ branchname="${branch////_}"
+ local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function"
+ local bld_log="$DOCKER_BUILD_LOG_DIR"
+ bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_install_hugo-bld.log"
+
+ description="Install CSIT hugo packages from $branch branch"
+ echo_log " Starting $description..."
+ git clean -qfdx
+
+ source "$csit_bash_function_dir"/hugo.sh
+ go_install 2>&1 | tee -a "$bld_log"
+ hugo_install 2>&1 | tee -a "$bld_log"
+
+ else
+ echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!"
+ return 1
+ fi
+}
+
csit_pip_cache() {
local branch="$1"
# ensure PS1 is defined (used by virtualenv activate script)
@@ -113,60 +142,27 @@ csit_pip_cache() {
local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function"
local bld_log="$DOCKER_BUILD_LOG_DIR"
bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_pip_cache-bld.log"
- local pip_cmd="python3 -m pip --disable-pip-version-check"
export PYTHONPATH=$CSIT_DIR
description="Install CSIT python packages from $branch branch"
- echo_log " Starting $description..."
- git clean -qfdx
+ echo_log " Starting $description..." 2>&1 | tee -a "$bld_log"
+ git clean -qfdx 2>&1 | tee -a "$bld_log"
rm -rf "$PYTHONPATH/env"
- # Virtualenv version is pinned in common.sh in newer csit branches.
- # (note: xargs removes leading/trailing spaces)
+ # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt)
local common_sh="$csit_bash_function_dir/common.sh"
- install_virtualenv="$(grep 'virtualenv' $common_sh | grep pip | grep install | cut -d'|' -f1 | xargs)"
- $install_virtualenv
- virtualenv --no-download --python="$(which python3)" "$CSIT_DIR/env"
- source "$CSIT_DIR/env/bin/activate"
-
- if [ "$OS_ARCH" = "aarch64" ] ; then
- local numpy_ver
- numpy_ver="$(grep numpy $PYTHONPATH/requirements.txt)"
- [ -n "$numpy_ver" ] && $pip_cmd install $numpy_ver 2>&1 | \
- tee -a $bld_log
- fi
-
- # TODO: Update CSIT release branches to PyYAML==6.0.1 or greater
- # then remove this workaround: https://github.com/yaml/pyyaml/issues/736
- if grep -q 'PyYAML==5.4.1' "$PYTHONPATH/requirements.txt" ; then
- local constraintfile
- constraintfile="/tmp/constraint.txt"
-
- # create a constraint file that limits the Cython version to one that should work
- echo 'Cython < 3.0' > "$constraintfile"
+ # shellcheck disable=1090
+ source "$common_sh"
+ activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log"
- # seed pip's local wheel cache with a PyYAML wheel
- PIP_CONSTRAINT="$constraintfile" $pip_cmd wheel PyYAML==5.4.1
-
- rm -f "$constraintfile"
- fi
-
- # Install csit python requirements
- $pip_cmd install -r "$CSIT_DIR/requirements.txt" 2>&1 | \
- tee -a "$bld_log"
# Install tox python requirements
- $pip_cmd install -r "$CSIT_DIR/tox-requirements.txt" 2>&1 | \
+ activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\
tee -a "$bld_log"
- # Run tox which installs pylint requirments
- pushd "$CSIT_DIR" >& /dev/null || exit 1
- tox || true
- popd >& /dev/null || exit 1
# Clean up virtualenv directories
- deactivate
git checkout -q -- .
git clean -qfdx
- echo_log " Completed $description!"
+ echo_log " Completed $description!" 2>&1 | tee -a "$bld_log"
else
echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!"
return 1
diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index 312bbf760..39e2ca192 100644
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -96,7 +96,7 @@ docker_build_setup_vpp() {
# to create an enumerated set of jobs jobs that match the
# definitions here.
declare -A VPP_BRANCHES
-VPP_BRANCHES["debian-11"]="stable/2306 stable/2310 master"
-VPP_BRANCHES["ubuntu-20.04"]="stable/2306 stable/2310 master"
-VPP_BRANCHES["ubuntu-22.04"]="stable/2306 stable/2310 master"
+VPP_BRANCHES["debian-11"]="stable/2310 stable/2402 master"
+VPP_BRANCHES["ubuntu-20.04"]="stable/2310 stable/2402 master"
+VPP_BRANCHES["ubuntu-22.04"]="stable/2310 stable/2402 master"
export VPP_BRANCHES