Merge "Add stable/2402 branch and remove stable/2306 branch to docker executor image scripts"

10 files changed, 75 insertions, 126 deletions

diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py
index 9da578d2..52ed4c3b 100755
--- a/docker/scripts/dbld_csit_find_ansible_packages.py
+++ b/docker/scripts/dbld_csit_find_ansible_packages.py
@@ -58,7 +58,7 @@ def is_csit_ansible_yaml_file(filename: str):
      (root,ext) = os.path.splitext(filename)
      if ext == '.yaml' \
         and filename.find('csit/') >= 0 \
-        and filename.find('/ansible/') > 0 \
+        and filename.find('ansible/') > 0 \
         and os.path.isfile(filename):
          return True
      else:
diff --git a/docker/scripts/dbld_csit_install_packages.sh b/docker/scripts/dbld_csit_install_packages.sh
index d8ce597f..e303ce28 100755
--- a/docker/scripts/dbld_csit_install_packages.sh
+++ b/docker/scripts/dbld_csit_install_packages.sh
@@ -42,7 +42,19 @@ for vpp_branch in ${VPP_BRANCHES[$OS_NAME]} ; do
     csit_install_packages "$csit_branch"
 
     # Install/cache python packages
+    csit_install_hugo "$csit_branch"
+
+    # Install/cache python packages
     csit_pip_cache "$csit_branch"
 done
 
+# Install csit OS packages
+csit_install_packages "master"
+
+# Install/cache python packages
+csit_install_hugo "master"
+
+# Install/cache python packages
+csit_pip_cache "master"
+
 echo_log -e "Completed $(basename $0)!\n\n=========="
diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index b60e24df..ca54edd8 100755
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -53,6 +53,6 @@ mkdir -p "$LF_VENV"
 OLD_PATH="$PATH"
 python3 -m venv "$LF_VENV"
 PATH="$LF_VENV/bin:$PATH"
-python3 -m pip install --upgrade pip==21.2.4
+python3 -m pip install pip --upgrade
 python3 -m pip install --upgrade --upgrade-strategy eager lftools
 PATH="$OLD_PATH"
diff --git a/docker/scripts/dbld_vpp_install_packages.sh b/docker/scripts/dbld_vpp_install_packages.sh
index c30fb25b..5e1e0f8f 100755
--- a/docker/scripts/dbld_vpp_install_packages.sh
+++ b/docker/scripts/dbld_vpp_install_packages.sh
@@ -35,11 +35,6 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
 
     # Install OS packages
     make_vpp "install-dep" "$branch"
-    if [ "$OS_ID" = "ubuntu" ] && [ "$OS_ARCH" = "x86_64" ] ; then
-        # 'Make docs jobs are only run on ubuntu x86_64 executors
-        #  so only run for ubuntu build executors.
-        make_vpp "docs-venv" "$branch"
-    fi
 
     # Download, build, and cache external deps packages
     make_vpp "install-ext-deps" "$branch"
@@ -57,6 +52,12 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
         echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'"
         exit 1
     fi
+    # TODO: remove this after all supported VPP branches have removed
+    # python3-virtualenv & virtualenv from install-deps which are no longer
+    # used in vpp repo.  These packages can mess up csit virtualenv
+    # installation which uses pip3
+    sudo apt remove -y --purge --autoremove python3-virtualenv virtualenv || true
+
     # Install/cache python packages
     make_vpp_test "test-dep" "$branch"
     if [ "$OS_ID" = "ubuntu" ] ; then
diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index eae97b43..6cf37ae7 100644
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -104,22 +104,6 @@ ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"
 #        libpcap-dev      for python pypcap install (CSIT)
 #        sshpass          for CSIT jobs
 #
-#        From .../csit/resources/tools/presentation/run_report_*.sh: // TO BE REMOVED IN RLS2306
-#        libxml2
-#        libxml2-dev
-#        libxslt-dev
-#        build-essential
-#        zlib1g-dev
-#        unzip
-#        xvrb
-#        texlive-latex-recommended
-#        texlive-fonts-recommended
-#        texlive-fonts-extra
-#        texlive-latex-extra
-#        latexmk
-#        wkhtmltopdf
-#        inkscape
-#
 RUN apt-get update -qq \\
   && apt-get install -y \\
              apt-transport-https \\
@@ -138,21 +122,16 @@ RUN apt-get update -qq \\
              git-review \\
              gnupg-agent \\
              graphviz \\
-             inkscape \\
              iproute2 \\
              iputils-clockdiff \\
              iputils-ping \\
              iputils-tracepath \\
              jq \\
-             latexmk \\
              libffi-dev \\
              liblapack-dev \\
              libopenblas-dev \\
              libpcap-dev \\
-             libxml2 \\
-             libxml2-dev \\
              libxml-xpath-perl \\
-             libxslt-dev \\
              make \\
              python3-pip \\
              python3-venv \\
@@ -162,42 +141,22 @@ RUN apt-get update -qq \\
              sshpass \\
              sysstat \\
              sudo \\
-             texlive-fonts-extra \\
-             texlive-fonts-recommended \\
-             texlive-latex-extra \\
-             texlive-latex-recommended \\
              traceroute \\
              tree \\
-             unzip \\
              vim \\
              wget \\
-             wkhtmltopdf \\
              xmlstarlet \\
-             xvfb \\
              xz-utils \\
-             zlib1g-dev \\
   && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\
   && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\
-EOF
-
-    cat <<EOF >>"$DOCKERFILE"
   && rm -r /var/lib/apt/lists/*
 
 # Install terraform for CSIT
 #
-RUN wget https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_$dpkg_arch.zip \\
-  && unzip terraform_1.4.2_linux_$dpkg_arch.zip \\
+RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$dpkg_arch.zip \\
+  && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\
   && mv terraform /usr/bin \\
-  && rm -f terraform_1.4.2_linux_$dpkg_arch.zip
-
-# Install Hugo and Go for CSIT
-RUN wget https://github.com/gohugoio/hugo/releases/download/v0.111.3/hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
-  && dpkg -i hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
-  && rm -f hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
-  && wget https://go.dev/dl/go1.20.2.linux-$dpkg_arch.tar.gz \\
-  && rm -rf /usr/local/go \\
-  && tar -C /usr/local -xzf go1.20.2.linux-$dpkg_arch.tar.gz \\
-  && rm -f go1.20.2.linux-$dpkg_arch.tar.gz
+  && rm -f terraform_1.7.3_linux_$dpkg_arch.zip
 
 # Install packages for all project branches
 #
diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh
index 36ab0b27..2e877f80 100644
--- a/docker/scripts/lib_csit.sh
+++ b/docker/scripts/lib_csit.sh
@@ -20,7 +20,7 @@ if [ -n "$(alias lib_csit_imported 2> /dev/null)" ] ; then
 fi
 alias lib_csit_imported=true
 
-export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname $BASH_SOURCE)}"
+export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]})}"
 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
 . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh"
 
@@ -50,7 +50,7 @@ csit_checkout_branch_for_vpp() {
         && source "$csit_bash_function_dir/branch.sh"
     CSIT_DIR="$csit_dir" checkout_csit_for_vpp "$vpp_branch"
 
-    # shellcheck disable=SC2034
+    # shellcheck disable=SC2034,SC2063
     csit_branch="$(git branch | grep -e '^*' | mawk '{print $2}')"
 }
 
@@ -69,14 +69,14 @@ csit_install_packages() {
     git clean -qfdx
     python3 -m pip install pyyaml
 
-    local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul"
+    local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp"
     [ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf"
 
     # Not in double quotes to let bash remove newline characters
     local yaml_files
     yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)"
     packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)"
-    packages="${packages/bionic /}"
+    packages="${packages/jammy /}"
     packages="${packages/focal /}"
     packages="${packages/libmbedcrypto1/libmbedcrypto3}"
     packages="${packages/libmbedtls10/libmbedtls12}"
@@ -85,10 +85,10 @@ csit_install_packages() {
     if [ -n "$packages" ] ; then
         case "$OS_NAME" in
             ubuntu*)
-                apt_install_packages $packages
+                apt_install_packages "$packages" 2>&1 | tee -a "$bld_log"
                 ;;
             debian*)
-                apt_install_packages $packages
+                apt_install_packages "$packages" 2>&1 | tee -a "$bld_log"
                 ;;
             *)
                 echo "Unsupported OS ($OS_ID): CSIT packages NOT INSTALLED!"
@@ -97,6 +97,35 @@ csit_install_packages() {
     fi
 }
 
+csit_install_hugo() {
+    local branch="$1"
+    CSIT_DIR="$DOCKER_CSIT_DIR"
+
+    if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \
+           && [ -f "$CSIT_DIR/requirements.txt" ]; then
+
+        local branchname
+        # use bash variable substitution to replace '/' with '_' to convert from
+        # vpp to csit branch name nomenclature
+        branchname="${branch////_}"
+        local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function"
+        local bld_log="$DOCKER_BUILD_LOG_DIR"
+        bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_install_hugo-bld.log"
+
+        description="Install CSIT hugo packages from $branch branch"
+        echo_log "    Starting  $description..."
+        git clean -qfdx
+
+        source "$csit_bash_function_dir"/hugo.sh
+        go_install 2>&1 | tee -a "$bld_log"
+        hugo_install 2>&1 | tee -a "$bld_log"
+
+    else
+        echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!"
+        return 1
+    fi
+}
+
 csit_pip_cache() {
     local branch="$1"
     # ensure PS1 is defined (used by virtualenv activate script)
@@ -113,60 +142,27 @@ csit_pip_cache() {
         local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function"
         local bld_log="$DOCKER_BUILD_LOG_DIR"
         bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_pip_cache-bld.log"
-        local pip_cmd="python3 -m pip --disable-pip-version-check"
         export PYTHONPATH=$CSIT_DIR
 
         description="Install CSIT python packages from $branch branch"
-        echo_log "    Starting  $description..."
-        git clean -qfdx
+        echo_log "    Starting  $description..." 2>&1 | tee -a "$bld_log"
+        git clean -qfdx 2>&1 | tee -a "$bld_log"
         rm -rf "$PYTHONPATH/env"
 
-        # Virtualenv version is pinned in common.sh in newer csit branches.
-        # (note: xargs removes leading/trailing spaces)
+        # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt)
         local common_sh="$csit_bash_function_dir/common.sh"
-        install_virtualenv="$(grep 'virtualenv' $common_sh | grep pip | grep install | cut -d'|' -f1 | xargs)"
-        $install_virtualenv
-        virtualenv --no-download --python="$(which python3)" "$CSIT_DIR/env"
-        source "$CSIT_DIR/env/bin/activate"
-
-        if [ "$OS_ARCH" = "aarch64" ] ; then
-            local numpy_ver
-            numpy_ver="$(grep numpy $PYTHONPATH/requirements.txt)"
-            [ -n "$numpy_ver" ] && $pip_cmd install $numpy_ver 2>&1 | \
-                tee -a $bld_log
-        fi
-
-        # TODO: Update CSIT release branches to PyYAML==6.0.1 or greater
-        # then remove this workaround: https://github.com/yaml/pyyaml/issues/736
-        if grep -q 'PyYAML==5.4.1' "$PYTHONPATH/requirements.txt" ; then
-            local constraintfile
-            constraintfile="/tmp/constraint.txt"
-
-            # create a constraint file that limits the Cython version to one that should work
-            echo 'Cython < 3.0' > "$constraintfile"
+        # shellcheck disable=1090
+        source "$common_sh"
+        activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log"
 
-            # seed pip's local wheel cache with a PyYAML wheel
-            PIP_CONSTRAINT="$constraintfile" $pip_cmd wheel PyYAML==5.4.1
-
-            rm -f "$constraintfile"
-        fi
-
-        # Install csit python requirements
-        $pip_cmd install -r "$CSIT_DIR/requirements.txt" 2>&1 | \
-            tee -a "$bld_log"
         # Install tox python requirements
-        $pip_cmd install -r "$CSIT_DIR/tox-requirements.txt" 2>&1 | \
+        activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\
             tee -a "$bld_log"
-        # Run tox which installs pylint requirments
-        pushd "$CSIT_DIR" >& /dev/null || exit 1
-        tox || true
-        popd >& /dev/null || exit 1
 
         # Clean up virtualenv directories
-        deactivate
         git checkout -q -- .
         git clean -qfdx
-        echo_log "    Completed $description!"
+        echo_log "    Completed $description!" 2>&1 | tee -a "$bld_log"
     else
         echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!"
         return 1
diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index 312bbf76..39e2ca19 100644
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -96,7 +96,7 @@ docker_build_setup_vpp() {
 #       to create an enumerated set of jobs jobs that match the
 #       definitions here.
 declare -A VPP_BRANCHES
-VPP_BRANCHES["debian-11"]="stable/2306 stable/2310 master"
-VPP_BRANCHES["ubuntu-20.04"]="stable/2306 stable/2310 master"
-VPP_BRANCHES["ubuntu-22.04"]="stable/2306 stable/2310 master"
+VPP_BRANCHES["debian-11"]="stable/2310 stable/2402 master"
+VPP_BRANCHES["ubuntu-20.04"]="stable/2310 stable/2402 master"
+VPP_BRANCHES["ubuntu-22.04"]="stable/2310 stable/2402 master"
 export VPP_BRANCHES
diff --git a/extras/bash/sandbox_test_functions.sh b/extras/bash/sandbox_test_functions.sh
index c4145675..8db8d4c5 100644
--- a/extras/bash/sandbox_test_functions.sh
+++ b/extras/bash/sandbox_test_functions.sh
@@ -72,6 +72,7 @@ jjb-sandbox-env()
     rm -rf $VENV_DIR \
        && python3 -m venv $VENV_DIR \
        && source $VENV_DIR/bin/activate \
+       && pip3 install --upgrade pip \
        && pip3 install wheel \
        && pip3 install jenkins-job-builder==$jjb_version
 
diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml
index 8480ef3a..006801f4 100644
--- a/jjb/vpp/docs.yaml
+++ b/jjb/vpp/docs.yaml
@@ -15,9 +15,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''
diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 0b3483af..62975f00 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -39,9 +39,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''
@@ -74,11 +71,6 @@
           branch-refspec: ''
           make-test-os: 'ubuntu-20.04 ubuntu-22.04'
           make-test-multiworker-os: 'debian-11'
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: 'debian-11'
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''
@@ -110,9 +102,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''
@@ -217,11 +206,6 @@
           branch-refspec: ''
           make-test-os: 'ubuntu-20.04 ubuntu-22.04'
           make-test-multiworker-os: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
-          make-test-os: 'ubuntu-20.04 ubuntu-22.04'
-          make-test-multiworker-os: ''
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''
@@ -250,9 +234,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''
@@ -268,6 +249,11 @@
           os: 'ubuntu2204'
           executor-arch: 'aarch64'
           skip-vote: 'true'
+    exclude:
+      # 1n-alt testbed not supported
+      # in CSIT prior to 2402
+      - testbed-arch: 1n-alt
+        stream: '2310'
 # [end] VPP-CSIT-VERIFY-DEVICE PROJECT
 
 # VPP-CSIT-VERIFY-PERF PROJECT
@@ -339,9 +325,6 @@
       - 'master':
           branch: 'master'
           branch-refspec: ''
-      - '2306':
-          branch: 'stable/2306'
-          branch-refspec: ''
       - '2310':
           branch: 'stable/2310'
           branch-refspec: ''