diff options
Diffstat (limited to 'docker/scripts/lib_csit.sh')
| -rw-r--r-- | docker/scripts/lib_csit.sh | 130 |
1 files changed, 57 insertions, 73 deletions
diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh index 6c064d0a..2e877f80 100644 --- a/docker/scripts/lib_csit.sh +++ b/docker/scripts/lib_csit.sh @@ -1,7 +1,7 @@ # lib_csit.sh - Docker build script CSIT library. # For import only. -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2023 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -20,11 +20,9 @@ if [ -n "$(alias lib_csit_imported 2> /dev/null)" ] ; then fi alias lib_csit_imported=true -export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname $BASH_SOURCE)}" +export CIMAN_DOCKER_SCRIPTS="${CIMAN_DOCKER_SCRIPTS:-$(dirname ${BASH_SOURCE[0]})}" . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh" . "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh" -. "$CIMAN_DOCKER_SCRIPTS/lib_yum.sh" -. "$CIMAN_DOCKER_SCRIPTS/lib_dnf.sh" CSIT_SUPPORTED_EXECUTOR_CLASSES="builder csit_dut" csit_supported_executor_class() { @@ -36,10 +34,7 @@ csit_supported_executor_class() { csit_supported_os() { case "$1" in - # TODO: Remove ubuntu-18.04 once CSIT has completed transition - # to ubuntu-20.04 - ubuntu-18.04) return 0 ;; - ubuntu-20.04) return 0 ;; + ubuntu-22.04) return 0 ;; *) ;; esac return 1 @@ -55,12 +50,14 @@ csit_checkout_branch_for_vpp() { && source "$csit_bash_function_dir/branch.sh" CSIT_DIR="$csit_dir" checkout_csit_for_vpp "$vpp_branch" + # shellcheck disable=SC2034,SC2063 csit_branch="$(git branch | grep -e '^*' | mawk '{print $2}')" } csit_install_packages() { local branch="$1" - local branchname="$(echo $branch | sed -e 's,/,_,')" + local branchname + branchname="$(echo $branch | sed -e 's,/,_,')" local csit_dir="$DOCKER_CSIT_DIR" local csit_ansible_dir="$csit_dir/fdio.infra.ansible" if [ ! -d "$csit_ansible_dir" ] ; then @@ -70,48 +67,28 @@ csit_install_packages() { bld_log="${bld_log}-$branchname-csit_install_packages-bld.log" git clean -qfdx + python3 -m pip install pyyaml - # Install PyYAML required by dbld_csit_find_ansible_packages.py - # - # Note: Conditional install due to Bug 1696324 - - # Update to python3.6 breaks PyYAML dependencies - # Status: CLOSED CANTFIX - # https://bugzilla.redhat.com/show_bug.cgi?id=1696324 - if [ "$OS_NAME" = "centos-8" ] ; then - dnf_install_packages python3-pyyaml - else - python3 -m pip install pyyaml - fi - - local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul" + local exclude_roles="-e calibration -e kernel -e mellanox -e nomad -e consul -e aws -e vpp" [ "$OS_ARCH" = "aarch64" ] && exclude_roles="$exclude_roles -e iperf" # Not in double quotes to let bash remove newline characters - local yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)" + local yaml_files + yaml_files="$(grep -r packages_by $csit_ansible_dir | cut -d: -f1 | sort -u | grep -v $exclude_roles)" packages="$(dbld_csit_find_ansible_packages.py --$OS_ID --$OS_ARCH $yaml_files)" - packages="${packages/bionic /}" + packages="${packages/jammy /}" packages="${packages/focal /}" + packages="${packages/libmbedcrypto1/libmbedcrypto3}" + packages="${packages/libmbedtls10/libmbedtls12}" + packages="$(echo ${packages//python\-/python3\-} | tr ' ' '\n' | sort -u | xargs)" - # TODO: Fix Ubuntu-18.04 specific package names that fail on Ubuntu-20.04 - # (remove when CSIT is updated) - if [ "$OS_NAME" = "ubuntu-20.04" ] ; then - packages="${packages/libmbedcrypto1/libmbedcrypto3}" - packages="${packages/libmbedtls10/libmbedtls12}" - packages="$(echo ${packages//python\-/python3\-} | tr ' ' '\n' | sort -u | xargs)" - fi if [ -n "$packages" ] ; then case "$OS_NAME" in ubuntu*) - apt_install_packages $packages + apt_install_packages "$packages" 2>&1 | tee -a "$bld_log" ;; debian*) - apt_install_packages $packages - ;; - centos-7) - yum_install_packages $packages - ;; - centos-8) - dnf_install_packages $packages + apt_install_packages "$packages" 2>&1 | tee -a "$bld_log" ;; *) echo "Unsupported OS ($OS_ID): CSIT packages NOT INSTALLED!" @@ -120,9 +97,37 @@ csit_install_packages() { fi } +csit_install_hugo() { + local branch="$1" + CSIT_DIR="$DOCKER_CSIT_DIR" + + if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \ + && [ -f "$CSIT_DIR/requirements.txt" ]; then + + local branchname + # use bash variable substitution to replace '/' with '_' to convert from + # vpp to csit branch name nomenclature + branchname="${branch////_}" + local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function" + local bld_log="$DOCKER_BUILD_LOG_DIR" + bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_install_hugo-bld.log" + + description="Install CSIT hugo packages from $branch branch" + echo_log " Starting $description..." + git clean -qfdx + + source "$csit_bash_function_dir"/hugo.sh + go_install 2>&1 | tee -a "$bld_log" + hugo_install 2>&1 | tee -a "$bld_log" + + else + echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!" + return 1 + fi +} + csit_pip_cache() { local branch="$1" - local VENV_OPTS="" # ensure PS1 is defined (used by virtualenv activate script) PS1=${PS1:-"#"} CSIT_DIR="$DOCKER_CSIT_DIR" @@ -130,55 +135,34 @@ csit_pip_cache() { if [ -f "$CSIT_DIR/VPP_REPO_URL" ] \ && [ -f "$CSIT_DIR/requirements.txt" ]; then + local branchname + # use bash variable substitution to replace '/' with '_' to convert from + # vpp to csit branch name nomenclature + branchname="${branch////_}" local csit_bash_function_dir="$CSIT_DIR/resources/libraries/bash/function" - local branchname="$(echo $branch | sed -e 's,/,_,')" local bld_log="$DOCKER_BUILD_LOG_DIR" bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname-csit_pip_cache-bld.log" - local pip_cmd="python3 -m pip --disable-pip-version-check" export PYTHONPATH=$CSIT_DIR description="Install CSIT python packages from $branch branch" - echo_log " Starting $description..." - git clean -qfdx + echo_log " Starting $description..." 2>&1 | tee -a "$bld_log" + git clean -qfdx 2>&1 | tee -a "$bld_log" rm -rf "$PYTHONPATH/env" - # TODO: Update CSIT release branches to avoid build breakage - # Fixes https://github.com/pypa/pip/issues/8260 - $pip_cmd install pip==21.0.1 - # rls2009_lts-* branches missing cherry-pick of - # https://gerrit.fd.io/r/c/csit/+/31338 - sed -i 's/scipy==1.1.0/scipy==1.5.4/' "$PYTHONPATH/requirements.txt" - - # Virtualenv version is pinned in common.sh in newer csit branches. - # (note: xargs removes leading/trailing spaces) + # Activate / install CSIT python virtualenv ($CSIT_DIR/requirements.txt) local common_sh="$csit_bash_function_dir/common.sh" - install_virtualenv="$(grep 'virtualenv' $common_sh | grep pip | grep install | cut -d'|' -f1 | xargs)" - $install_virtualenv - virtualenv --no-download --python=$(which python3) "$CSIT_DIR/env" - source "$CSIT_DIR/env/bin/activate" - - if [ "$OS_ARCH" = "aarch64" ] ; then - local numpy_ver="$(grep numpy $PYTHONPATH/requirements.txt)" - [ -n "$numpy_ver" ] && $pip_cmd install $numpy_ver 2>&1 | \ - tee -a $bld_log - fi + # shellcheck disable=1090 + source "$common_sh" + activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/requirements.txt" 2>&1 | tee -a "$bld_log" - # Install csit python requirements - $pip_cmd install -r "$CSIT_DIR/requirements.txt" 2>&1 | \ - tee -a "$bld_log" # Install tox python requirements - $pip_cmd install -r "$CSIT_DIR/tox-requirements.txt" 2>&1 | \ + activate_virtualenv "${CSIT_DIR}" "${CSIT_DIR}/tox-requirements.txt" 2>&1 |\ tee -a "$bld_log" - # Run tox which installs pylint requirments - pushd $CSIT_DIR >& /dev/null - tox || true - popd >& /dev/null # Clean up virtualenv directories - deactivate git checkout -q -- . git clean -qfdx - echo_log " Completed $description!" + echo_log " Completed $description!" 2>&1 | tee -a "$bld_log" else echo_log "ERROR: Missing or invalid CSIT_DIR: '$CSIT_DIR'!" return 1 |