diff options
Diffstat (limited to 'resources/libraries/bash/function/common.sh')
| -rw-r--r-- | resources/libraries/bash/function/common.sh | 562 |
1 files changed, 562 insertions, 0 deletions
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh new file mode 100644 index 0000000000..b3a06d497b --- /dev/null +++ b/resources/libraries/bash/function/common.sh @@ -0,0 +1,562 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# This library defines functions used by multiple entry scripts. +# Keep functions ordered alphabetically, please. + +# TODO: Add a link to bash style guide. +# TODO: Consider putting every die into a {} block, +# the code might become more readable (but longer). + + +function activate_virtualenv () { + + set -exuo pipefail + + # Arguments: + # - ${1} - Non-empty path to existing directory for creating virtualenv in. + # Variables read: + # - CSIT_DIR - Path to existing root of local CSIT git repository. + # Variables set: + # - ENV_DIR - Path to the created virtualenv subdirectory. + # Variables exported: + # - PYTHONPATH - CSIT_DIR, as CSIT Python scripts usually need this. + # Functions called: + # - die - Print to stderr and exit. + + # TODO: Do we really need to have ENV_DIR available as a global variable? + + if [[ "${1-}" == "" ]]; then + die "Root location of virtualenv to create is not specified." + fi + ENV_DIR="${1}/env" + rm -rf "${ENV_DIR}" || die "Failed to clean previous virtualenv." + + pip install --upgrade virtualenv || { + die "Virtualenv package install failed." + } + virtualenv --system-site-packages "${ENV_DIR}" || { + die "Virtualenv creation failed." + } + set +u + source "${ENV_DIR}/bin/activate" || die "Virtualenv activation failed." + set -u + pip install -r "${CSIT_DIR}/requirements.txt" || { + die "CSIT requirements installation failed." + } + + # Most CSIT Python scripts assume PYTHONPATH is set and exported. + export PYTHONPATH="${CSIT_DIR}" || die "Export failed." +} + + +function check_download_dir () { + + set -exuo pipefail + + # Fail if there are no files visible in ${DOWNLOAD_DIR}. + # TODO: Do we need this as a function, if it is (almost) a one-liner? + # + # Variables read: + # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from. + # Directories read: + # - ${DOWNLOAD_DIR} - Has to be non-empty to proceed. + # Functions called: + # - die - Print to stderr and exit. + + if [[ ! "$(ls -A "${DOWNLOAD_DIR}")" ]]; then + die "No artifacts downloaded!" + fi +} + + +function common_dirs () { + + set -exuo pipefail + + # Variables set: + # - BASH_FUNCTION_DIR - Path to existing directory this file is located in. + # - CSIT_DIR - Path to existing root of local CSIT git repository. + # - TOPOLOGIES_DIR - Path to existing directory with available tpologies. + # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources". + # - TOOLS_DIR - Path to existing resources subdirectory "tools". + # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts". + # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive". + # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir". + # Functions called: + # - die - Print to stderr and exit. + + BASH_FUNCTION_DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")" || { + die "Some error during localizing this source directory." + } + # Current working directory could be in a different repo, e.g. VPP. + pushd "${BASH_FUNCTION_DIR}" || die "Pushd failed" + CSIT_DIR="$(readlink -e "$(git rev-parse --show-toplevel)")" || { + die "Readlink or git rev-parse failed." + } + popd || die "Popd failed." + TOPOLOGIES_DIR="$(readlink -e "${CSIT_DIR}/topologies/available")" || { + die "Readlink failed." + } + RESOURCES_DIR="$(readlink -e "${CSIT_DIR}/resources")" || { + die "Readlink failed." + } + TOOLS_DIR="$(readlink -e "${RESOURCES_DIR}/tools")" || { + die "Readlink failed." + } + PYTHON_SCRIPTS_DIR="$(readlink -e "${TOOLS_DIR}/scripts")" || { + die "Readlink failed." + } + + ARCHIVE_DIR="$(readlink -f "${CSIT_DIR}/archive")" || { + die "Readlink failed." + } + mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed." + DOWNLOAD_DIR="$(readlink -f "${CSIT_DIR}/download_dir")" || { + die "Readlink failed." + } + mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed." +} + + +function compose_pybot_arguments () { + + set -exuo pipefail + + # Variables read: + # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. + # - DUT - CSIT test/ subdirectory, set while processing tags. + # - TAGS - Array variable holding selected tag boolean expressions. + # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology. + # Variables set: + # - PYBOT_ARGS - String holding part of all arguments for pybot. + # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags. + + # No explicit check needed with "set -u". + PYBOT_ARGS=("--loglevel" "TRACE" "--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}") + PYBOT_ARGS+=("--suite" "tests.${DUT}.perf") + + EXPANDED_TAGS=() + for tag in "${TAGS[@]}"; do + if [[ ${tag} == "!"* ]]; then + EXPANDED_TAGS+=("--exclude" "${tag#$"!"}") + else + EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}") + fi + done +} + + +function copy_archives () { + + set -exuo pipefail + + # Variables read: + # - WORKSPACE - Jenkins workspace, copy only if the value is not empty. + # Can be unset, then it speeds up manual testing. + # - ARCHIVE_DIR - Path to directory with content to be copied. + # Directories updated: + # - ${WORKSPACE}/archives/ - Created if does not exist. + # Content of ${ARCHIVE_DIR}/ is copied here. + # Functions called: + # - die - Print to stderr and exit. + + # We will create additional archive if workspace variable is set. + # This way if script is running in jenkins all will be + # automatically archived to logs.fd.io. + if [[ -n "${WORKSPACE-}" ]]; then + mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed." + cp -r "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed." + fi +} + + +function die () { + # Print the message to standard error end exit with error code specified + # by the second argument. + # + # Hardcoded values: + # - The default error message. + # Arguments: + # - ${1} - The whole error message, be sure to quote. Optional + # - ${2} - the code to exit with, default: 1. + + set -x + set +eu + warn "${1:-Unspecified run-time error occurred!}" + exit "${2:-1}" +} + + +function die_on_pybot_error () { + + set -exuo pipefail + + # Source this fragment if you want to abort on any failed test case. + # + # Variables read: + # - PYBOT_EXIT_STATUS - Set by a pybot running fragment. + # Functions called: + # - die - Print to stderr and exit. + + if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then + die "${PYBOT_EXIT_STATUS}" "Test failures are present!" + fi +} + + +function get_test_code () { + + set -exuo pipefail + + # Arguments: + # - ${1} - Optional, argument of entry script (or empty as unset). + # Test code value to override job name from environment. + # Variables read: + # - JOB_NAME - String affecting test selection, default if not argument. + # Variables set: + # - TEST_CODE - The test selection string from environment or argument. + # - NODENESS - Node multiplicity of desired testbed. + # - FLAVOR - Node flavor string, usually describing the processor. + + TEST_CODE="${1-}" || die "Reading optional argument failed, somehow." + if [[ -z "${TEST_CODE}" ]]; then + TEST_CODE="${JOB_NAME-}" || die "Reading job name failed, somehow." + fi + + case "${TEST_CODE}" in + *"2n-skx"*) + NODENESS="2n" + FLAVOR="skx" + ;; + *"3n-skx"*) + NODENESS="3n" + FLAVOR="skx" + ;; + *) + # Fallback to 3-node Haswell by default (backward compatibility) + NODENESS="3n" + FLAVOR="hsw" + ;; + esac +} + + +function get_test_tag_string () { + + set -exuo pipefail + + # Variables read: + # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset. + # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type. + # Variables set: + # - TEST_TAG_STRING - The string following "perftest" in gerrit comment, + # or empty. + + # TODO: ci-management scripts no longer need to perform this. + + trigger="" + if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then + # On parsing error, ${trigger} stays empty. + trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \ + | grep -oE '(perftest$|perftest[[:space:]].+$)')" || true + fi + # Set test tags as string. + TEST_TAG_STRING="${trigger#$"perftest"}" +} + + +function reserve_testbed () { + + set -exuo pipefail + + # Reserve physical testbed, perform cleanup, register trap to unreserve. + # + # Variables read: + # - TOPOLOGIES - Array of paths to topology yaml to attempt reservation on. + # - PYTHON_SCRIPTS_DIR - Path to directory holding the reservation script. + # Variables set: + # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. + # Functions called: + # - die - Print to stderr and exit. + # Traps registered: + # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}. + + while true; do + for topo in "${TOPOLOGIES[@]}"; do + set +e + python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -t "${topo}" + result="$?" + set -e + if [[ "${result}" == "0" ]]; then + WORKING_TOPOLOGY="${topo}" + echo "Reserved: ${WORKING_TOPOLOGY}" + python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${topo}" || { + die "Testbed cleanup failed." + } + trap "untrap_and_unreserve_testbed" EXIT || { + message="TRAP ATTEMPT AND UNRESERVE FAILED, FIX MANUALLY." + untrap_and_unreserve_testbed "${message}" || { + die "Teardown should have died, not failed." + } + die "Trap attempt failed, unreserve succeeded. Aborting." + } + break + fi + done + + if [[ -n "${WORKING_TOPOLOGY-}" ]]; then + # Exit the infinite while loop if we made a reservation. + break + fi + + # Wait ~3minutes before next try. + sleep_time="$[ ( $RANDOM % 20 ) + 180 ]s" || { + die "Sleep time calculation failed." + } + echo "Sleeping ${sleep_time}" + sleep "${sleep_time}" || die "Sleep failed." + done +} + + +function run_pybot () { + + set -exuo pipefail + + # Currently, VPP-1361 causes occasional test failures. + # If real result is more important than time, we can retry few times. + # TODO: We should be retrying on test case level instead. + + # Arguments: + # - ${1} - Optional number of pybot invocations to try to avoid failures. + # Default: 1. + # Variables read: + # - CSIT_DIR - Path to existing root of local CSIT git repository. + # - ARCHIVE_DIR - Path to store robot result files in. + # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh + # Variables set: + # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation. + # Functions called: + # - die - Print to stderr and exit. + + # Set ${tries} as an integer variable, to fail on non-numeric input. + local -i "tries" || die "Setting type of variable failed." + tries="${1:-1}" || die "Argument evaluation failed." + all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}") + all_options+=("${EXPANDED_TAGS[@]}") + + while true; do + if [[ "${tries}" -le 0 ]]; then + break + else + tries="$((${tries} - 1))" + fi + pushd "${CSIT_DIR}" || die "Change directory operation failed." + set +e + # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}". + pybot "${all_options[@]}" "${CSIT_DIR}/tests/" + PYBOT_EXIT_STATUS="$?" + set -e + popd || die "Change directory operation failed." + if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then + break + fi + done +} + + +function select_tags () { + + set -exuo pipefail + + # Variables read: + # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. + # - TEST_CODE - String affecting test selection, usually jenkins job name. + # - TEST_TAG_STRING - String selecting tags, from gerrit comment. + # Can be unset. + # - TOPOLOGIES_DIR - Path to existing directory with available tpologies. + # Variables set: + # - TAGS - Array of processed tag boolean expressions. + + # TODO: Empty exclude_nics (with failing grep) is expected, + # but others possible errors coule be checked explicitly. + # NIC SELECTION + # All topologies NICs + available=$(grep -hoPR "model: \K.*" "${TOPOLOGIES_DIR}"/* | sort -u) + # Selected topology NICs + reserved=$(grep -hoPR "model: \K.*" "${WORKING_TOPOLOGY}" | sort -u) + # All topologies NICs - Selected topology NICs + exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}"))) + + case "${TEST_CODE}" in + # Select specific performance tests based on jenkins job type variable. + *"ndrpdr-weekly"* ) + test_tag_array=("ndrpdrAND64bAND1c" + "ndrpdrAND78bAND1c") + ;; + *"mrr-daily"* | *"mrr-weekly"* ) + test_tag_array=("mrrAND64bAND1c" + "mrrAND64bAND2c" + "mrrAND64bAND4c" + "mrrAND78bAND1c" + "mrrAND78bAND2c" + "mrrAND78bAND4c" + "mrrANDimixAND1cANDvhost" + "mrrANDimixAND2cANDvhost" + "mrrANDimixAND4cANDvhost" + "mrrANDimixAND1cANDmemif" + "mrrANDimixAND2cANDmemif" + "mrrANDimixAND4cANDmemif") + ;; + * ) + if [[ -z "${TEST_TAG_STRING-}" ]]; then + # If nothing is specified, we will run pre-selected tests by + # following tags. Items of array will be concatenated by OR + # in Robot Framework. + test_tag_array=("mrrANDnic_intel-x710AND1cAND64bANDip4base" + "mrrANDnic_intel-x710AND1cAND78bANDip6base" + "mrrANDnic_intel-x710AND1cAND64bANDl2bdbase" + "mrrANDnic_intel-x710AND1cAND64bANDl2xcbase") + else + # If trigger contains tags, split them into array. + test_tag_array=(${TEST_TAG_STRING//:/ }) + fi + ;; + esac + + # We will add excluded NICs. + test_tag_array+=("${exclude_nics[@]/#/!NIC_}") + + TAGS=() + + # We will prefix with perftest to prevent running other tests + # (e.g. Functional). + prefix="perftestAND" + if [[ "${TEST_CODE}" == "vpp-"* ]]; then + # Automatic prefixing for VPP jobs to limit the NIC used and + # traffic evaluation to MRR. + prefix="${prefix}mrrANDnic_intel-x710AND" + fi + for tag in "${test_tag_array[@]}"; do + if [[ ${tag} == "!"* ]]; then + # Exclude tags are not prefixed. + TAGS+=("${tag}") + else + TAGS+=("${prefix}${tag}") + fi + done +} + + +function select_topology () { + + set -exuo pipefail + + # Variables read: + # - NODENESS - Node multiplicity of testbed, either "2n" or "3n". + # - FLAVOR - Node flavor string, currently either "hsw" or "skx". + # - CSIT_DIR - Path to existing root of local CSIT git repository. + # - TOPOLOGIES_DIR - Path to existing directory with available tpologies. + # Variables set: + # - TOPOLOGIES - Array of paths to suitable topology yaml files. + # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology. + # Functions called: + # - die - Print to stderr and exit. + + case_text="${NODENESS}_${FLAVOR}" + case "${case_text}" in + "3n_hsw") + TOPOLOGIES=( + "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml" + "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml" + "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml" + ) + TOPOLOGIES_TAGS="3_node_*_link_topo" + ;; + "2n_skx") + TOPOLOGIES=( + "${TOPOLOGIES_DIR}/lf_2n_skx_testbed21.yaml" + #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed22.yaml" + #"${TOPOLOGIES_DIR}/lf_2n_skx_testbed23.yaml" + "${TOPOLOGIES_DIR}/lf_2n_skx_testbed24.yaml" + ) + TOPOLOGIES_TAGS="2_node_*_link_topo" + ;; + "3n_skx") + TOPOLOGIES=( + "${TOPOLOGIES_DIR}/lf_3n_skx_testbed31.yaml" + "${TOPOLOGIES_DIR}/lf_3n_skx_testbed32.yaml" + ) + TOPOLOGIES_TAGS="3_node_*_link_topo" + ;; + *) + # No falling back to 3n_hsw default, that should have been done + # by the function which has set NODENESS and FLAVOR. + die "Unknown specification: ${case_text}" + esac + + if [[ -z "${TOPOLOGIES-}" ]]; then + die "No applicable topology found!" + fi +} + + +function untrap_and_unreserve_testbed () { + # Use this as a trap function to ensure testbed does not remain reserved. + # Perhaps call directly before script exit, to free testbed for other jobs. + # This function is smart enough to avoid multiple unreservations (so safe). + # Topo cleanup is executed (call it best practice), ignoring failures. + # + # Hardcoded values: + # - default message to die with if testbed might remain reserved. + # Arguments: + # - ${1} - Message to die with if unreservation fails. Default hardcoded. + # Variables read (by inner function): + # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. + # - PYTHON_SCRIPTS_DIR - Path to directory holding Python scripts. + # Variables written: + # - WORKING_TOPOLOGY - Set to empty string on successful unreservation. + # Trap unregistered: + # - EXIT - Failure to untrap is reported, but ignored otherwise. + # Functions called: + # - die - Print to stderr and exit. + + set -xo pipefail + set +eu # We do not want to exit early in a "teardown" function. + trap - EXIT || echo "Trap deactivation failed, continuing anyway." + wt="${WORKING_TOPOLOGY}" # Just to avoid too long lines. + if [[ -z "${wt-}" ]]; then + set -eu + echo "Testbed looks unreserved already. Trap removal failed before?" + else + python "${PYTHON_SCRIPTS_DIR}/topo_cleanup.py" -t "${wt}" || true + python "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || { + die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2 + } + WORKING_TOPOLOGY="" + set -eu + fi +} + + +function warn () { + # Print the message to standard error. + # + # Arguments: + # - ${@} - The text of the message. + + echo "$@" >&2 +} |