diff options
Diffstat (limited to 'resources/libraries/bash/function/common.sh')
| -rw-r--r-- | resources/libraries/bash/function/common.sh | 165 |
1 files changed, 108 insertions, 57 deletions
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 8e0ee97be1..15b7921bb3 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -145,7 +145,7 @@ function archive_tests () { set -exuo pipefail - tar c "${GENERATED_DIR}/tests" | xz -9e > "${ARCHIVE_DIR}/tests.tar.xz" || { + tar c "${GENERATED_DIR}/tests" | xz -3 > "${ARCHIVE_DIR}/tests.tar.xz" || { die "Error creating archive of generated tests." } } @@ -198,7 +198,8 @@ function common_dirs () { # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources". # - TOOLS_DIR - Path to existing resources subdirectory "tools". # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts". - # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive". + # - ARCHIVE_DIR - Path to created CSIT subdirectory "archives". + # The name is chosen to match what ci-management expects. # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir". # - GENERATED_DIR - Path to created CSIT subdirectory "generated". # Directories created if not present: @@ -240,7 +241,7 @@ function common_dirs () { die "Readlink failed." } - ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archive") || { + ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archives") || { die "Readlink failed." } mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed." @@ -263,6 +264,7 @@ function compose_pybot_arguments () { # - TAGS - Array variable holding selected tag boolean expressions. # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology. # - TEST_CODE - The test selection string from environment or argument. + # - SELECTION_MODE - Selection criteria [test, suite, include, exclude]. # Variables set: # - PYBOT_ARGS - String holding part of all arguments for pybot. # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags. @@ -289,33 +291,16 @@ function compose_pybot_arguments () { if [[ ${tag} == "!"* ]]; then EXPANDED_TAGS+=("--exclude" "${tag#$"!"}") else - EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}") + if [[ ${SELECTION_MODE} == "--test" ]]; then + EXPANDED_TAGS+=("--test" "${tag}") + else + EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}") + fi fi done -} - -function copy_archives () { - - # Create additional archive if workspace variable is set. - # This way if script is running in jenkins all will be - # automatically archived to logs.fd.io. - # - # Variables read: - # - WORKSPACE - Jenkins workspace, copy only if the value is not empty. - # Can be unset, then it speeds up manual testing. - # - ARCHIVE_DIR - Path to directory with content to be copied. - # Directories updated: - # - ${WORKSPACE}/archives/ - Created if does not exist. - # Content of ${ARCHIVE_DIR}/ is copied here. - # Functions called: - # - die - Print to stderr and exit. - - set -exuo pipefail - - if [[ -n "${WORKSPACE-}" ]]; then - mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed." - cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed." + if [[ ${SELECTION_MODE} == "--test" ]]; then + EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}") fi } @@ -460,6 +445,10 @@ function get_test_code () { NODENESS="2n" FLAVOR="skx" ;; + *"2n-zn2"*) + NODENESS="2n" + FLAVOR="zn2" + ;; *"3n-skx"*) NODENESS="3n" FLAVOR="skx" @@ -515,14 +504,20 @@ function get_test_tag_string () { die "Unknown specification: ${TEST_CODE}" esac # Ignore lines not containing the trigger word. - comment=$(fgrep "${trigger}" <<< "${GERRIT_EVENT_COMMENT_TEXT}") || true + comment=$(fgrep "${trigger}" <<< "${GERRIT_EVENT_COMMENT_TEXT}" || true) # The vpp-csit triggers trail stuff we are not interested in. # Removing them and trigger word: https://unix.stackexchange.com/a/13472 # (except relying on \s whitespace, \S non-whitespace and . both). # The last string is concatenated, only the middle part is expanded. cmd=("grep" "-oP" '\S*'"${trigger}"'\S*\s\K.+$') || die "Unset trigger?" # On parsing error, TEST_TAG_STRING probably stays empty. - TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}") || true + TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}" || true) + if [[ -z "${TEST_TAG_STRING-}" ]]; then + # Probably we got a base64 encoded comment. + comment=$(base64 --decode <<< "${GERRIT_EVENT_COMMENT_TEXT}" || true) + comment=$(fgrep "${trigger}" <<< "${comment}" || true) + TEST_TAG_STRING=$("${cmd[@]}" <<< "${comment}" || true) + fi if [[ -n "${TEST_TAG_STRING-}" ]]; then test_tag_array=(${TEST_TAG_STRING}) if [[ "${test_tag_array[0]}" == "icl" ]]; then @@ -558,6 +553,47 @@ function installed () { } +function move_archives () { + + # Move archive directory to top of workspace, if not already there. + # + # ARCHIVE_DIR is positioned relative to CSIT_DIR, + # but in some jobs CSIT_DIR is not same as WORKSPACE + # (e.g. under VPP_DIR). To simplify ci-management settings, + # we want to move the data to the top. We do not want simple copy, + # as ci-management is eager with recursive search. + # + # As some scripts may call this function multiple times, + # the actual implementation use copying and deletion, + # so the workspace gets "union" of contents (except overwrites on conflict). + # The consequence is empty ARCHIVE_DIR remaining after this call. + # + # As the source directory is emptied, + # the check for dirs being different is essential. + # + # Variables read: + # - WORKSPACE - Jenkins workspace, move only if the value is not empty. + # Can be unset, then it speeds up manual testing. + # - ARCHIVE_DIR - Path to directory with content to be moved. + # Directories updated: + # - ${WORKSPACE}/archives/ - Created if does not exist. + # Content of ${ARCHIVE_DIR}/ is moved. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + if [[ -n "${WORKSPACE-}" ]]; then + target=$(readlink -f "${WORKSPACE}/archives") + if [[ "${target}" != "${ARCHIVE_DIR}" ]]; then + mkdir -p "${target}" || die "Archives dir create failed." + cp -rf "${ARCHIVE_DIR}"/* "${target}" || die "Copy failed." + rm -rf "${ARCHIVE_DIR}"/* || die "Delete failed." + fi + fi +} + + function reserve_and_cleanup_testbed () { # Reserve physical testbed, perform cleanup, register trap to unreserve. @@ -729,6 +765,7 @@ function select_tags () { # - BASH_FUNCTION_DIR - Directory with input files to process. # Variables set: # - TAGS - Array of processed tag boolean expressions. + # - SELECTION_MODE - Selection criteria [test, suite, include, exclude]. set -exuo pipefail @@ -756,7 +793,7 @@ function select_tags () { *"3n-tsh"*) default_nic="nic_intel-x520-da2" ;; - *"3n-skx"* | *"2n-skx"* | *"2n-clx"*) + *"3n-skx"* | *"2n-skx"* | *"2n-clx"* | *"2n-zn2"*) default_nic="nic_intel-xxv710" ;; *"3n-hsw"* | *"mrr-daily-master") @@ -768,48 +805,59 @@ function select_tags () { esac sed_nic_sub_cmd="sed s/\${default_nic}/${default_nic}/" - sed_nics_sub_cmd="sed -e s/ANDxxv710/ANDnic_intel-xxv710/" - sed_nics_sub_cmd+=" | sed -e s/ANDx710/ANDnic_intel-x710/" - sed_nics_sub_cmd+=" | sed -e s/ANDxl710/ANDnic_intel-xl710/" - sed_nics_sub_cmd+=" | sed -e s/ANDx520-da2/ANDnic_intel-x520-da2/" - sed_nics_sub_cmd+=" | sed -e s/ANDx553/ANDnic_intel-x553/" - sed_nics_sub_cmd+=" | sed -e s/ANDcx556a/ANDnic_mellanox-cx556a/" - sed_nics_sub_cmd+=" | sed -e s/ANDvic1227/ANDnic_cisco-vic-1227/" - sed_nics_sub_cmd+=" | sed -e s/ANDvic1385/ANDnic_cisco-vic-1385/" + awk_nics_sub_cmd="" + awk_nics_sub_cmd+='gsub("xxv710","25ge2p1xxv710");' + awk_nics_sub_cmd+='gsub("x710","10ge2p1x710");' + awk_nics_sub_cmd+='gsub("xl710","40ge2p1xl710");' + awk_nics_sub_cmd+='gsub("x520-da2","10ge2p1x520");' + awk_nics_sub_cmd+='gsub("x553","10ge2p1x553");' + awk_nics_sub_cmd+='gsub("cx556a","10ge2p1cx556a");' + awk_nics_sub_cmd+='gsub("vic1227","10ge2p1vic1227");' + awk_nics_sub_cmd+='gsub("vic1385","10ge2p1vic1385");' + awk_nics_sub_cmd+='if ($9 =="drv_avf") drv="avf-";' + awk_nics_sub_cmd+='else if ($9 =="drv_rdma_core") drv ="rdma-";' + awk_nics_sub_cmd+='else drv="";' + awk_nics_sub_cmd+='print "*"$7"-" drv $11"-"$5"."$3"-"$1"-" drv $11"-"$5' + # Tag file directory shorthand. tfd="${JOB_SPECS_DIR}" case "${TEST_CODE}" in # Select specific performance tests based on jenkins job type variable. *"ndrpdr-weekly"* ) - readarray -t test_tag_array <<< $(sed 's/ //g' \ + readarray -t test_tag_array <<< $(grep -v "#" \ ${tfd}/mlr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md | - eval ${sed_nics_sub_cmd} || echo "perftest") || die + awk {"$awk_nics_sub_cmd"} || echo "perftest") || die + SELECTION_MODE="--test" ;; *"mrr-daily"* ) - readarray -t test_tag_array <<< $(sed 's/ //g' \ + readarray -t test_tag_array <<< $(grep -v "#" \ ${tfd}/mrr_daily/${DUT}-${NODENESS}-${FLAVOR}.md | - eval ${sed_nics_sub_cmd} || echo "perftest") || die + awk {"$awk_nics_sub_cmd"} || echo "perftest") || die + SELECTION_MODE="--test" ;; *"mrr-weekly"* ) - readarray -t test_tag_array <<< $(sed 's/ //g' \ + readarray -t test_tag_array <<< $(grep -v "#" \ ${tfd}/mrr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md | - eval ${sed_nics_sub_cmd} || echo "perftest") || die + awk {"$awk_nics_sub_cmd"} || echo "perftest") || die + SELECTION_MODE="--test" ;; *"report-iterative"* ) test_sets=(${TEST_TAG_STRING//:/ }) # Run only one test set per run report_file=${test_sets[0]}.md - readarray -t test_tag_array <<< $(sed 's/ //g' \ + readarray -t test_tag_array <<< $(grep -v "#" \ ${tfd}/report_iterative/${NODENESS}-${FLAVOR}/${report_file} | - eval ${sed_nics_sub_cmd} || echo "perftest") || die + awk {"$awk_nics_sub_cmd"} || echo "perftest") || die + SELECTION_MODE="--test" ;; *"report-coverage"* ) test_sets=(${TEST_TAG_STRING//:/ }) # Run only one test set per run report_file=${test_sets[0]}.md - readarray -t test_tag_array <<< $(sed 's/ //g' \ + readarray -t test_tag_array <<< $(grep -v "#" \ ${tfd}/report_coverage/${NODENESS}-${FLAVOR}/${report_file} | - eval ${sed_nics_sub_cmd} || echo "perftest") || die + awk {"$awk_nics_sub_cmd"} || echo "perftest") || die + SELECTION_MODE="--test" ;; * ) if [[ -z "${TEST_TAG_STRING-}" ]]; then @@ -824,6 +872,7 @@ function select_tags () { # If trigger contains tags, split them into array. test_tag_array=(${TEST_TAG_STRING//:/ }) fi + SELECTION_MODE="--include" ;; esac @@ -834,7 +883,7 @@ function select_tags () { # TODO: Add missing reasons here (if general) or where used (if specific). case "${TEST_CODE}" in *"2n-skx"*) - test_tag_array+=("!ipsechw") + test_tag_array+=("!ipsec") ;; *"3n-skx"*) test_tag_array+=("!ipsechw") @@ -842,7 +891,10 @@ function select_tags () { test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710") ;; *"2n-clx"*) - test_tag_array+=("!ipsechw") + test_tag_array+=("!ipsec") + ;; + *"2n-zn2"*) + test_tag_array+=("!ipsec") ;; *"2n-dnv"*) test_tag_array+=("!ipsechw") @@ -865,10 +917,6 @@ function select_tags () { test_tag_array+=("!ipsechw") ;; *"3n-hsw"*) - # TODO: Introduce NOIOMMU version of AVF tests. - # TODO: Make (both) AVF tests work on Haswell, - # or document why (some of) it is not possible. - # https://github.com/FDio/vpp/blob/master/src/plugins/avf/README.md test_tag_array+=("!drv_avf") # All cards have access to QAT. But only one card (xl710) # resides in same NUMA as QAT. Other cards must go over QPI @@ -886,10 +934,8 @@ function select_tags () { test_tag_array+=("${exclude_nics[@]/#/!NIC_}") TAGS=() + prefix="" - # We will prefix with perftest to prevent running other tests - # (e.g. Functional). - prefix="perftestAND" set +x if [[ "${TEST_CODE}" == "vpp-"* ]]; then # Automatic prefixing for VPP jobs to limit the NIC used and @@ -958,6 +1004,10 @@ function select_topology () { TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_skx*.yaml ) TOPOLOGIES_TAGS="2_node_*_link_topo" ;; + "2n_zn2") + TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_zn2*.yaml ) + TOPOLOGIES_TAGS="2_node_*_link_topo" + ;; "3n_skx") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_skx*.yaml ) TOPOLOGIES_TAGS="3_node_*_link_topo" @@ -1017,6 +1067,7 @@ function select_vpp_device_tags () { # If trigger contains tags, split them into array. test_tag_array=(${TEST_TAG_STRING//:/ }) fi + SELECTION_MODE="--include" ;; esac |