diff options
| -rw-r--r-- | resources/libraries/bash/entry/per_patch_device.sh | 57 | ||||
| -rw-r--r-- | resources/libraries/bash/entry/per_patch_perf.sh | 11 | ||||
| -rw-r--r-- | resources/libraries/bash/function/per_patch.sh | 111 |
3 files changed, 108 insertions, 71 deletions
diff --git a/resources/libraries/bash/entry/per_patch_device.sh b/resources/libraries/bash/entry/per_patch_device.sh new file mode 100644 index 0000000000..3f52bcbc0b --- /dev/null +++ b/resources/libraries/bash/entry/per_patch_device.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# This entry script does not change CSIT branch, +# use "with_oper_for_vpp.sh" wrapper for that. +# +# Assumptions: +# + There is a directory holding VPP repo with patch under test checked out. +# + It contains csit subdirectory with CSIT code to use (this script is there). +# + Everything needed to build VPP is already installed locally. +# Consequences: +# + The following directories (relative to VPP repo) are (re)created: +# ++ csit_current, build_current, archive, csit/archive, csit_download_dir. + +# TODO: Implement some kind of VPP build caching. + +# "set -eu" handles failures from the following two lines. +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/per_patch.sh" || die "Source failed." +common_dirs || die +set_perpatch_vpp_dir || die +build_vpp_ubuntu_amd64 "CURRENT" || die +set_aside_commit_build_artifacts || die +initialize_csit_dirs || die +get_test_code "${1-}" || die +get_test_tag_string || die +set_perpatch_dut || die +select_topology || die +select_build "build_current" || die +check_download_dir || die +activate_virtualenv "${VPP_DIR}" || die +activate_docker_topology || die +select_vpp_device_tags || die +compose_pybot_arguments || die +run_pybot || die +copy_archives || die +archive_parse_test_results "csit_current" || die +die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh index 73d4f69500..c176bbfd5f 100644 --- a/resources/libraries/bash/entry/per_patch_perf.sh +++ b/resources/libraries/bash/entry/per_patch_perf.sh @@ -41,10 +41,10 @@ source "${BASH_FUNCTION_DIR}/common.sh" || { source "${BASH_FUNCTION_DIR}/per_patch.sh" || die "Source failed." common_dirs || die set_perpatch_vpp_dir || die -build_vpp_ubuntu_amd64 "NEW" || die -prepare_build_parent || die +build_vpp_ubuntu_amd64 "CURRENT" || die +set_aside_commit_build_artifacts || die build_vpp_ubuntu_amd64 "PARENT" || die -prepare_test || die +set_aside_parent_build_artifacts || die ## Replace previous 4 lines with this to speed up testing. #download_builds "REPLACE_WITH_URL" || die initialize_csit_dirs || die @@ -67,13 +67,12 @@ for ((iter=0; iter<iterations; iter++)); do archive_parse_test_results "csit_parent/${iter}" || die die_on_pybot_error || die cleanup_topo - select_build "build_new" || die + select_build "build_current" || die check_download_dir || die run_pybot || die copy_archives || die - archive_parse_test_results "csit_new/${iter}" || die + archive_parse_test_results "csit_current/${iter}" || die die_on_pybot_error || die done untrap_and_unreserve_testbed || die compare_test_results # The error code becomes this script's error code. -# TODO: After merging, make sure archiving works as expected. diff --git a/resources/libraries/bash/function/per_patch.sh b/resources/libraries/bash/function/per_patch.sh index 854392eca6..83e8427824 100644 --- a/resources/libraries/bash/function/per_patch.sh +++ b/resources/libraries/bash/function/per_patch.sh @@ -54,45 +54,21 @@ function build_vpp_ubuntu_amd64 () { set -exuo pipefail - # TODO: Make sure whether this works on other distros/archs too. - + # This function is using Vagrant script to build VPP with all dependencies + # that is ARCH/OS aware. VPP repo is SSOT for building mechanics and CSIT + # is consuming artifacts. This way if VPP will introduce change in building + # mechanics they will not be blocked by CSIT repo. # Arguments: # - ${1} - String identifier for echo, can be unset. # Variables read: # - VPP_DIR - Path to existing directory, parent to accessed directories. # Directories updated: # - ${VPP_DIR} - Whole subtree, many files (re)created by the build process. - # - ${VPP_DIR}/build-root - Final build artifacts for CSIT end up here. - # - ${VPP_DIR}/dpdk - The dpdk artifact is built, but moved to build-root/. # Functions called: # - die - Print to stderr and exit, defined in common.sh cd "${VPP_DIR}" || die "Change directory command failed." echo 'Building using "make build-root/vagrant/build.sh"' - # TODO: Do we want to support "${DRYRUN}" == "True"? - make UNATTENDED=yes install-dep || die "Make install-dep failed." - # The per_patch script calls this function twice, first for the new commit, - # then for its parent commit. On Jenkins, no dpdk is installed at first, - # locally it might have been installed. New dpdk is installed second call. - # If make detects installed vpp-ext-deps with matching version, - # it skips building vpp-ext-deps entirely. - # On the other hand, if parent uses different dpdk version, - # the new vpp-ext-deps is built, but the old one is not removed - # from the build directory if present. (Further functions move both, - # and during test dpkg would decide randomly which version gets installed.) - # As per_patch is too dumb (yet) to detect any of that, - # the only safe solution is to clean build directory and force rebuild. - # TODO: Make this function smarter and skip DPDK rebuilds if possible. - cmd=("dpkg-query" "--showformat='$${Version}'" "--show" "vpp-ext-deps") - installed_deb_ver="$(sudo "${cmd[@]}" || true)" - if [[ -n "${installed_deb_ver}" ]]; then - sudo dpkg --purge "vpp-ext-deps" || { - die "DPDK package uninstalation failed." - } - fi - make UNATTENDED=yes install-ext-deps || { - die "Make install-ext-deps failed." - } build-root/vagrant/"build.sh" || die "Vagrant VPP build script failed." echo "*******************************************************************" echo "* VPP ${1-} BUILD SUCCESSFULLY COMPLETED" || { @@ -142,19 +118,20 @@ function download_builds () { # - VPP_DIR - Path to WORKSPACE, parent of created directories. # Directories created: # - archive - Ends up empty, not to be confused with ${ARCHIVE_DIR}. - # - build_new - Holding built artifacts of the patch under test (PUT). + # - build_current - Holding built artifacts of the patch under test (PUT). # - built_parent - Holding built artifacts of parent of PUT. # Functions called: # - die - Print to stderr and exit, defined in common.sh cd "${VPP_DIR}" || die "Change directory operation failed." - rm -rf "build-root" "build_parent" "build_new" "archive" "csit_new" || { + dirs=("build-root" "build_parent" "build_current" "archive" "csit_current") + rm -rf ${dirs[@]} || { die "Directory removal failed." } wget -N --progress=dot:giga "${1}" || die "Wget download failed." unzip "archive.zip" || die "Archive extraction failed." mv "archive/build_parent" ./ || die "Move operation failed." - mv "archive/build_new" ./ || die "Move operation failed." + mv "archive/build_current" ./ || die "Move operation failed." } @@ -167,16 +144,16 @@ function initialize_csit_dirs () { # Variables read: # - VPP_DIR - Path to WORKSPACE, parent of created directories. # Directories created: - # - csit_new - Holding test results of the patch under test (PUT). + # - csit_current - Holding test results of the patch under test (PUT). # - csit_parent - Holding test results of parent of PUT. # Functions called: # - die - Print to stderr and exit, defined in common.sh cd "${VPP_DIR}" || die "Change directory operation failed." - rm -rf "csit_new" "csit_parent" || { + rm -rf "csit_current" "csit_parent" || { die "Directory deletion failed." } - mkdir -p "csit_new" "csit_parent" || { + mkdir -p "csit_current" "csit_parent" || { die "Directory creation failed." } } @@ -212,24 +189,51 @@ function parse_bmrr_results () { } -function prepare_build_parent () { +function select_build () { set -exuo pipefail + # Arguments: + # - ${1} - Path to directory to copy VPP artifacts from. Required. + # Variables read: + # - DOWNLOAD_DIR - Path to directory where Robot takes builds to test from. + # - VPP_DIR - Path to existing directory, root for relative paths. + # Directories read: + # - ${1} - Existing directory with built new VPP artifacts (and DPDK). + # Directories updated: + # - ${DOWNLOAD_DIR} - Old content removed, .deb files from ${1} copied here. + # Functions called: + # - die - Print to stderr and exit, defined in common.sh + + cd "${VPP_DIR}" || die "Change directory operation failed." + source_dir="$(readlink -e "$1")" + rm -rf "${DOWNLOAD_DIR}"/* || die "Cleanup of download dir failed." + cp "${source_dir}"/*".deb" "${DOWNLOAD_DIR}" || die "Copy operation failed." + # TODO: Is there a nice way to create symlinks, + # so that if job fails on robot, results can be archived? +} + + +function set_aside_commit_build_artifacts () { + + set -exuo pipefail + + # Function is copying VPP built artifacts from actual checkout commit for + # further use and clean git. # Variables read: # - VPP_DIR - Path to existing directory, parent to accessed directories. # Directories read: # - build-root - Existing directory with built VPP artifacts (also DPDK). # Directories updated: # - ${VPP_DIR} - A local git repository, parent commit gets checked out. - # - build_new - Old contents removed, content of build-root copied here. + # - build_current - Old contents removed, content of build-root copied here. # Functions called: # - die - Print to stderr and exit, defined in common.sh cd "${VPP_DIR}" || die "Change directory operation failed." - rm -rf "build_new" || die "Remove operation failed." - mkdir -p "build_new" || die "Directory creation failed." - mv "build-root"/*".deb" "build_new"/ || die "Move operation failed." + rm -rf "build_current" || die "Remove operation failed." + mkdir -p "build_current" || die "Directory creation failed." + mv "build-root"/*".deb" "build_current"/ || die "Move operation failed." # The previous build could have left some incompatible leftovers, # e.g. DPDK artifacts of different version (in build/external). # Also, there usually is a copy of dpdk artifact in build-root. @@ -241,10 +245,12 @@ function prepare_build_parent () { } -function prepare_test () { +function set_aside_parent_build_artifacts () { set -exuo pipefail + # Function is copying VPP built artifacts from parent checkout commit for + # further use. Checkout to parent is not part of this function. # Variables read: # - VPP_DIR - Path to existing directory, parent of accessed directories. # Directories read: @@ -261,31 +267,6 @@ function prepare_test () { } -function select_build () { - - set -exuo pipefail - - # Arguments: - # - ${1} - Path to directory to copy VPP artifacts from. Required. - # Variables read: - # - DOWNLOAD_DIR - Path to directory where Robot takes builds to test from. - # - VPP_DIR - Path to existing directory, root for relative paths. - # Directories read: - # - ${1} - Existing directory with built new VPP artifacts (and DPDK). - # Directories updated: - # - ${DOWNLOAD_DIR} - Old content removed, .deb files from ${1} copied here. - # Functions called: - # - die - Print to stderr and exit, defined in common.sh - - cd "${VPP_DIR}" || die "Change directory operation failed." - source_dir="$(readlink -e "$1")" - rm -rf "${DOWNLOAD_DIR}"/* || die "Cleanup of download dir failed." - cp "${source_dir}"/*".deb" "${DOWNLOAD_DIR}" || die "Copy operation failed." - # TODO: Is there a nice way to create symlinks, - # so that if job fails on robot, results can be archived? -} - - function set_perpatch_dut () { set -exuo pipefail |