diff options
6 files changed, 54 insertions, 70 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index 2098f789cf..fc3344010b 100644 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -48,5 +48,5 @@ select_tags || die compose_pybot_arguments || die run_pybot || die untrap_and_unreserve_testbed || die -copy_archives || die +move_archives || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh index ce887f4b5b..ef7bb08dfa 100755 --- a/resources/libraries/bash/entry/bootstrap_vpp_device.sh +++ b/resources/libraries/bash/entry/bootstrap_vpp_device.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -46,5 +46,5 @@ activate_docker_topology || die select_vpp_device_tags || die compose_pybot_arguments || die run_pybot || die -copy_archives || die +move_archives || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/per_patch_device.sh b/resources/libraries/bash/entry/per_patch_device.sh index 65577c2909..01e4de0089 100644 --- a/resources/libraries/bash/entry/per_patch_device.sh +++ b/resources/libraries/bash/entry/per_patch_device.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -24,7 +24,7 @@ set -exuo pipefail # + Everything needed to build VPP is already installed locally. # Consequences: # + The following directories (relative to VPP repo) are (re)created: -# ++ csit_current, build_current, archive, csit/archive, csit_download_dir. +# ++ csit_current, build_current, archives, csit/archives, csit_download_dir. # TODO: Implement some kind of VPP build caching. @@ -56,6 +56,6 @@ activate_docker_topology || die select_vpp_device_tags || die compose_pybot_arguments || die run_pybot || die -copy_archives || die +move_archives || die archive_test_results "csit_current" || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh index deae8b0bd3..7b64aea67f 100644 --- a/resources/libraries/bash/entry/per_patch_perf.sh +++ b/resources/libraries/bash/entry/per_patch_perf.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -27,7 +27,7 @@ set -exuo pipefail # + Directories build_root, build and csit are reset during the run. # + The following directories (relative to VPP repo) are (re)created: # ++ csit_new, csit_parent, build_new, build_parent, -# ++ archive, csit/archive, csit_download_dir. +# ++ archives, csit/archives, csit_download_dir. # TODO: Implement some kind of VPP build caching. @@ -48,8 +48,6 @@ build_vpp_ubuntu_amd64 "CURRENT" || die set_aside_commit_build_artifacts || die build_vpp_ubuntu_amd64 "PARENT" || die set_aside_parent_build_artifacts || die -## Replace previous 4 lines with this to speed up testing. -#download_builds "REPLACE_WITH_URL" || die initialize_csit_dirs || die get_test_code "${1-}" || die get_test_tag_string || die @@ -74,7 +72,6 @@ for ((iter=0; iter<iterations; iter++)); do select_build "build_current" || die check_download_dir || die run_pybot || die - copy_archives || die archive_parse_test_results "csit_current/${iter}" || die die_on_pybot_error || die # TODO: Use less heavy way to avoid apt remove failures. @@ -82,7 +79,6 @@ for ((iter=0; iter<iterations; iter++)); do select_build "build_parent" || die check_download_dir || die run_pybot || die - copy_archives || die archive_parse_test_results "csit_parent/${iter}" || die die_on_pybot_error || die done diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index af59bc4ed3..c0aa5f9f7d 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -198,7 +198,8 @@ function common_dirs () { # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources". # - TOOLS_DIR - Path to existing resources subdirectory "tools". # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts". - # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive". + # - ARCHIVE_DIR - Path to created CSIT subdirectory "archives". + # The name is chosen to match what ci-management expects. # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir". # - GENERATED_DIR - Path to created CSIT subdirectory "generated". # Directories created if not present: @@ -240,7 +241,7 @@ function common_dirs () { die "Readlink failed." } - ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archive") || { + ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archives") || { die "Readlink failed." } mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed." @@ -295,31 +296,6 @@ function compose_pybot_arguments () { } -function copy_archives () { - - # Create additional archive if workspace variable is set. - # This way if script is running in jenkins all will be - # automatically archived to logs.fd.io. - # - # Variables read: - # - WORKSPACE - Jenkins workspace, copy only if the value is not empty. - # Can be unset, then it speeds up manual testing. - # - ARCHIVE_DIR - Path to directory with content to be copied. - # Directories updated: - # - ${WORKSPACE}/archives/ - Created if does not exist. - # Content of ${ARCHIVE_DIR}/ is copied here. - # Functions called: - # - die - Print to stderr and exit. - - set -exuo pipefail - - if [[ -n "${WORKSPACE-}" ]]; then - mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed." - cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed." - fi -} - - function deactivate_docker_topology () { # Deactivate virtual vpp-device topology by removing containers. @@ -568,6 +544,47 @@ function installed () { } +function move_archives () { + + # Move archive directory to top of workspace, if not already there. + # + # ARCHIVE_DIR is positioned relative to CSIT_DIR, + # but in some jobs CSIT_DIR is not same as WORKSPACE + # (e.g. under VPP_DIR). To simplify ci-management settings, + # we want to move the data to the top. We do not want simple copy, + # as ci-management is eager with recursive search. + # + # As some scripts may call this function multiple times, + # the actual implementation use copying and deletion, + # so the workspace gets "union" of contents (except overwrites on conflict). + # The consequence is empty ARCHIVE_DIR remaining after this call. + # + # As the source directory is emptied, + # the check for dirs being different is essential. + # + # Variables read: + # - WORKSPACE - Jenkins workspace, move only if the value is not empty. + # Can be unset, then it speeds up manual testing. + # - ARCHIVE_DIR - Path to directory with content to be moved. + # Directories updated: + # - ${WORKSPACE}/archives/ - Created if does not exist. + # Content of ${ARCHIVE_DIR}/ is moved. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + if [[ -n "${WORKSPACE-}" ]]; then + target=$(readlink -f "${WORKSPACE}/archives") + if [[ "${target}" != "${ARCHIVE_DIR}" ]]; then + mkdir -p "${target}" || die "Archives dir create failed." + cp -rf "${ARCHIVE_DIR}"/* "${target}" || die "Copy failed." + rm -rf "${ARCHIVE_DIR}"/* || die "Delete failed." + fi + fi +} + + function reserve_and_cleanup_testbed () { # Reserve physical testbed, perform cleanup, register trap to unreserve. diff --git a/resources/libraries/bash/function/per_patch.sh b/resources/libraries/bash/function/per_patch.sh index 8b97780cec..43a3f971bf 100644 --- a/resources/libraries/bash/function/per_patch.sh +++ b/resources/libraries/bash/function/per_patch.sh @@ -128,39 +128,10 @@ function compare_test_results () { } -function download_builds () { - - # This is mostly useful only for Sandbox testing, to avoid recompilation. - # - # Arguments: - # - ${1} - URL to download VPP builds from. - # Variables read: - # - VPP_DIR - Path to WORKSPACE, parent of created directories. - # Directories created: - # - archive - Ends up empty, not to be confused with ${ARCHIVE_DIR}. - # - build_current - Holding built artifacts of the patch under test (PUT). - # - built_parent - Holding built artifacts of parent of PUT. - # Functions called: - # - die - Print to stderr and exit, defined in common.sh +function initialize_csit_dirs () { set -exuo pipefail - cd "${VPP_DIR}" || die "Change directory operation failed." - dirs=("build-root" "build_parent" "build_current" "archive" "csit_current") - rm -rf ${dirs[@]} || { - die "Directory removal failed." - } - wget -N --progress=dot:giga "${1}" || die "Wget download failed." - unzip "archive.zip" || die "Archive extraction failed." - mv "archive/build_parent" ./ || die "Move operation failed." - mv "archive/build_current" ./ || die "Move operation failed." -} - - -function initialize_csit_dirs () { - - # This could be in prepare_test, but download_builds also needs this. - # # Variables read: # - VPP_DIR - Path to WORKSPACE, parent of created directories. # Directories created: |