From 7ea5bcd7dc334246307a50459fad80561580fa7e Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Thu, 26 Nov 2020 10:25:06 +0100 Subject: Avoid ci-management archiving files twice This is a follow-up to https://gerrit.fd.io/r/c/csit/+/20394 https://gerrit.fd.io/r/20119 has changed the way archival works, everything should now go to logs.fd.io (instead of appearing on run page in jenkins.fd.io). The glob pattern for archiving is quite eager, doing recursive search. That is good, as it can find also misplaced useful outputs. But it also means our usage of copy_archives function creates two copies of archived directtories, usually archives/ and archives/archive/. This change renames copy_archives to move_archives, with few workarounds to support multiple calls. I also renamed ARCHIVE_DIR value from $CSIT_DIR/archive to $CSIT_DIR/archives to make "move" operation look natural. Finally, download_builds function is being removed, as after recent improvements to VPP compilation speed nobody seems to be using it. Change-Id: I19a429e1dfdfaab7fcf32a9609963b1aebd33c6c Signed-off-by: Vratko Polak (cherry picked from commit 523c6e6e24101206ff1318ca17c310dff8b3c9d2) --- .../libraries/bash/entry/bootstrap_verify_perf.sh | 4 +- .../libraries/bash/entry/bootstrap_vpp_device.sh | 4 +- resources/libraries/bash/entry/per_patch_device.sh | 6 +- resources/libraries/bash/entry/per_patch_perf.sh | 8 +-- resources/libraries/bash/function/common.sh | 71 ++++++++++++++-------- resources/libraries/bash/function/per_patch.sh | 31 +--------- 6 files changed, 54 insertions(+), 70 deletions(-) diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index 2098f789cf..fc3344010b 100644 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -48,5 +48,5 @@ select_tags || die compose_pybot_arguments || die run_pybot || die untrap_and_unreserve_testbed || die -copy_archives || die +move_archives || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh index ce887f4b5b..ef7bb08dfa 100755 --- a/resources/libraries/bash/entry/bootstrap_vpp_device.sh +++ b/resources/libraries/bash/entry/bootstrap_vpp_device.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -46,5 +46,5 @@ activate_docker_topology || die select_vpp_device_tags || die compose_pybot_arguments || die run_pybot || die -copy_archives || die +move_archives || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/per_patch_device.sh b/resources/libraries/bash/entry/per_patch_device.sh index 65577c2909..01e4de0089 100644 --- a/resources/libraries/bash/entry/per_patch_device.sh +++ b/resources/libraries/bash/entry/per_patch_device.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -24,7 +24,7 @@ set -exuo pipefail # + Everything needed to build VPP is already installed locally. # Consequences: # + The following directories (relative to VPP repo) are (re)created: -# ++ csit_current, build_current, archive, csit/archive, csit_download_dir. +# ++ csit_current, build_current, archives, csit/archives, csit_download_dir. # TODO: Implement some kind of VPP build caching. @@ -56,6 +56,6 @@ activate_docker_topology || die select_vpp_device_tags || die compose_pybot_arguments || die run_pybot || die -copy_archives || die +move_archives || die archive_test_results "csit_current" || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh index deae8b0bd3..7b64aea67f 100644 --- a/resources/libraries/bash/entry/per_patch_perf.sh +++ b/resources/libraries/bash/entry/per_patch_perf.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -27,7 +27,7 @@ set -exuo pipefail # + Directories build_root, build and csit are reset during the run. # + The following directories (relative to VPP repo) are (re)created: # ++ csit_new, csit_parent, build_new, build_parent, -# ++ archive, csit/archive, csit_download_dir. +# ++ archives, csit/archives, csit_download_dir. # TODO: Implement some kind of VPP build caching. @@ -48,8 +48,6 @@ build_vpp_ubuntu_amd64 "CURRENT" || die set_aside_commit_build_artifacts || die build_vpp_ubuntu_amd64 "PARENT" || die set_aside_parent_build_artifacts || die -## Replace previous 4 lines with this to speed up testing. -#download_builds "REPLACE_WITH_URL" || die initialize_csit_dirs || die get_test_code "${1-}" || die get_test_tag_string || die @@ -74,7 +72,6 @@ for ((iter=0; iter