From 5e6145a4260ffce1c302e94b9b241851f90838e1 Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Thu, 4 Apr 2019 13:45:11 +0200 Subject: Upgrade autogen to NICs and search types + All perf suites updated, as autogen change is backward incompatible. + Only x710 -ndrpdr suites remain in git repository. + Removed retry feature of run_tests. + Renamed topology_type and traffic_type to traffic_profile. + Renamed framesize to frame_size; nic_model to nic_name. + Reordered Variables table to start with nic_name and overhead. + Fixed wrong overhead value in some cbc-sha1 suites. + Fixed some suite tags. ++ Introduced tags to distinguish suites, such as IPSECINT. + Bound crypto hardware to NIC name. + Implemented NIC limit hiding. + Also search and teardown related arguments are now hidden. + Main measurement keywords updated to new arg handling. + Max rate related keywords moved to performance_limits.robot + Teardowns unified. + Generated tests are archived. ++ Generated directory is .gitignore-d. + Regenerator raises an exception on seeing non-compatible suite. ++ Relatively helpful message should be seen in exception. + Suite and template Documentation has generated parts. +- With short NIC names only. + Autogen checker also upgraded. + Tag expressions replaced with long files to ensure analogous tests. Change-Id: I60e9a999187e7da1f60d0eb4fb02afa14682aa46 Signed-off-by: Vratko Polak --- resources/libraries/bash/entry/README.txt | 13 +++++++++ .../libraries/bash/entry/bootstrap_verify_perf.sh | 2 ++ .../libraries/bash/entry/bootstrap_vpp_device.sh | 2 ++ resources/libraries/bash/entry/check/autogen.sh | 34 ++++++++++++---------- resources/libraries/bash/entry/per_patch_device.sh | 2 ++ resources/libraries/bash/entry/per_patch_perf.sh | 3 ++ 6 files changed, 41 insertions(+), 15 deletions(-) (limited to 'resources/libraries/bash/entry') diff --git a/resources/libraries/bash/entry/README.txt b/resources/libraries/bash/entry/README.txt index 1f6cf522bb..bf7897f8a6 100644 --- a/resources/libraries/bash/entry/README.txt +++ b/resources/libraries/bash/entry/README.txt @@ -1,3 +1,16 @@ +# Copyright (c) 2019 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Scripts in this directory are to be executed (or sourced) directly from Jenkins executor (or manually by user when testing locally), as opposed to indirectly from other CSIT scripts (or from Robot). diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index cfe94686cb..7c00491c90 100644 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -43,6 +43,8 @@ reserve_testbed || die ansible_hosts "run-jitter-tool" || die select_tags || die compose_pybot_arguments || die +generate_tests || die +archive_tests || die run_pybot || die untrap_and_unreserve_testbed || die copy_archives || die diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh index 139e1e99ab..156a8b32ac 100755 --- a/resources/libraries/bash/entry/bootstrap_vpp_device.sh +++ b/resources/libraries/bash/entry/bootstrap_vpp_device.sh @@ -41,6 +41,8 @@ activate_virtualenv || die activate_docker_topology || die select_vpp_device_tags || die compose_pybot_arguments || die +generate_tests || die +archive_tests || die run_pybot || die copy_archives || die die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/check/autogen.sh b/resources/libraries/bash/entry/check/autogen.sh index 35ecc281ee..0164b65962 100644 --- a/resources/libraries/bash/entry/check/autogen.sh +++ b/resources/libraries/bash/entry/check/autogen.sh @@ -18,13 +18,16 @@ set -exuo pipefail # This file does not have executable flag nor shebang, # to dissuade non-tox callers. -# This script run every executable *.py script anywhere within tests/ dir, +# This script starts with copying ${CSIT_DIR}/tests to ${GENERATED_DIR}/. +# Then the script runs every executable *.py script anywhere in the copied dir, # the working directory temporarily changed to where the *.py file is. # Proper virtualenv is assumed to be active. -# If "git diff" sees any change, this script fails. -# The diff output stored to autogen.log (overwriting). -# The *.py files are assumed to be robot suite generators, -# any change means the contribution does not match the generated code. +# Then another directory in ${GENERATED_DIR} is created, where +# the just generated content is copied and then overwitten by the non-generated. +# If "diff -dur" sees any changes by the overwrite, this script fails. +# The diff output is stored to autogen.log (overwriting). +# The executed *.py files are assumed to be robot suite generators, +# any change means the contribution is not consistent with the regenerated code. # "set -eu" handles failures from the following two lines. BASH_CHECKS_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" @@ -33,20 +36,21 @@ source "${BASH_FUNCTION_DIR}/common.sh" || { echo "Source failed." >&2 exit 1 } - +common_dirs work_dir="$(pwd)" || die trap "cd '${work_dir}'" EXIT || die -file_list="$(find ./tests -type f -executable -name '*.py')" || die -for gen in ${file_list}; do - directory="$(dirname "${gen}")" || die - filename="$(basename "${gen}")" || die - pushd "${directory}" || die - ./"${filename}" || die - popd || die -done +generate_tests + +rm -rf "${GENERATED_DIR}/tests_tmp" +cp -r "${GENERATED_DIR}/tests" "${GENERATED_DIR}/tests_tmp" +# Default cp behavior is to put inside a targed dir, not to override. +cp -rf "${CSIT_DIR}/tests"/* "${GENERATED_DIR}/tests_tmp"/ +# TODO: Do we want to archive ${GENERATED_DIR}? +# I think archiving the diff is enough. -lines="$(git diff | tee "autogen.log" | wc -l)" || die +diff_cmd=("diff" "-dur" "${GENERATED_DIR}/tests_tmp" "${GENERATED_DIR}/tests") +lines="$("${diff_cmd[@]}" | tee "autogen.log" | wc -l)" || die if [ "${lines}" != "0" ]; then # TODO: Decide which text goes to stdout and which to stderr. warn "Autogen conflict diff nonzero lines: ${lines}" diff --git a/resources/libraries/bash/entry/per_patch_device.sh b/resources/libraries/bash/entry/per_patch_device.sh index 11203316fb..7115dc14fc 100644 --- a/resources/libraries/bash/entry/per_patch_device.sh +++ b/resources/libraries/bash/entry/per_patch_device.sh @@ -51,6 +51,8 @@ activate_virtualenv "${VPP_DIR}" || die activate_docker_topology || die select_vpp_device_tags || die compose_pybot_arguments || die +generate_tests || die +archive_tests || die run_pybot || die copy_archives || die archive_test_results "csit_current" || die diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh index c176bbfd5f..d3aabfae17 100644 --- a/resources/libraries/bash/entry/per_patch_perf.sh +++ b/resources/libraries/bash/entry/per_patch_perf.sh @@ -56,9 +56,12 @@ activate_virtualenv "${VPP_DIR}" || die reserve_testbed || die select_tags || die compose_pybot_arguments || die +generate_tests || die +archive_tests || die iterations=8 for ((iter=0; iter