Upgrade autogen to NICs and search types

+ All perf suites updated, as autogen change is backward incompatible.
+ Only x710 -ndrpdr suites remain in git repository.
+ Removed retry feature of run_tests.
+ Renamed topology_type and traffic_type to traffic_profile.
+ Renamed framesize to frame_size; nic_model to nic_name.
+ Reordered Variables table to start with nic_name and overhead.
+ Fixed wrong overhead value in some cbc-sha1 suites.
+ Fixed some suite tags.
++ Introduced tags to distinguish suites, such as IPSECINT.
+ Bound crypto hardware to NIC name.
+ Implemented NIC limit hiding.
+ Also search and teardown related arguments are now hidden.
+ Main measurement keywords updated to new arg handling.
+ Max rate related keywords moved to performance_limits.robot
+ Teardowns unified.
+ Generated tests are archived.
++ Generated directory is .gitignore-d.
+ Regenerator raises an exception on seeing non-compatible suite.
++ Relatively helpful message should be seen in exception.
+ Suite and template Documentation has generated parts.
+- With short NIC names only.
+ Autogen checker also upgraded.
+ Tag expressions replaced with long files to ensure analogous tests.

Change-Id: I60e9a999187e7da1f60d0eb4fb02afa14682aa46
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

6 files changed, 41 insertions, 15 deletions

diff --git a/resources/libraries/bash/entry/README.txt b/resources/libraries/bash/entry/README.txt
index 1f6cf522bb..bf7897f8a6 100644
--- a/resources/libraries/bash/entry/README.txt
+++ b/resources/libraries/bash/entry/README.txt
@@ -1,3 +1,16 @@
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 Scripts in this directory are to be executed (or sourced)
 directly from Jenkins executor (or manually by user when testing locally),
 as opposed to indirectly from other CSIT scripts (or from Robot).
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index cfe94686cb..7c00491c90 100644
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -43,6 +43,8 @@ reserve_testbed || die
 ansible_hosts "run-jitter-tool" || die
 select_tags || die
 compose_pybot_arguments || die
+generate_tests || die
+archive_tests || die
 run_pybot || die
 untrap_and_unreserve_testbed || die
 copy_archives || die
diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh
index 139e1e99ab..156a8b32ac 100755
--- a/resources/libraries/bash/entry/bootstrap_vpp_device.sh
+++ b/resources/libraries/bash/entry/bootstrap_vpp_device.sh
@@ -41,6 +41,8 @@ activate_virtualenv || die
 activate_docker_topology || die
 select_vpp_device_tags || die
 compose_pybot_arguments || die
+generate_tests || die
+archive_tests || die
 run_pybot || die
 copy_archives || die
 die_on_pybot_error || die
diff --git a/resources/libraries/bash/entry/check/autogen.sh b/resources/libraries/bash/entry/check/autogen.sh
index 35ecc281ee..0164b65962 100644
--- a/resources/libraries/bash/entry/check/autogen.sh
+++ b/resources/libraries/bash/entry/check/autogen.sh
@@ -18,13 +18,16 @@ set -exuo pipefail
 # This file does not have executable flag nor shebang,
 # to dissuade non-tox callers.
 
-# This script run every executable *.py script anywhere within tests/ dir,
+# This script starts with copying ${CSIT_DIR}/tests to ${GENERATED_DIR}/.
+# Then the script runs every executable *.py script anywhere in the copied dir,
 # the working directory temporarily changed to where the *.py file is.
 # Proper virtualenv is assumed to be active.
-# If "git diff" sees any change, this script fails.
-# The diff output stored to autogen.log (overwriting).
-# The *.py files are assumed to be robot suite generators,
-# any change means the contribution does not match the generated code.
+# Then another directory in ${GENERATED_DIR} is created, where
+# the just generated content is copied and then overwitten by the non-generated.
+# If "diff -dur" sees any changes by the overwrite, this script fails.
+# The diff output is stored to autogen.log (overwriting).
+# The executed *.py files are assumed to be robot suite generators,
+# any change means the contribution is not consistent with the regenerated code.
 
 # "set -eu" handles failures from the following two lines.
 BASH_CHECKS_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
@@ -33,20 +36,21 @@ source "${BASH_FUNCTION_DIR}/common.sh" || {
     echo "Source failed." >&2
     exit 1
 }
-
+common_dirs
 work_dir="$(pwd)" || die
 trap "cd '${work_dir}'" EXIT || die
-file_list="$(find ./tests -type f -executable -name '*.py')" || die
 
-for gen in ${file_list}; do
-    directory="$(dirname "${gen}")" || die
-    filename="$(basename "${gen}")" || die
-    pushd "${directory}" || die
-    ./"${filename}" || die
-    popd || die
-done
+generate_tests
+
+rm -rf "${GENERATED_DIR}/tests_tmp"
+cp -r "${GENERATED_DIR}/tests" "${GENERATED_DIR}/tests_tmp"
+# Default cp behavior is to put inside a targed dir, not to override.
+cp -rf "${CSIT_DIR}/tests"/* "${GENERATED_DIR}/tests_tmp"/
+# TODO: Do we want to archive ${GENERATED_DIR}?
+# I think archiving the diff is enough.
 
-lines="$(git diff | tee "autogen.log" | wc -l)" || die
+diff_cmd=("diff" "-dur" "${GENERATED_DIR}/tests_tmp" "${GENERATED_DIR}/tests")
+lines="$("${diff_cmd[@]}" | tee "autogen.log" | wc -l)" || die
 if [ "${lines}" != "0" ]; then
     # TODO: Decide which text goes to stdout and which to stderr.
     warn "Autogen conflict diff nonzero lines: ${lines}"
diff --git a/resources/libraries/bash/entry/per_patch_device.sh b/resources/libraries/bash/entry/per_patch_device.sh
index 11203316fb..7115dc14fc 100644
--- a/resources/libraries/bash/entry/per_patch_device.sh
+++ b/resources/libraries/bash/entry/per_patch_device.sh
@@ -51,6 +51,8 @@ activate_virtualenv "${VPP_DIR}" || die
 activate_docker_topology || die
 select_vpp_device_tags || die
 compose_pybot_arguments || die
+generate_tests || die
+archive_tests || die
 run_pybot || die
 copy_archives || die
 archive_test_results "csit_current" || die
diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh
index c176bbfd5f..d3aabfae17 100644
--- a/resources/libraries/bash/entry/per_patch_perf.sh
+++ b/resources/libraries/bash/entry/per_patch_perf.sh
@@ -56,9 +56,12 @@ activate_virtualenv "${VPP_DIR}" || die
 reserve_testbed || die
 select_tags || die
 compose_pybot_arguments || die
+generate_tests || die
+archive_tests || die
 iterations=8
 for ((iter=0; iter<iterations; iter++)); do
     # TODO: Use less heavy way to avoid apt remove falilures.
+    # Also, reserve_testbed has already called cleanup once.
     cleanup_topo
     select_build "build_parent" || die
     check_download_dir || die