Upgrade autogen to NICs and search types

+ All perf suites updated, as autogen change is backward incompatible.
+ Only x710 -ndrpdr suites remain in git repository.
+ Removed retry feature of run_tests.
+ Renamed topology_type and traffic_type to traffic_profile.
+ Renamed framesize to frame_size; nic_model to nic_name.
+ Reordered Variables table to start with nic_name and overhead.
+ Fixed wrong overhead value in some cbc-sha1 suites.
+ Fixed some suite tags.
++ Introduced tags to distinguish suites, such as IPSECINT.
+ Bound crypto hardware to NIC name.
+ Implemented NIC limit hiding.
+ Also search and teardown related arguments are now hidden.
+ Main measurement keywords updated to new arg handling.
+ Max rate related keywords moved to performance_limits.robot
+ Teardowns unified.
+ Generated tests are archived.
++ Generated directory is .gitignore-d.
+ Regenerator raises an exception on seeing non-compatible suite.
++ Relatively helpful message should be seen in exception.
+ Suite and template Documentation has generated parts.
+- With short NIC names only.
+ Autogen checker also upgraded.
+ Tag expressions replaced with long files to ensure analogous tests.

Change-Id: I60e9a999187e7da1f60d0eb4fb02afa14682aa46
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

1 files changed, 81 insertions, 131 deletions

diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 2445a5deae..879602674a 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -23,6 +23,8 @@ set -exuo pipefail
 
 
 function activate_docker_topology () {
+    set -exuo pipefail
+
     # Create virtual vpp-device topology. Output of the function is topology
     # file describing created environment saved to a file.
     #
@@ -34,8 +36,6 @@ function activate_docker_topology () {
     # Variables set:
     # - WORKING_TOPOLOGY - Path to topology file.
 
-    set -exuo pipefail
-
     source "${BASH_FUNCTION_DIR}/device.sh" || {
         die "Source failed!"
     }
@@ -50,7 +50,8 @@ function activate_docker_topology () {
             hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3)
             ssh="ssh root@${hostname} -p 6022"
             run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}"
-            env_vars=$(${ssh} "$(declare -f); ${run}") || {
+            # backtics to avoid https://midnight-commander.org/ticket/2142
+            env_vars=`${ssh} "$(declare -f); ${run}"` || {
                 die "Topology reservation via shim-dcr failed!"
             }
             set -a
@@ -131,6 +132,24 @@ function activate_virtualenv () {
 }
 
 
+function archive_tests () {
+
+    set -exuo pipefail
+
+    # Create .tar.xz of generated/tests for archiving.
+    # To be run after generate_tests, kept separate to offer more flexibility.
+
+    # Directory read:
+    # - ${GENERATED_DIR}/tests - Tree of executed suites to archive.
+    # File rewriten:
+    # - ${ARCHIVE_DIR}/tests.tar.xz - Archive of generated tests.
+
+    tar c "${GENERATED_DIR}/tests" | xz -9e > "${ARCHIVE_DIR}/tests.tar.xz" || {
+        die "Error creating archive of generated tests."
+    }
+}
+
+
 function check_download_dir () {
 
     set -exuo pipefail
@@ -169,6 +188,8 @@ function common_dirs () {
 
     set -exuo pipefail
 
+    # Set global variables, create some directories (without touching content).
+
     # Variables set:
     # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
     # - CSIT_DIR - Path to existing root of local CSIT git repository.
@@ -178,6 +199,9 @@ function common_dirs () {
     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
     # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
+    # - GENERATED_DIR - Path to created CSIT subdirectory "generated".
+    # Directories created if not present:
+    # ARCHIVE_DIR, DOWNLOAD_DIR, GENERATED_DIR.
     # Functions called:
     # - die - Print to stderr and exit.
 
@@ -211,6 +235,10 @@ function common_dirs () {
         die "Readlink failed."
     }
     mkdir -p "${DOWNLOAD_DIR}" || die "Mkdir failed."
+    GENERATED_DIR="$(readlink -f "${CSIT_DIR}/generated")" || {
+        die "Readlink failed."
+    }
+    mkdir -p "${GENERATED_DIR}" || die "Mkdir failed."
 }
 
 
@@ -342,11 +370,44 @@ function die_on_pybot_error () {
     # - die - Print to stderr and exit.
 
     if [[ "${PYBOT_EXIT_STATUS}" != "0" ]]; then
-        die "${PYBOT_EXIT_STATUS}" "Test failures are present!"
+        die "Test failures are present!" "${PYBOT_EXIT_STATUS}"
     fi
 }
 
 
+function generate_tests () {
+
+    set -exuo pipefail
+
+    # Populate ${GENERATED_DIR}/tests based on ${CSIT_DIR}/tests/.
+    # Any previously existing content of ${GENERATED_DIR}/tests is wiped before.
+    # The generation is done by executing any *.py executable
+    # within any subdirectory after copying.
+
+    # This is a separate function, because this code is called
+    # both by autogen checker and entries calling run_pybot.
+
+    # Directories read:
+    # - ${CSIT_DIR}/tests - Used as templates for the generated tests.
+    # Directories replaced:
+    # - ${GENERATED_DIR}/tests - Overwritten by the generated tests.
+
+    rm -rf "${GENERATED_DIR}/tests"
+    cp -r "${CSIT_DIR}/tests" "${GENERATED_DIR}/tests"
+    cmd_line=("find" "${GENERATED_DIR}/tests" "-type" "f")
+    cmd_line+=("-executable" "-name" "*.py")
+    file_list=$("${cmd_line[@]}") || die
+
+    for gen in ${file_list}; do
+        directory="$(dirname "${gen}")" || die
+        filename="$(basename "${gen}")" || die
+        pushd "${directory}" || die
+        ./"${filename}" || die
+        popd || die
+    done
+}
+
+
 function get_test_code () {
 
     set -exuo pipefail
@@ -500,45 +561,26 @@ function run_pybot () {
 
     set -exuo pipefail
 
-    # Currently, VPP-1361 causes occasional test failures.
-    # If real result is more important than time, we can retry few times.
-    # TODO: We should be retrying on test case level instead.
-
-    # Arguments:
-    # - ${1} - Optional number of pybot invocations to try to avoid failures.
-    #   Default: 1.
     # Variables read:
     # - CSIT_DIR - Path to existing root of local CSIT git repository.
     # - ARCHIVE_DIR - Path to store robot result files in.
     # - PYBOT_ARGS, EXPANDED_TAGS - See compose_pybot_arguments.sh
+    # - GENERATED_DIR - Tests are assumed to be generated under there.
     # Variables set:
     # - PYBOT_EXIT_STATUS - Exit status of most recent pybot invocation.
     # Functions called:
     # - die - Print to stderr and exit.
 
-    # Set ${tries} as an integer variable, to fail on non-numeric input.
-    local -i "tries" || die "Setting type of variable failed."
-    tries="${1:-1}" || die "Argument evaluation failed."
     all_options=("--outputdir" "${ARCHIVE_DIR}" "${PYBOT_ARGS[@]}")
     all_options+=("${EXPANDED_TAGS[@]}")
 
-    while true; do
-        if [[ "${tries}" -le 0 ]]; then
-            break
-        else
-            tries="$((${tries} - 1))"
-        fi
-        pushd "${CSIT_DIR}" || die "Change directory operation failed."
-        set +e
-        # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
-        pybot "${all_options[@]}" "${CSIT_DIR}/tests/"
-        PYBOT_EXIT_STATUS="$?"
-        set -e
-        popd || die "Change directory operation failed."
-        if [[ "${PYBOT_EXIT_STATUS}" == "0" ]]; then
-            break
-        fi
-    done
+    pushd "${CSIT_DIR}" || die "Change directory operation failed."
+    set +e
+    # TODO: Make robot tests not require "$(pwd)" == "${CSIT_DIR}".
+    pybot "${all_options[@]}" "${GENERATED_DIR}/tests/"
+    PYBOT_EXIT_STATUS="$?"
+    set -e
+    popd || die "Change directory operation failed."
 }
 
 
@@ -582,107 +624,13 @@ function select_tags () {
     case "${TEST_CODE}" in
         # Select specific performance tests based on jenkins job type variable.
         *"ndrpdr-weekly"* )
-            test_tag_array=("ndrpdrAND64bAND1c"
-                            "ndrpdrAND78bAND1c")
+            readarray -t test_tag_array < "${BASH_FUNCTION_DIR}/mlr-weekly.txt"
             ;;
         *"mrr-daily"* )
-            test_tag_array=(# vic
-                            "mrrANDnic_cisco-vic-1227AND64b"
-                            "mrrANDnic_cisco-vic-1385AND64b"
-                            # memif
-                            "mrrANDmemifANDethAND64b"
-                            "mrrANDmemifANDethANDimix"
-                            # crypto
-                            "mrrANDipsecAND64b"
-                            # ip4 base
-                            "mrrANDip4baseAND64b"
-                            # ip4 scale FIB 2M
-                            "mrrANDip4fwdANDfib_2mAND64b"
-                            # ip4 scale FIB 200k
-                            "mrrANDip4fwdANDfib_200kANDnic_intel-*710AND64b"
-                            # ip4 scale FIB 20k
-                            "mrrANDip4fwdANDfib_20kANDnic_intel-*710AND64b"
-                            # ip4 scale ACL
-                            "mrrANDip4fwdANDacl1AND10k_flowsAND64b"
-                            "mrrANDip4fwdANDacl50AND10k_flowsAND64b"
-                            # ip4 scale NAT44
-                            "mrrANDip4fwdANDnat44ANDbaseAND64b"
-                            "mrrANDip4fwdANDnat44ANDsrc_user_4000AND64b"
-                            # ip4 features
-                            "mrrANDip4fwdANDfeatureANDnic_intel-*710AND64b"
-                            # TODO: Remove when tags in
-                            # tests/vpp/perf/ip4/*-ipolicemarkbase-*.robot
-                            # are fixed
-                            "mrrANDip4fwdANDpolice_markANDnic_intel-*710AND64b"
-                            # ip4 tunnels
-                            "mrrANDip4fwdANDencapANDip6unrlayANDip4ovrlayANDnic_intel-x520-da2AND64b"
-                            "mrrANDip4fwdANDencapANDnic_intel-*710AND64b"
-                            "mrrANDl2ovrlayANDencapANDnic_intel-*710AND64b"
-                            # ip6 base
-                            "mrrANDip6baseANDethAND78b"
-                            # ip6 features
-                            "mrrANDip6fwdANDfeatureANDnic_intel-*710AND78b"
-                            # ip6 scale FIB 2M
-                            "mrrANDip6fwdANDfib_2mANDnic_intel-*710AND78b"
-                            # ip6 scale FIB 200k
-                            "mrrANDip6fwdANDfib_200kANDnic_intel-*710AND78b"
-                            # ip6 scale FIB 20k
-                            "mrrANDip6fwdANDfib_20kANDnic_intel-*710AND78b"
-                            # ip6 tunnels
-                            "mrrANDip6fwdANDencapANDnic_intel-x520-da2AND78b"
-                            # l2xc base
-                            "mrrANDl2xcfwdANDbaseAND64b"
-                            # l2xc scale ACL
-                            "mrrANDl2xcANDacl1AND10k_flowsAND64b"
-                            "mrrANDl2xcANDacl50AND10k_flowsAND64b"
-                            # l2xc scale FIB 2M
-                            "mrrANDl2xcANDfib_2mAND64b"
-                            # l2xc scale FIB 200k
-                            "mrrANDl2xcANDfib_200kANDnic_intel-*710AND64b"
-                            # l2xc scale FIB 20k
-                            "mrrANDl2xcANDfib_20kANDnic_intel-*710AND64b"
-                            # l2bd base
-                            "mrrANDl2bdmaclrnANDbaseAND64b"
-                            # l2bd scale ACL
-                            "mrrANDl2bdmaclrnANDacl1AND10k_flowsAND64b"
-                            "mrrANDl2bdmaclrnANDacl50AND10k_flowsAND64b"
-                            # l2bd scale FIB 2M
-                            "mrrANDl2bdmaclrnANDfib_1mAND64b"
-                            # l2bd scale FIB 200k
-                            "mrrANDl2bdmaclrnANDfib_100kANDnic_intel-*710AND64b"
-                            # l2bd scale FIB 20k
-                            "mrrANDl2bdmaclrnANDfib_10kANDnic_intel-*710AND64b"
-                            # l2 patch base
-                            "mrrANDl2patchAND64b"
-                            # srv6
-                            "mrrANDsrv6ANDnic_intel-x520-da2AND78b"
-                            # vts
-                            "mrrANDvtsANDnic_intel-x520-da2AND114b"
-                            # vm vhost l2xc base
-                            "mrrANDvhostANDl2xcfwdANDbaseAND64b"
-                            "mrrANDvhostANDl2xcfwdANDbaseANDimix"
-                            # vm vhost l2bd base
-                            "mrrANDvhostANDl2bdmaclrnANDbaseAND64b"
-                            "mrrANDvhostANDl2bdmaclrnANDbaseANDimix"
-                            # vm vhost ip4 base
-                            "mrrANDvhostANDip4fwdANDbaseAND64b"
-                            "mrrANDvhostANDip4fwdANDbaseANDimix"
-                            # DPDK
-                            "mrrANDdpdkAND64b"
-                            # Exclude
-                            "!mrrANDip6baseANDdot1qAND78b"
-                            "!vhost_256ANDnic_intel-x520-da2"
-                            "!vhostANDnic_intel-xl710"
-                            "!cfs_opt"
-                            "!lbond_dpdk"
-                            "!nf_density")
+            readarray -t test_tag_array < "${BASH_FUNCTION_DIR}/mrr-daily.txt"
             ;;
         *"mrr-weekly"* )
-            test_tag_array=(# NF Density tests
-                            "mrrANDnf_densityAND64b"
-                            "mrrANDnf_densityANDimix"
-                            # DPDK
-                            "mrrANDdpdkAND64b")
+            readarray -t test_tag_array < "${BASH_FUNCTION_DIR}/mrr-weekly.txt"
             ;;
         * )
             if [[ -z "${TEST_TAG_STRING-}" ]]; then
@@ -692,7 +640,7 @@ function select_tags () {
                                 "mrrAND${DEFAULT_NIC}AND1cAND78bANDip6base"
                                 "mrrAND${DEFAULT_NIC}AND1cAND64bANDl2bdbase"
                                 "mrrAND${DEFAULT_NIC}AND1cAND64bANDl2xcbase"
-                                "!dot1q")
+                                "!dot1q" "!drv_avf")
             else
                 # If trigger contains tags, split them into array.
                 test_tag_array=(${TEST_TAG_STRING//:/ })
@@ -738,10 +686,12 @@ function select_tags () {
         prefix="${prefix}mrrAND${DEFAULT_NIC}AND"
     fi
     for tag in "${test_tag_array[@]}"; do
-        if [[ ${tag} == "!"* ]]; then
+        if [[ "${tag}" == "!"* ]]; then
             # Exclude tags are not prefixed.
             TAGS+=("${tag}")
-        else
+        elif [[ "${tag}" != "" && "${tag}" != "#"* ]]; then
+            # Empty and comment lines are skipped.
+            # Other lines are normal tags, they are to be prefixed.
             TAGS+=("${prefix}${tag}")
         fi
     done