Upgrade autogen to NICs and search types

+ All perf suites updated, as autogen change is backward incompatible.
+ Only x710 -ndrpdr suites remain in git repository.
+ Removed retry feature of run_tests.
+ Renamed topology_type and traffic_type to traffic_profile.
+ Renamed framesize to frame_size; nic_model to nic_name.
+ Reordered Variables table to start with nic_name and overhead.
+ Fixed wrong overhead value in some cbc-sha1 suites.
+ Fixed some suite tags.
++ Introduced tags to distinguish suites, such as IPSECINT.
+ Bound crypto hardware to NIC name.
+ Implemented NIC limit hiding.
+ Also search and teardown related arguments are now hidden.
+ Main measurement keywords updated to new arg handling.
+ Max rate related keywords moved to performance_limits.robot
+ Teardowns unified.
+ Generated tests are archived.
++ Generated directory is .gitignore-d.
+ Regenerator raises an exception on seeing non-compatible suite.
++ Relatively helpful message should be seen in exception.
+ Suite and template Documentation has generated parts.
+- With short NIC names only.
+ Autogen checker also upgraded.
+ Tag expressions replaced with long files to ensure analogous tests.

Change-Id: I60e9a999187e7da1f60d0eb4fb02afa14682aa46
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

3 files changed, 128 insertions, 36 deletions

diff --git a/resources/libraries/python/autogen/DefaultTestcase.py b/resources/libraries/python/autogen/DefaultTestcase.py
index 0aaf40d7ca..5db81a80d4 100644
--- a/resources/libraries/python/autogen/DefaultTestcase.py
+++ b/resources/libraries/python/autogen/DefaultTestcase.py
@@ -30,6 +30,6 @@ class DefaultTestcase(Testcase):
         template_string = r'''
 | ${tc_num}-${frame_str}-${cores_str}c-''' + suite_id + r'''
 | | [Tags] | ${frame_str} | ${cores_str}C
-| | framesize=${frame_num} | phy_cores=${cores_num}
+| | frame_size=${frame_num} | phy_cores=${cores_num}
 '''
         super(DefaultTestcase, self).__init__(template_string)
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index bae0e4f9fe..f47c88b022 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -20,6 +20,7 @@ from os import getcwd
 import sys
 
 from .DefaultTestcase import DefaultTestcase
+from resources.libraries.python.Constants import Constants
 
 
 # Copied from https://stackoverflow.com/a/14981125
@@ -63,18 +64,18 @@ class Regenerator(object):
             When None, default list is used.
             List item is a dict, argument names are keys.
             The initialized testcase_class should accept those, and "num".
-            DefaultTestcase accepts "framesize" and "phy_cores".
+            DefaultTestcase accepts "frame_size" and "phy_cores".
         :type pattern: str
         :type is_ip6: boolean
         :type tc_kwargs_list: list of tuple or None
         """
 
-        protocol_to_min_framesize = {
+        protocol_to_min_frame_size = {
             "ip4": 64,
             "ip6": 78,
             "vxlan+ip4": 114  # What is the real minimum for latency stream?
         }
-        min_framesize_values = protocol_to_min_framesize.values()
+        min_frame_size_values = protocol_to_min_frame_size.values()
 
         def get_iface_and_suite_id(filename):
             """Get interface and suite ID.
@@ -110,7 +111,7 @@ class Regenerator(object):
             """
             # TODO: Is there a better way to disable some combinations?
             emit = True
-            if kwargs["framesize"] == 9000:
+            if kwargs["frame_size"] == 9000:
                 if "vic1227" in iface:
                     # Not supported in HW.
                     emit = False
@@ -127,7 +128,7 @@ class Regenerator(object):
                 # Soak test take too long, do not risk other than tc01.
                 if kwargs["phy_cores"] != 1:
                     emit = False
-                if kwargs["framesize"] not in min_framesize_values:
+                if kwargs["frame_size"] not in min_frame_size_values:
                     emit = False
             if emit:
                 file_out.write(testcase.generate(num=num, **kwargs))
@@ -151,37 +152,128 @@ class Regenerator(object):
             """
             num = 1
             for tc_kwargs in tc_kwargs_list:
-                num = add_testcase(testcase, iface, suite_id, file_out, num,
-                                   **tc_kwargs)
+                num = add_testcase(
+                    testcase, iface, suite_id, file_out, num, **tc_kwargs)
+
+        def replace_defensively(
+                whole, to_replace, replace_with, how_many, msg, in_filename):
+            """Replace substrings while checking the number of occurences.
+
+            Return edited copy of the text (so original string is not affected).
+
+            :param whole: The text to perform replacements on.
+            :param to_replace: Substring occurences of which to replace.
+            :param replace_with: Substring to replace occurences with.
+            :param how_many: Number of occurences to expect.
+            :param msg: Error message to raise.
+            :param in_filename: File name in which the error occured.
+            :type whole: str
+            :type to_replace: str
+            :type replace_with: str
+            :type how_many: int
+            :type msg: str
+            :type in_filename: str
+            :return: The whole text after replacements are done.
+            :rtype: str
+            :raise ValueError: If number of occurences does not match.
+            """
+            found = whole.count(to_replace)
+            if found != how_many:
+                raise ValueError(in_filename + ": " + msg)
+            return whole.replace(to_replace, replace_with)
+
+        def write_files(in_filename, in_prolog, kwargs_list):
+            """Using given filename and prolog, write all generated suites.
+
+            :param in_filename: Template filename to derive real filenames from.
+            :param in_prolog: Template content to derive real content from.
+            :param kwargs_list: List of kwargs for add_testcase.
+            :type in_filename: str
+            :type in_prolog: str
+            :type kwargs_list: list of dict
+            """
+            for suite_type in Constants.PERF_TYPE_TO_KEYWORD.keys():
+                tmp_filename = replace_defensively(
+                    in_filename, "ndrpdr", suite_type, 1,
+                    "File name should contain suite type once.", in_filename)
+                tmp_prolog = replace_defensively(
+                    in_prolog, "ndrpdr".upper(), suite_type.upper(), 1,
+                    "Suite type should appear once in uppercase (as tag).",
+                    in_filename)
+                tmp_prolog = replace_defensively(
+                    tmp_prolog,
+                    "Find NDR and PDR intervals using optimized search",
+                    Constants.PERF_TYPE_TO_KEYWORD[suite_type], 1,
+                    "Main search keyword should appear once in suite.",
+                    in_filename)
+                tmp_prolog = replace_defensively(
+                    tmp_prolog,
+                    Constants.PERF_TYPE_TO_SUITE_DOC_VER["ndrpdr"],
+                    Constants.PERF_TYPE_TO_SUITE_DOC_VER[suite_type],
+                    1, "Exact suite type doc not found.", in_filename)
+                tmp_prolog = replace_defensively(
+                    tmp_prolog,
+                    Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER["ndrpdr"],
+                    Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER[suite_type],
+                    1, "Exact template type doc not found.", in_filename)
+                _, suite_id = get_iface_and_suite_id(tmp_filename)
+                testcase = self.testcase_class(suite_id)
+                for nic_name in Constants.NIC_NAME_TO_CODE.keys():
+                    out_filename = replace_defensively(
+                        tmp_filename, "10ge2p1x710",
+                        Constants.NIC_NAME_TO_CODE[nic_name], 1,
+                        "File name should contain NIC code once.", in_filename)
+                    out_prolog = replace_defensively(
+                        tmp_prolog, "Intel-X710", nic_name, 2,
+                        "NIC name should appear twice (tag and variable).",
+                        in_filename)
+                    if out_prolog.count("HW_") == 2:
+                        # TODO CSIT-1481: Crypto HW should be read
+                        # from topology file instead.
+                        if nic_name in Constants.NIC_NAME_TO_CRYPTO_HW.keys():
+                            out_prolog = replace_defensively(
+                                out_prolog, "HW_DH895xcc",
+                                Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1,
+                                "HW crypto name should appear.", in_filename)
+                    iface, suite_id = get_iface_and_suite_id(out_filename)
+                    with open(out_filename, "w") as file_out:
+                        file_out.write(out_prolog)
+                        add_testcases(
+                            testcase, iface, suite_id, file_out, kwargs_list)
 
         if not self.quiet:
             eprint("Regenerator starts at {cwd}".format(cwd=getcwd()))
-        min_framesize = protocol_to_min_framesize[protocol]
+        min_frame_size = protocol_to_min_frame_size[protocol]
         kwargs_list = tc_kwargs_list if tc_kwargs_list else [
-            {"framesize": min_framesize, "phy_cores": 1},
-            {"framesize": min_framesize, "phy_cores": 2},
-            {"framesize": min_framesize, "phy_cores": 4},
-            {"framesize": 1518, "phy_cores": 1},
-            {"framesize": 1518, "phy_cores": 2},
-            {"framesize": 1518, "phy_cores": 4},
-            {"framesize": 9000, "phy_cores": 1},
-            {"framesize": 9000, "phy_cores": 2},
-            {"framesize": 9000, "phy_cores": 4},
-            {"framesize": "IMIX_v4_1", "phy_cores": 1},
-            {"framesize": "IMIX_v4_1", "phy_cores": 2},
-            {"framesize": "IMIX_v4_1", "phy_cores": 4}
+            {"frame_size": min_frame_size, "phy_cores": 1},
+            {"frame_size": min_frame_size, "phy_cores": 2},
+            {"frame_size": min_frame_size, "phy_cores": 4},
+            {"frame_size": 1518, "phy_cores": 1},
+            {"frame_size": 1518, "phy_cores": 2},
+            {"frame_size": 1518, "phy_cores": 4},
+            {"frame_size": 9000, "phy_cores": 1},
+            {"frame_size": 9000, "phy_cores": 2},
+            {"frame_size": 9000, "phy_cores": 4},
+            {"frame_size": "IMIX_v4_1", "phy_cores": 1},
+            {"frame_size": "IMIX_v4_1", "phy_cores": 2},
+            {"frame_size": "IMIX_v4_1", "phy_cores": 4}
         ]
-        for filename in glob(pattern):
+        for in_filename in glob(pattern):
             if not self.quiet:
-                eprint("Regenerating filename:", filename)
-            with open(filename, "r") as file_in:
-                text = file_in.read()
-                text_prolog = "".join(text.partition("*** Test Cases ***")[:-1])
-            iface, suite_id = get_iface_and_suite_id(filename)
-            testcase = self.testcase_class(suite_id)
-            with open(filename, "w") as file_out:
-                file_out.write(text_prolog)
-                add_testcases(testcase, iface, suite_id, file_out, kwargs_list)
+                eprint("Regenerating in_filename:", in_filename)
+            if not in_filename.endswith("ndrpdr.robot"):
+                eprint("Error in {fil}: non-primary suite type encountered."
+                       .format(fil=in_filename))
+                sys.exit(1)
+            iface, _ = get_iface_and_suite_id(in_filename)
+            if not iface.endswith("10ge2p1x710"):
+                eprint("Error in {fil}: non-primary NIC encountered."
+                       .format(fil=in_filename))
+                sys.exit(1)
+            with open(in_filename, "r") as file_in:
+                in_prolog = "".join(
+                    file_in.read().partition("*** Test Cases ***")[:-1])
+            write_files(in_filename, in_prolog, kwargs_list)
         if not self.quiet:
             eprint("Regenerator ends.")
         eprint()  # To make autogen check output more readable.
diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py
index dd58547f33..7d0a29b0ad 100644
--- a/resources/libraries/python/autogen/Testcase.py
+++ b/resources/libraries/python/autogen/Testcase.py
@@ -34,29 +34,29 @@ class Testcase(object):
         """
         self.template = Template(template_string)
 
-    def generate(self, num, framesize, phy_cores):
+    def generate(self, num, frame_size, phy_cores):
         """Return string of test case code with placeholders filled.
 
         Fail if there are placeholders left unfilled.
 
         :param num: Test case number. Example value: 4.
-        :param framesize: Imix string or numeric frame size. Example: 74.
+        :param frame_size: Imix string or numeric frame size. Example: 74.
         :param phy_cores: Number of physical cores to use. Example: 2.
         :type num: int
-        :type framesize: str or int
+        :type frame_size: str or int
         :type phy_cores: int or str
         :returns: Filled template, usable as test case code.
         :rtype: str
         """
         try:
-            fsize = int(framesize)
+            fsize = int(frame_size)
             subst_dict = {
                 "frame_num": "${%d}" % fsize,
                 "frame_str": "%dB" % fsize
             }
         except ValueError:  # Assuming an IMIX string.
             subst_dict = {
-                "frame_num": str(framesize),
+                "frame_num": str(frame_size),
                 "frame_str": "IMIX"
             }
         cores_str = str(phy_cores)