Support suite tags in autogen

+ Include a script to add suite tags to many suites at once.
+ Add suite tags also to device tests (not covered by autogen).

Change-Id: I514ee6178e22999b43460028fe2696738b012f04
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

3 files changed, 175 insertions, 22 deletions

diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 0e06857472..1b4d44c636 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -303,6 +303,14 @@ class Constants:
         u"rdma-core": u"rdma-",
     }
 
+    # Some identifiers constructed from suite names
+    # have to be independent of NIC driver used.
+    # In order to remove or reject the NIC driver part,
+    # it is useful to have a list of such prefixes precomputed.
+    FORBIDDEN_SUITE_PREFIX_LIST = [
+        prefix for prefix in NIC_DRIVER_TO_SUITE_PREFIX.values() if prefix
+    ]
+
     # Additional step for perf needs to know driver type.
     # Contains part of suite setup line, matching both single and double link.
     NIC_DRIVER_TO_SETUP_ARG = {
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index b1d0c9e993..d47680ccd0 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -11,7 +11,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Module defining utilities for test directory regeneration."""
+"""Module defining utilities for test directory regeneration.
+
+TODO: How can we check each suite id is unique,
+when currently the suite generation is run on each directory separately?
+"""
 
 import sys
 
@@ -52,9 +56,9 @@ def replace_defensively(
     :type how_many: int
     :type msg: str
     :type in_filename: str
-    :return: The whole text after replacements are done.
+    :returns: The whole text after replacements are done.
     :rtype: str
-    :raise ValueError: If number of occurrences does not match.
+    :raises ValueError: If number of occurrences does not match.
     """
     found = whole.count(to_replace)
     if found != how_many:
@@ -62,24 +66,54 @@ def replace_defensively(
     return whole.replace(to_replace, replace_with)
 
 
-def get_iface_and_suite_id(filename):
-    """Get interface and suite ID.
+def get_iface_and_suite_ids(filename):
+    """Get NIC code, suite ID and suite tag.
 
-    Interface ID is the part of suite name
+    NIC code is the part of suite name
     which should be replaced for other NIC.
     Suite ID is the part os suite name
-    which si appended to test case names.
+    which is appended to test case names.
+    Suite tag is suite ID without both test type and NIC driver parts.
 
     :param filename: Suite file.
     :type filename: str
-    :returns: Interface ID, Suite ID.
-    :rtype: (str, str)
+    :returns: NIC code, suite ID, suite tag.
+    :rtype: 3-tuple of str
     """
     dash_split = filename.split(u"-", 1)
     if len(dash_split[0]) <= 4:
         # It was something like "2n1l", we need one more split.
         dash_split = dash_split[1].split(u"-", 1)
-    return dash_split[0], dash_split[1].split(u".", 1)[0]
+    nic_code = dash_split[0]
+    suite_id = dash_split[1].split(u".", 1)[0]
+    suite_tag = suite_id.rsplit(u"-", 1)[0]
+    for prefix in Constants.FORBIDDEN_SUITE_PREFIX_LIST:
+        if suite_tag.startswith(prefix):
+            suite_tag = suite_tag[len(prefix):]
+    return nic_code, suite_id, suite_tag
+
+
+def check_suite_tag(suite_tag, prolog):
+    """Verify suite tag occurres once in prolog.
+
+    Call this after all edits are done,
+    to confirm the (edited) suite tag still matches the (edited) suite name.
+
+    Currently, the edited suite tag is expect to be identical
+    to the primary suite tag, but having a function is more flexible.
+
+    The occurences are counted including "| " prefix,
+    to lower the chance to match a comment.
+
+    :param suite_tag: Part of suite name, between NIC driver and suite type.
+    :param prolog: The part of .robot file content without test cases.
+    :type suite_tag: str
+    :type prolog: str
+    :raises ValueError: If suite_tag not found exactly once.
+    """
+    found = prolog.count(u"| " + suite_tag)
+    if found != 1:
+        raise ValueError(f"Suite tag found {found} times for {suite_id}")
 
 
 def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list):
@@ -185,7 +219,7 @@ def write_default_files(in_filename, in_prolog, kwargs_list):
             Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER[suite_type],
             1, u"Exact template type doc not found.", in_filename
         )
-        _, suite_id = get_iface_and_suite_id(tmp_filename)
+        _, suite_id, _ = get_iface_and_suite_ids(tmp_filename)
         testcase = Testcase.default(suite_id)
         for nic_name in Constants.NIC_NAME_TO_CODE:
             tmp2_filename = replace_defensively(
@@ -207,8 +241,11 @@ def write_default_files(in_filename, in_prolog, kwargs_list):
                         Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1,
                         u"HW crypto name should appear.", in_filename
                     )
-            iface, old_suite_id = get_iface_and_suite_id(tmp2_filename)
+            iface, old_suite_id, old_suite_tag = get_iface_and_suite_ids(
+                tmp2_filename
+            )
             if u"DPDK" in in_prolog:
+                check_suite_tag(old_suite_tag, tmp2_prolog)
                 with open(tmp2_filename, u"wt") as file_out:
                     file_out.write(tmp2_prolog)
                     add_default_testcases(
@@ -240,7 +277,17 @@ def write_default_files(in_filename, in_prolog, kwargs_list):
                     Constants.NIC_DRIVER_TO_SETUP_ARG[driver], 1,
                     u"Perf setup argument should appear once.", in_filename
                 )
-                iface, suite_id = get_iface_and_suite_id(out_filename)
+                iface, suite_id, suite_tag = get_iface_and_suite_ids(
+                    out_filename
+                )
+                # The next replace is probably a noop, but it is safer to maintain
+                # the same structure as for other edits.
+                out_prolog = replace_defensively(
+                    out_prolog, old_suite_tag, suite_tag, 1,
+                    f"Perf suite tag {old_suite_tag} should appear once.",
+                    in_filename
+                )
+                check_suite_tag(suite_tag, out_prolog)
                 # TODO: Reorder loops so suite_id is finalized sooner.
                 testcase = Testcase.default(suite_id)
                 with open(out_filename, u"wt") as file_out:
@@ -264,7 +311,7 @@ def write_reconf_files(in_filename, in_prolog, kwargs_list):
     :type in_prolog: str
     :type kwargs_list: list of dict
     """
-    _, suite_id = get_iface_and_suite_id(in_filename)
+    _, suite_id, _ = get_iface_and_suite_ids(in_filename)
     testcase = Testcase.default(suite_id)
     for nic_name in Constants.NIC_NAME_TO_CODE:
         tmp_filename = replace_defensively(
@@ -286,7 +333,9 @@ def write_reconf_files(in_filename, in_prolog, kwargs_list):
                     Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1,
                     u"HW crypto name should appear.", in_filename
                 )
-        iface, old_suite_id = get_iface_and_suite_id(tmp_filename)
+        iface, old_suite_id, old_suite_tag = get_iface_and_suite_ids(
+            tmp_filename
+        )
         for driver in Constants.NIC_NAME_TO_DRIVER[nic_name]:
             out_filename = replace_defensively(
                 tmp_filename, old_suite_id,
@@ -312,7 +361,12 @@ def write_reconf_files(in_filename, in_prolog, kwargs_list):
                 Constants.NIC_DRIVER_TO_SETUP_ARG[driver], 1,
                 u"Perf setup argument should appear once.", in_filename
             )
-            iface, suite_id = get_iface_and_suite_id(out_filename)
+            iface, suite_id, suite_tag = get_iface_and_suite_ids(out_filename)
+            out_prolog = replace_defensively(
+                out_prolog, old_suite_tag, suite_tag, 1,
+                u"Perf suite tag should appear once.", in_filename
+            )
+            check_suite_tag(suite_tag, out_prolog)
             # TODO: Reorder loops so suite_id is finalized sooner.
             testcase = Testcase.default(suite_id)
             with open(out_filename, u"wt") as file_out:
@@ -335,7 +389,7 @@ def write_tcp_files(in_filename, in_prolog, kwargs_list):
     :type kwargs_list: list of dict
     """
     # TODO: Generate rps from cps? There are subtle differences.
-    _, suite_id = get_iface_and_suite_id(in_filename)
+    _, suite_id, suite_tag = get_iface_and_suite_ids(in_filename)
     testcase = Testcase.tcp(suite_id)
     for nic_name in Constants.NIC_NAME_TO_CODE:
         out_filename = replace_defensively(
@@ -348,6 +402,7 @@ def write_tcp_files(in_filename, in_prolog, kwargs_list):
             u"NIC name should appear twice (tag and variable).",
             in_filename
         )
+        check_suite_tag(suite_tag, out_prolog)
         with open(out_filename, u"wt") as file_out:
             file_out.write(out_prolog)
             add_tcp_testcases(testcase, file_out, kwargs_list)
@@ -401,20 +456,17 @@ class Regenerator:
         tcp_kwargs_list = [
             {u"phy_cores": i, u"frame_size": 0} for i in (1, 2, 4)
         ]
-        forbidden = [
-            v for v in Constants.NIC_DRIVER_TO_SUITE_PREFIX.values() if v
-        ]
         for in_filename in glob(pattern):
             if not self.quiet:
                 print(
                     u"Regenerating in_filename:", in_filename, file=sys.stderr
                 )
-            iface, _ = get_iface_and_suite_id(in_filename)
+            iface, _, _ = get_iface_and_suite_ids(in_filename)
             if not iface.endswith(u"10ge2p1x710"):
                 raise RuntimeError(
                     f"Error in {in_filename}: non-primary NIC found."
                 )
-            for prefix in forbidden:
+            for prefix in Constants.FORBIDDEN_SUITE_PREFIX_LIST:
                 if prefix in in_filename:
                     raise RuntimeError(
                         f"Error in {in_filename}: non-primary driver found."
diff --git a/resources/libraries/python/autogen/add_suite_tag.py b/resources/libraries/python/autogen/add_suite_tag.py
new file mode 100755
index 0000000000..3e07316b64
--- /dev/null
+++ b/resources/libraries/python/autogen/add_suite_tag.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script for mass editing suites to add suite tag there."""
+
+import sys
+
+from io import open
+from glob import glob
+
+from resources.libraries.python.autogen.Regenerator import Regenerator
+
+
+def edit(text, suite_tag):
+    """Return the edited text.
+
+    :param text: Content of .robot file as read.
+    :param suite_tag: The value of suite tag to insert if not present.
+    :type text: str
+    :type suite_tag: str
+    :returns: New content to rewrite the file with.
+    :rtype: str
+    :raises RuntimeError: If something failed during the editing.
+    """
+    lines_out = list()
+    # Using an iterator to allow several loops in sequence.
+    lines_in = iter(text.splitlines())
+    # Searching where tags begin.
+    while 1:
+        line = next(lines_in)
+        if u"Force Tags" in line:
+            break
+        lines_out.append(line)
+    # The foce tags line has not been written yet.
+    # Search for "empty" line after tags.
+    while 1:
+        line_previous = line
+        lines_out.append(line)
+        line = next(lines_in)
+        if u"|" == line:
+            break
+    # All tags are written, we remember the last one.
+    line_suite = u"| ... | " + suite_tag
+    if line_suite != line_previous:
+        lines_out.append(line_suite)
+    # Write the empty line and copy the rest.
+    lines_out.append(line)
+    for line in lines_in:
+        lines_out.append(line)
+    # Make sure the last line ends properly.
+    lines_out.append(u"")
+    while lines_out[-2] == u"":
+        lines_out.pop()
+    return u"\n".join(lines_out)
+
+
+def main():
+    """Do it all, return return code.
+
+    :returns: 0 as everything works.
+    :rtype: int
+    """
+    for filename in glob(u"*.robot"):
+        if u"__init__" in filename:
+            continue
+        with open(filename, u"rt") as file_in:
+            text_in = file_in.read()
+        dash_split = filename.split(u"-", 1)
+        if len(dash_split[0]) <= 4:
+            # It was something like "2n1l", we need one more split.
+            dash_split = dash_split[1].split(u"-", 1)
+        suite_id = dash_split[1].split(u".", 1)[0]
+        suite_tag = suite_id.rsplit(u"-", 1)[0]
+        text_out = edit(text_in, suite_tag)
+        with open(filename, u"wt") as file_out:
+            file_out.write(text_out)
+    return 0
+
+
+if __name__ == u"__main__":
+    sys.exit(main())