aboutsummaryrefslogtreecommitdiffstats
path: root/resources/libraries/python/autogen
diff options
context:
space:
mode:
authorVratko Polak <vrpolak@cisco.com>2019-04-04 13:45:11 +0200
committerVratko Polak <vrpolak@cisco.com>2019-04-08 16:34:10 +0200
commit5e6145a4260ffce1c302e94b9b241851f90838e1 (patch)
tree2480a5419cf127ac64262d271b8094f4834b2a66 /resources/libraries/python/autogen
parent5221220aa5a893e93d78ba31f84b0ad4d6245ec3 (diff)
Upgrade autogen to NICs and search types
+ All perf suites updated, as autogen change is backward incompatible. + Only x710 -ndrpdr suites remain in git repository. + Removed retry feature of run_tests. + Renamed topology_type and traffic_type to traffic_profile. + Renamed framesize to frame_size; nic_model to nic_name. + Reordered Variables table to start with nic_name and overhead. + Fixed wrong overhead value in some cbc-sha1 suites. + Fixed some suite tags. ++ Introduced tags to distinguish suites, such as IPSECINT. + Bound crypto hardware to NIC name. + Implemented NIC limit hiding. + Also search and teardown related arguments are now hidden. + Main measurement keywords updated to new arg handling. + Max rate related keywords moved to performance_limits.robot + Teardowns unified. + Generated tests are archived. ++ Generated directory is .gitignore-d. + Regenerator raises an exception on seeing non-compatible suite. ++ Relatively helpful message should be seen in exception. + Suite and template Documentation has generated parts. +- With short NIC names only. + Autogen checker also upgraded. + Tag expressions replaced with long files to ensure analogous tests. Change-Id: I60e9a999187e7da1f60d0eb4fb02afa14682aa46 Signed-off-by: Vratko Polak <vrpolak@cisco.com>
Diffstat (limited to 'resources/libraries/python/autogen')
-rw-r--r--resources/libraries/python/autogen/DefaultTestcase.py2
-rw-r--r--resources/libraries/python/autogen/Regenerator.py152
-rw-r--r--resources/libraries/python/autogen/Testcase.py10
3 files changed, 128 insertions, 36 deletions
diff --git a/resources/libraries/python/autogen/DefaultTestcase.py b/resources/libraries/python/autogen/DefaultTestcase.py
index 0aaf40d7ca..5db81a80d4 100644
--- a/resources/libraries/python/autogen/DefaultTestcase.py
+++ b/resources/libraries/python/autogen/DefaultTestcase.py
@@ -30,6 +30,6 @@ class DefaultTestcase(Testcase):
template_string = r'''
| ${tc_num}-${frame_str}-${cores_str}c-''' + suite_id + r'''
| | [Tags] | ${frame_str} | ${cores_str}C
-| | framesize=${frame_num} | phy_cores=${cores_num}
+| | frame_size=${frame_num} | phy_cores=${cores_num}
'''
super(DefaultTestcase, self).__init__(template_string)
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index bae0e4f9fe..f47c88b022 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -20,6 +20,7 @@ from os import getcwd
import sys
from .DefaultTestcase import DefaultTestcase
+from resources.libraries.python.Constants import Constants
# Copied from https://stackoverflow.com/a/14981125
@@ -63,18 +64,18 @@ class Regenerator(object):
When None, default list is used.
List item is a dict, argument names are keys.
The initialized testcase_class should accept those, and "num".
- DefaultTestcase accepts "framesize" and "phy_cores".
+ DefaultTestcase accepts "frame_size" and "phy_cores".
:type pattern: str
:type is_ip6: boolean
:type tc_kwargs_list: list of tuple or None
"""
- protocol_to_min_framesize = {
+ protocol_to_min_frame_size = {
"ip4": 64,
"ip6": 78,
"vxlan+ip4": 114 # What is the real minimum for latency stream?
}
- min_framesize_values = protocol_to_min_framesize.values()
+ min_frame_size_values = protocol_to_min_frame_size.values()
def get_iface_and_suite_id(filename):
"""Get interface and suite ID.
@@ -110,7 +111,7 @@ class Regenerator(object):
"""
# TODO: Is there a better way to disable some combinations?
emit = True
- if kwargs["framesize"] == 9000:
+ if kwargs["frame_size"] == 9000:
if "vic1227" in iface:
# Not supported in HW.
emit = False
@@ -127,7 +128,7 @@ class Regenerator(object):
# Soak test take too long, do not risk other than tc01.
if kwargs["phy_cores"] != 1:
emit = False
- if kwargs["framesize"] not in min_framesize_values:
+ if kwargs["frame_size"] not in min_frame_size_values:
emit = False
if emit:
file_out.write(testcase.generate(num=num, **kwargs))
@@ -151,37 +152,128 @@ class Regenerator(object):
"""
num = 1
for tc_kwargs in tc_kwargs_list:
- num = add_testcase(testcase, iface, suite_id, file_out, num,
- **tc_kwargs)
+ num = add_testcase(
+ testcase, iface, suite_id, file_out, num, **tc_kwargs)
+
+ def replace_defensively(
+ whole, to_replace, replace_with, how_many, msg, in_filename):
+ """Replace substrings while checking the number of occurences.
+
+ Return edited copy of the text (so original string is not affected).
+
+ :param whole: The text to perform replacements on.
+ :param to_replace: Substring occurences of which to replace.
+ :param replace_with: Substring to replace occurences with.
+ :param how_many: Number of occurences to expect.
+ :param msg: Error message to raise.
+ :param in_filename: File name in which the error occured.
+ :type whole: str
+ :type to_replace: str
+ :type replace_with: str
+ :type how_many: int
+ :type msg: str
+ :type in_filename: str
+ :return: The whole text after replacements are done.
+ :rtype: str
+ :raise ValueError: If number of occurences does not match.
+ """
+ found = whole.count(to_replace)
+ if found != how_many:
+ raise ValueError(in_filename + ": " + msg)
+ return whole.replace(to_replace, replace_with)
+
+ def write_files(in_filename, in_prolog, kwargs_list):
+ """Using given filename and prolog, write all generated suites.
+
+ :param in_filename: Template filename to derive real filenames from.
+ :param in_prolog: Template content to derive real content from.
+ :param kwargs_list: List of kwargs for add_testcase.
+ :type in_filename: str
+ :type in_prolog: str
+ :type kwargs_list: list of dict
+ """
+ for suite_type in Constants.PERF_TYPE_TO_KEYWORD.keys():
+ tmp_filename = replace_defensively(
+ in_filename, "ndrpdr", suite_type, 1,
+ "File name should contain suite type once.", in_filename)
+ tmp_prolog = replace_defensively(
+ in_prolog, "ndrpdr".upper(), suite_type.upper(), 1,
+ "Suite type should appear once in uppercase (as tag).",
+ in_filename)
+ tmp_prolog = replace_defensively(
+ tmp_prolog,
+ "Find NDR and PDR intervals using optimized search",
+ Constants.PERF_TYPE_TO_KEYWORD[suite_type], 1,
+ "Main search keyword should appear once in suite.",
+ in_filename)
+ tmp_prolog = replace_defensively(
+ tmp_prolog,
+ Constants.PERF_TYPE_TO_SUITE_DOC_VER["ndrpdr"],
+ Constants.PERF_TYPE_TO_SUITE_DOC_VER[suite_type],
+ 1, "Exact suite type doc not found.", in_filename)
+ tmp_prolog = replace_defensively(
+ tmp_prolog,
+ Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER["ndrpdr"],
+ Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER[suite_type],
+ 1, "Exact template type doc not found.", in_filename)
+ _, suite_id = get_iface_and_suite_id(tmp_filename)
+ testcase = self.testcase_class(suite_id)
+ for nic_name in Constants.NIC_NAME_TO_CODE.keys():
+ out_filename = replace_defensively(
+ tmp_filename, "10ge2p1x710",
+ Constants.NIC_NAME_TO_CODE[nic_name], 1,
+ "File name should contain NIC code once.", in_filename)
+ out_prolog = replace_defensively(
+ tmp_prolog, "Intel-X710", nic_name, 2,
+ "NIC name should appear twice (tag and variable).",
+ in_filename)
+ if out_prolog.count("HW_") == 2:
+ # TODO CSIT-1481: Crypto HW should be read
+ # from topology file instead.
+ if nic_name in Constants.NIC_NAME_TO_CRYPTO_HW.keys():
+ out_prolog = replace_defensively(
+ out_prolog, "HW_DH895xcc",
+ Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1,
+ "HW crypto name should appear.", in_filename)
+ iface, suite_id = get_iface_and_suite_id(out_filename)
+ with open(out_filename, "w") as file_out:
+ file_out.write(out_prolog)
+ add_testcases(
+ testcase, iface, suite_id, file_out, kwargs_list)
if not self.quiet:
eprint("Regenerator starts at {cwd}".format(cwd=getcwd()))
- min_framesize = protocol_to_min_framesize[protocol]
+ min_frame_size = protocol_to_min_frame_size[protocol]
kwargs_list = tc_kwargs_list if tc_kwargs_list else [
- {"framesize": min_framesize, "phy_cores": 1},
- {"framesize": min_framesize, "phy_cores": 2},
- {"framesize": min_framesize, "phy_cores": 4},
- {"framesize": 1518, "phy_cores": 1},
- {"framesize": 1518, "phy_cores": 2},
- {"framesize": 1518, "phy_cores": 4},
- {"framesize": 9000, "phy_cores": 1},
- {"framesize": 9000, "phy_cores": 2},
- {"framesize": 9000, "phy_cores": 4},
- {"framesize": "IMIX_v4_1", "phy_cores": 1},
- {"framesize": "IMIX_v4_1", "phy_cores": 2},
- {"framesize": "IMIX_v4_1", "phy_cores": 4}
+ {"frame_size": min_frame_size, "phy_cores": 1},
+ {"frame_size": min_frame_size, "phy_cores": 2},
+ {"frame_size": min_frame_size, "phy_cores": 4},
+ {"frame_size": 1518, "phy_cores": 1},
+ {"frame_size": 1518, "phy_cores": 2},
+ {"frame_size": 1518, "phy_cores": 4},
+ {"frame_size": 9000, "phy_cores": 1},
+ {"frame_size": 9000, "phy_cores": 2},
+ {"frame_size": 9000, "phy_cores": 4},
+ {"frame_size": "IMIX_v4_1", "phy_cores": 1},
+ {"frame_size": "IMIX_v4_1", "phy_cores": 2},
+ {"frame_size": "IMIX_v4_1", "phy_cores": 4}
]
- for filename in glob(pattern):
+ for in_filename in glob(pattern):
if not self.quiet:
- eprint("Regenerating filename:", filename)
- with open(filename, "r") as file_in:
- text = file_in.read()
- text_prolog = "".join(text.partition("*** Test Cases ***")[:-1])
- iface, suite_id = get_iface_and_suite_id(filename)
- testcase = self.testcase_class(suite_id)
- with open(filename, "w") as file_out:
- file_out.write(text_prolog)
- add_testcases(testcase, iface, suite_id, file_out, kwargs_list)
+ eprint("Regenerating in_filename:", in_filename)
+ if not in_filename.endswith("ndrpdr.robot"):
+ eprint("Error in {fil}: non-primary suite type encountered."
+ .format(fil=in_filename))
+ sys.exit(1)
+ iface, _ = get_iface_and_suite_id(in_filename)
+ if not iface.endswith("10ge2p1x710"):
+ eprint("Error in {fil}: non-primary NIC encountered."
+ .format(fil=in_filename))
+ sys.exit(1)
+ with open(in_filename, "r") as file_in:
+ in_prolog = "".join(
+ file_in.read().partition("*** Test Cases ***")[:-1])
+ write_files(in_filename, in_prolog, kwargs_list)
if not self.quiet:
eprint("Regenerator ends.")
eprint() # To make autogen check output more readable.
diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py
index dd58547f33..7d0a29b0ad 100644
--- a/resources/libraries/python/autogen/Testcase.py
+++ b/resources/libraries/python/autogen/Testcase.py
@@ -34,29 +34,29 @@ class Testcase(object):
"""
self.template = Template(template_string)
- def generate(self, num, framesize, phy_cores):
+ def generate(self, num, frame_size, phy_cores):
"""Return string of test case code with placeholders filled.
Fail if there are placeholders left unfilled.
:param num: Test case number. Example value: 4.
- :param framesize: Imix string or numeric frame size. Example: 74.
+ :param frame_size: Imix string or numeric frame size. Example: 74.
:param phy_cores: Number of physical cores to use. Example: 2.
:type num: int
- :type framesize: str or int
+ :type frame_size: str or int
:type phy_cores: int or str
:returns: Filled template, usable as test case code.
:rtype: str
"""
try:
- fsize = int(framesize)
+ fsize = int(frame_size)
subst_dict = {
"frame_num": "${%d}" % fsize,
"frame_str": "%dB" % fsize
}
except ValueError: # Assuming an IMIX string.
subst_dict = {
- "frame_num": str(framesize),
+ "frame_num": str(frame_size),
"frame_str": "IMIX"
}
cores_str = str(phy_cores)