diff options
Diffstat (limited to 'resources/libraries/python/autogen')
| -rw-r--r-- | resources/libraries/python/autogen/DefaultTestcase.py | 35 | ||||
| -rw-r--r-- | resources/libraries/python/autogen/Regenerator.py | 440 | ||||
| -rw-r--r-- | resources/libraries/python/autogen/Testcase.py | 40 |
3 files changed, 270 insertions, 245 deletions
diff --git a/resources/libraries/python/autogen/DefaultTestcase.py b/resources/libraries/python/autogen/DefaultTestcase.py deleted file mode 100644 index 5db81a80d4..0000000000 --- a/resources/libraries/python/autogen/DefaultTestcase.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module with utilities for autogeneration of non-customizable testcases.""" - -from .Testcase import Testcase - - -class DefaultTestcase(Testcase): - """Testcase subclass with a rigid template string.""" - - def __init__(self, suite_id): - """Construct instance for identified suite. - - :param suite_id: Suite identifier, without NIC prefix and .robot suffix. - Example: ethip6srhip6-ip6base-srv6enc2sids-nodecaps-ndrpdr - :type suite_id: str - """ - self.suite_id = suite_id - template_string = r''' -| ${tc_num}-${frame_str}-${cores_str}c-''' + suite_id + r''' -| | [Tags] | ${frame_str} | ${cores_str}C -| | frame_size=${frame_num} | phy_cores=${cores_num} -''' - super(DefaultTestcase, self).__init__(template_string) diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py index 89acff73ae..a9f3ef433d 100644 --- a/resources/libraries/python/autogen/Regenerator.py +++ b/resources/libraries/python/autogen/Regenerator.py @@ -19,8 +19,17 @@ from glob import glob from os import getcwd import sys -from .DefaultTestcase import DefaultTestcase from resources.libraries.python.Constants import Constants +from resources.libraries.python.autogen.Testcase import Testcase + + +PROTOCOL_TO_MIN_FRAME_SIZE = { + "ip4": 64, + "ip6": 78, + "ethip4vxlan": 114, # What is the real minimum for latency stream? + "dot1qip4vxlan": 118 +} +MIN_FRAME_SIZE_VALUES = PROTOCOL_TO_MIN_FRAME_SIZE.values() # Copied from https://stackoverflow.com/a/14981125 @@ -29,227 +38,234 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) +def replace_defensively( + whole, to_replace, replace_with, how_many, msg, in_filename): + """Replace substrings while checking the number of occurences. + + Return edited copy of the text. Assuming "whole" is really a string, + or something else with .replace not affecting it. + + :param whole: The text to perform replacements on. + :param to_replace: Substring occurences of which to replace. + :param replace_with: Substring to replace occurences with. + :param how_many: Number of occurences to expect. + :param msg: Error message to raise. + :param in_filename: File name in which the error occured. + :type whole: str + :type to_replace: str + :type replace_with: str + :type how_many: int + :type msg: str + :type in_filename: str + :return: The whole text after replacements are done. + :rtype: str + :raise ValueError: If number of occurences does not match. + """ + found = whole.count(to_replace) + if found != how_many: + raise ValueError(in_filename + ": " + msg) + return whole.replace(to_replace, replace_with) + + +def get_iface_and_suite_id(filename): + """Get interface and suite ID. + + Interface ID is the part of suite name + which should be replaced for other NIC. + Suite ID is the part os suite name + which si appended to testcase names. + + :param filename: Suite file. + :type filename: str + :returns: Interface ID, Suite ID. + :rtype: (str, str) + """ + dash_split = filename.split("-", 1) + if len(dash_split[0]) <= 4: + # It was something like "2n1l", we need one more split. + dash_split = dash_split[1].split("-", 1) + return dash_split[0], dash_split[1].split(".", 1)[0] + + +def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list): + """Add default testcases to file. + + :param testcase: Testcase class. + :param iface: Interface. + :param suite_id: Suite ID. + :param file_out: File to write testcases to. + :param tc_kwargs_list: Key-value pairs used to construct testcases. + :type testcase: Testcase + :type iface: str + :type suite_id: str + :type file_out: file + :type tc_kwargs_list: dict + """ + # We bump tc number in any case, so that future enables/disables + # do not affect the numbering of other test cases. + for num, kwargs in enumerate(tc_kwargs_list, start=1): + # TODO: Is there a better way to disable some combinations? + emit = True + if kwargs["frame_size"] == 9000: + if "vic1227" in iface: + # Not supported in HW. + emit = False + if "vic1385" in iface: + # Not supported in HW. + emit = False + if "-16vm2t-" in suite_id or "-16dcr2t-" in suite_id: + if kwargs["phy_cores"] > 3: + # CSIT lab only has 28 (physical) core processors, + # so these test would fail when attempting to assign cores. + emit = False + if "-24vm1t-" in suite_id or "-24dcr1t-" in suite_id: + if kwargs["phy_cores"] > 3: + # CSIT lab only has 28 (physical) core processors, + # so these test would fail when attempting to assign cores. + emit = False + if "soak" in suite_id: + # Soak test take too long, do not risk other than tc01. + if kwargs["phy_cores"] != 1: + emit = False + if kwargs["frame_size"] not in MIN_FRAME_SIZE_VALUES: + emit = False + if emit: + file_out.write(testcase.generate(num=num, **kwargs)) + + +def add_tcp_testcases(testcase, file_out, tc_kwargs_list): + """Add TCP testcases to file. + + :param testcase: Testcase class. + :param file_out: File to write testcases to. + :param tc_kwargs_list: Key-value pairs used to construct testcases. + :type testcase: Testcase + :type file_out: file + :type tc_kwargs_list: dict + """ + for num, kwargs in enumerate(tc_kwargs_list, start=1): + file_out.write(testcase.generate(num=num, **kwargs)) + + +def write_default_files(in_filename, in_prolog, kwargs_list): + """Using given filename and prolog, write all generated suites. + + :param in_filename: Template filename to derive real filenames from. + :param in_prolog: Template content to derive real content from. + :param kwargs_list: List of kwargs for add_default_testcase. + :type in_filename: str + :type in_prolog: str + :type kwargs_list: list of dict + """ + for suite_type in Constants.PERF_TYPE_TO_KEYWORD: + tmp_filename = replace_defensively( + in_filename, "ndrpdr", suite_type, 1, + "File name should contain suite type once.", in_filename) + tmp_prolog = replace_defensively( + in_prolog, "ndrpdr".upper(), suite_type.upper(), 1, + "Suite type should appear once in uppercase (as tag).", + in_filename) + tmp_prolog = replace_defensively( + tmp_prolog, + "Find NDR and PDR intervals using optimized search", + Constants.PERF_TYPE_TO_KEYWORD[suite_type], 1, + "Main search keyword should appear once in suite.", + in_filename) + tmp_prolog = replace_defensively( + tmp_prolog, + Constants.PERF_TYPE_TO_SUITE_DOC_VER["ndrpdr"], + Constants.PERF_TYPE_TO_SUITE_DOC_VER[suite_type], + 1, "Exact suite type doc not found.", in_filename) + tmp_prolog = replace_defensively( + tmp_prolog, + Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER["ndrpdr"], + Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER[suite_type], + 1, "Exact template type doc not found.", in_filename) + _, suite_id = get_iface_and_suite_id(tmp_filename) + testcase = Testcase.default(suite_id) + for nic_name in Constants.NIC_NAME_TO_CODE: + out_filename = replace_defensively( + tmp_filename, "10ge2p1x710", + Constants.NIC_NAME_TO_CODE[nic_name], 1, + "File name should contain NIC code once.", in_filename) + out_prolog = replace_defensively( + tmp_prolog, "Intel-X710", nic_name, 2, + "NIC name should appear twice (tag and variable).", + in_filename) + if out_prolog.count("HW_") == 2: + # TODO CSIT-1481: Crypto HW should be read + # from topology file instead. + if nic_name in Constants.NIC_NAME_TO_CRYPTO_HW: + out_prolog = replace_defensively( + out_prolog, "HW_DH895xcc", + Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1, + "HW crypto name should appear.", in_filename) + iface, suite_id = get_iface_and_suite_id(out_filename) + with open(out_filename, "w") as file_out: + file_out.write(out_prolog) + add_default_testcases( + testcase, iface, suite_id, file_out, kwargs_list) + + +def write_tcp_files(in_filename, in_prolog, kwargs_list): + """Using given filename and prolog, write all generated suites. + + :param in_filename: Template filename to derive real filenames from. + :param in_prolog: Template content to derive real content from. + :param kwargs_list: List of kwargs for add_default_testcase. + :type in_filename: str + :type in_prolog: str + :type kwargs_list: list of dict + """ + # TODO: Generate rps from cps? There are subtle differences. + _, suite_id = get_iface_and_suite_id(in_filename) + testcase = Testcase.tcp(suite_id) + for nic_name in Constants.NIC_NAME_TO_CODE: + out_filename = replace_defensively( + in_filename, "10ge2p1x710", + Constants.NIC_NAME_TO_CODE[nic_name], 1, + "File name should contain NIC code once.", in_filename) + out_prolog = replace_defensively( + in_prolog, "Intel-X710", nic_name, 2, + "NIC name should appear twice (tag and variable).", + in_filename) + with open(out_filename, "w") as file_out: + file_out.write(out_prolog) + add_tcp_testcases(testcase, file_out, kwargs_list) + + class Regenerator(object): """Class containing file generating methods.""" - def __init__(self, testcase_class=DefaultTestcase, quiet=True): - """Initialize Testcase class to use. + def __init__(self, quiet=True): + """Initialize the instance. - TODO: See the type doc for testcase_class? - It implies the design is wrong. Fix it. - Easiest: Hardcode Regenerator to use DefaultTestcase only. - - :param testcase_class: Subclass of DefaultTestcase for generation. - Default: DefaultTestcase :param quiet: Reduce log prints (to stderr) when True (default). - :type testcase_class: subclass of DefaultTestcase accepting suite_id :type quiet: boolean """ - self.testcase_class = testcase_class self.quiet = quiet - def regenerate_glob(self, pattern, protocol="ip4", tc_kwargs_list=None): + def regenerate_glob(self, pattern, protocol="ip4"): """Regenerate files matching glob pattern based on arguments. In the current working directory, find all files matching - the glob pattern. Use testcase template (from init) to regenerate - test cases, autonumbering them, taking arguments from list. - If the list is None, use default list, which depends on ip6 usage. + the glob pattern. Use testcase template according to suffix + to regenerate test cases, autonumbering them, + taking arguments from list. Log-like prints are emited to sys.stderr. :param pattern: Glob pattern to select files. Example: *-ndrpdr.robot - :param is_ip6: Flag determining minimal frame size. Default: False - :param tc_kwargs_list: Arguments defining the testcases. Default: None - When None, default list is used. - List item is a dict, argument names are keys. - The initialized testcase_class should accept those, and "num". - DefaultTestcase accepts "frame_size" and "phy_cores". + :param protocol: String determining minimal frame size. Default: "ip4" :type pattern: str - :type is_ip6: boolean - :type tc_kwargs_list: list of tuple or None + :type protocol: str + :raises RuntimeError: If invalid source suite is encountered. """ - - protocol_to_min_frame_size = { - "ip4": 64, - "ip6": 78, - "ethip4vxlan": 114, # What is the real minimum for latency stream? - "dot1qip4vxlan": 118 - } - min_frame_size_values = protocol_to_min_frame_size.values() - - def get_iface_and_suite_id(filename): - """Get interface and suite ID. - - :param filename: Suite file. - :type filename: str - :returns: Interface ID, Suite ID. - :rtype: tuple - """ - dash_split = filename.split("-", 1) - if len(dash_split[0]) <= 4: - # It was something like "2n1l", we need one more split. - dash_split = dash_split[1].split("-", 1) - return dash_split[0], dash_split[1].split(".", 1)[0] - - def add_testcase(testcase, iface, suite_id, file_out, num, **kwargs): - """Add testcase to file. - - :param testcase: Testcase class. - :param iface: Interface. - :param suite_id: Suite ID. - :param file_out: File to write testcases to. - :param num: Testcase number. - :param kwargs: Key-value pairs used to construct testcase. - :type testcase: Testcase - :type iface: str - :type suite_id: str - :type file_out: file - :type num: int - :type kwargs: dict - :returns: Next testcase number. - :rtype: int - """ - # TODO: Is there a better way to disable some combinations? - emit = True - if kwargs["frame_size"] == 9000: - if "vic1227" in iface: - # Not supported in HW. - emit = False - if "vic1385" in iface: - # Not supported in HW. - emit = False - if "-16vm2t-" in suite_id or "-16dcr2t-" in suite_id: - if kwargs["phy_cores"] > 3: - # CSIT lab only has 28 (physical) core processors, - # so these test would fail when attempting to assign cores. - emit = False - if "-24vm1t-" in suite_id or "-24dcr1t-" in suite_id: - if kwargs["phy_cores"] > 3: - # CSIT lab only has 28 (physical) core processors, - # so these test would fail when attempting to assign cores. - emit = False - if "soak" in suite_id: - # Soak test take too long, do not risk other than tc01. - if kwargs["phy_cores"] != 1: - emit = False - if kwargs["frame_size"] not in min_frame_size_values: - emit = False - if emit: - file_out.write(testcase.generate(num=num, **kwargs)) - # We bump tc number in any case, so that future enables/disables - # do not affect the numbering of other test cases. - return num + 1 - - def add_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list): - """Add testcases to file. - - :param testcase: Testcase class. - :param iface: Interface. - :param suite_id: Suite ID. - :param file_out: File to write testcases to. - :param tc_kwargs_list: Key-value pairs used to construct testcases. - :type testcase: Testcase - :type iface: str - :type suite_id: str - :type file_out: file - :type tc_kwargs_list: dict - """ - num = 1 - for tc_kwargs in tc_kwargs_list: - num = add_testcase( - testcase, iface, suite_id, file_out, num, **tc_kwargs) - - def replace_defensively( - whole, to_replace, replace_with, how_many, msg, in_filename): - """Replace substrings while checking the number of occurences. - - Return edited copy of the text (so original string is not affected). - - :param whole: The text to perform replacements on. - :param to_replace: Substring occurences of which to replace. - :param replace_with: Substring to replace occurences with. - :param how_many: Number of occurences to expect. - :param msg: Error message to raise. - :param in_filename: File name in which the error occured. - :type whole: str - :type to_replace: str - :type replace_with: str - :type how_many: int - :type msg: str - :type in_filename: str - :return: The whole text after replacements are done. - :rtype: str - :raise ValueError: If number of occurences does not match. - """ - found = whole.count(to_replace) - if found != how_many: - raise ValueError(in_filename + ": " + msg) - return whole.replace(to_replace, replace_with) - - def write_files(in_filename, in_prolog, kwargs_list): - """Using given filename and prolog, write all generated suites. - - :param in_filename: Template filename to derive real filenames from. - :param in_prolog: Template content to derive real content from. - :param kwargs_list: List of kwargs for add_testcase. - :type in_filename: str - :type in_prolog: str - :type kwargs_list: list of dict - """ - for suite_type in Constants.PERF_TYPE_TO_KEYWORD.keys(): - tmp_filename = replace_defensively( - in_filename, "ndrpdr", suite_type, 1, - "File name should contain suite type once.", in_filename) - tmp_prolog = replace_defensively( - in_prolog, "ndrpdr".upper(), suite_type.upper(), 1, - "Suite type should appear once in uppercase (as tag).", - in_filename) - tmp_prolog = replace_defensively( - tmp_prolog, - "Find NDR and PDR intervals using optimized search", - Constants.PERF_TYPE_TO_KEYWORD[suite_type], 1, - "Main search keyword should appear once in suite.", - in_filename) - tmp_prolog = replace_defensively( - tmp_prolog, - Constants.PERF_TYPE_TO_SUITE_DOC_VER["ndrpdr"], - Constants.PERF_TYPE_TO_SUITE_DOC_VER[suite_type], - 1, "Exact suite type doc not found.", in_filename) - tmp_prolog = replace_defensively( - tmp_prolog, - Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER["ndrpdr"], - Constants.PERF_TYPE_TO_TEMPLATE_DOC_VER[suite_type], - 1, "Exact template type doc not found.", in_filename) - _, suite_id = get_iface_and_suite_id(tmp_filename) - testcase = self.testcase_class(suite_id) - for nic_name in Constants.NIC_NAME_TO_CODE.keys(): - out_filename = replace_defensively( - tmp_filename, "10ge2p1x710", - Constants.NIC_NAME_TO_CODE[nic_name], 1, - "File name should contain NIC code once.", in_filename) - out_prolog = replace_defensively( - tmp_prolog, "Intel-X710", nic_name, 2, - "NIC name should appear twice (tag and variable).", - in_filename) - if out_prolog.count("HW_") == 2: - # TODO CSIT-1481: Crypto HW should be read - # from topology file instead. - if nic_name in Constants.NIC_NAME_TO_CRYPTO_HW.keys(): - out_prolog = replace_defensively( - out_prolog, "HW_DH895xcc", - Constants.NIC_NAME_TO_CRYPTO_HW[nic_name], 1, - "HW crypto name should appear.", in_filename) - iface, suite_id = get_iface_and_suite_id(out_filename) - with open(out_filename, "w") as file_out: - file_out.write(out_prolog) - add_testcases( - testcase, iface, suite_id, file_out, kwargs_list) - if not self.quiet: eprint("Regenerator starts at {cwd}".format(cwd=getcwd())) - min_frame_size = protocol_to_min_frame_size[protocol] - kwargs_list = tc_kwargs_list if tc_kwargs_list else [ + + min_frame_size = PROTOCOL_TO_MIN_FRAME_SIZE[protocol] + default_kwargs_list = [ {"frame_size": min_frame_size, "phy_cores": 1}, {"frame_size": min_frame_size, "phy_cores": 2}, {"frame_size": min_frame_size, "phy_cores": 4}, @@ -263,22 +279,26 @@ class Regenerator(object): {"frame_size": "IMIX_v4_1", "phy_cores": 2}, {"frame_size": "IMIX_v4_1", "phy_cores": 4} ] + tcp_kwargs_list = [{"phy_cores": i, "frame_size": 0} for i in (1, 2, 4)] for in_filename in glob(pattern): if not self.quiet: eprint("Regenerating in_filename:", in_filename) - if not in_filename.endswith("ndrpdr.robot"): - eprint("Error in {fil}: non-primary suite type encountered." - .format(fil=in_filename)) - sys.exit(1) iface, _ = get_iface_and_suite_id(in_filename) if not iface.endswith("10ge2p1x710"): - eprint("Error in {fil}: non-primary NIC encountered." - .format(fil=in_filename)) - sys.exit(1) + raise RuntimeError( + "Error in {fil}: non-primary NIC found.".format( + fil=in_filename)) with open(in_filename, "r") as file_in: in_prolog = "".join( file_in.read().partition("*** Test Cases ***")[:-1]) - write_files(in_filename, in_prolog, kwargs_list) + if in_filename.endswith("-ndrpdr.robot"): + write_default_files(in_filename, in_prolog, default_kwargs_list) + elif in_filename[-10:] in ("-cps.robot", "-rps.robot"): + write_tcp_files(in_filename, in_prolog, tcp_kwargs_list) + else: + raise RuntimeError( + "Error in {fil}: non-primary suite type found.".format( + fil=in_filename)) if not self.quiet: eprint("Regenerator ends.") eprint() # To make autogen check output more readable. diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py index 7d0a29b0ad..70c212211c 100644 --- a/resources/libraries/python/autogen/Testcase.py +++ b/resources/libraries/python/autogen/Testcase.py @@ -38,6 +38,7 @@ class Testcase(object): """Return string of test case code with placeholders filled. Fail if there are placeholders left unfilled. + It is not required for all placeholders to be present in template. :param num: Test case number. Example value: 4. :param frame_size: Imix string or numeric frame size. Example: 74. @@ -68,3 +69,42 @@ class Testcase(object): "tc_num": "tc{num:02d}".format(num=num) }) return self.template.substitute(subst_dict) + + @classmethod + def default(cls, suite_id): + """Factory method for creating "default" testcase objects. + + Testcase name will contain both frame size and core count. + Used for most performance tests, except TCP ones. + + :param suite_id: Part of suite name to distinguish from other suites. + :type suite_id: str + :returns: Instance for generating testcase text of this type. + :rtype: Testcase + """ + template_string = r''' +| ${tc_num}-${frame_str}-${cores_str}c-''' + suite_id + r''' +| | [Tags] | ${frame_str} | ${cores_str}C +| | frame_size=${frame_num} | phy_cores=${cores_num} +''' + return cls(template_string) + + @classmethod + def tcp(cls, suite_id): + """Factory method for creating "tcp" testcase objects. + + Testcase name will contain core count, but not frame size. + + :param suite_id: Part of suite name to distinguish from other suites. + :type suite_id: str + :returns: Instance for generating testcase text of this type. + :rtype: Testcase + """ + # TODO: Choose a better frame size identifier for streamed protocols + # (TCP, QUIC, SCTP, ...) where DUT (not TG) decides frame size. + template_string = r''' +| ${tc_num}-IMIX-${cores_str}c-''' + suite_id + r''' +| | [Tags] | ${cores_str}C +| | phy_cores=${cores_num} +''' + return cls(template_string) |