aboutsummaryrefslogtreecommitdiffstats
path: root/resources/libraries/python/autogen/Regenerator.py
diff options
context:
space:
mode:
authorVratko Polak <vrpolak@cisco.com>2022-04-25 10:22:05 +0200
committerVratko Polak <vrpolak@cisco.com>2022-05-03 15:23:47 +0000
commit1daa6fdc0bae284dee1b61f34534e59b60b7526a (patch)
tree6314e011a8f029c4332586d5fd05b51d1a69a89c /resources/libraries/python/autogen/Regenerator.py
parentcd417be7f836eb9346fad4f87bd4f75dc1d9a429 (diff)
feat(astf): Support framesizes for ASTF
- No support for IMIX. + Fix a bad bug in padding (most ASTF profiles had wrong frame sizes). + Fix a big typo in TCP PPS profiles (s->c was not data, just RST). + Control transaction size via ASTF_N_DATA_FRAMES env variable. - Default value 5 leads to transactions smaller than before. + It ensures transaction is one burst (per direction) even for jumbo. + Edit autogen to set supported frame sizes based on suite id. + Both TCP and UDP use the same values: + 64B for CPS (exact for UDP, nominal for TCP). + 100B, 1518B and 9000B for TPUT and PPS. - TCP TPUT achievable minimum is 70B. + Used 100B to leave room for possible IPv6 ASTF tests. + Separate function for code reused by vpp and trex tests. - I do not really like the new "copy and edit" approach added here. + But it is a quick edit, better autogen refactor is low priority. + Consider both established and transitory sessions as valid. - Mostly for compatibility with 2202 behavior and to avoid ramp-ups. - Assuming both session states have similar enough VPP CPU overhead. + Added a TODO to investigate and maybe reconsider later. + Update the state timeout value to 240s. + That is the default for TCP (for transitory state). - UDP could keep using 300s. + But I prefer UDP and TCP to behave as similarly as possible. + Use TRex tunables to get the exact frame size (for data packets). - It is not clear why the recipe for MSS has to be this complicated. + Move code away from profile init, as frame size is not known there. + Change internal profile API, so values related to MSS are passed. + Lower ramp-up rate for TCP TPUT tests. + Because without lower rate, jumbo fails on packet loss in ramp-up. + UDP TPUT ramp-up rate also lowered (just to keep suites more similar). + Distinguish one-direction and aggregated average frame size. + Update keyword documentation where the distiction matters. + One-direction is needed for turning bandwidth limit to TPS limit. + Aggregated is needed for correct NDRPDR bandwidth result value. - TCP TPUT will always be few percent below bidirectional maximum. + That is unavoidable, as one direction sends more control packets. + Add runtime consistency checks so future refactors are safer. + Fail if padding requested would be negative. + Fail if suite claims unexpected values for packets per transaction. + Edit the 4 types of ASTF profiles to keep them similar to each other. + Move UDP TPUT limit value from a field back to direct argument. + Stop pretending first UDP packet is not data. + Apply small improvements where convenient. + Replace "aggregate" with "aggregated" where possible. + To lower probability of any future typos in variable names. + Avoid calling Set Numeric Frame Sizes twice. + Code formatting, keyword documentation, code comments, ... + Add TODOs for less important code quality improvements. - Postpone updating of methodology pages to a subsequent change. Change-Id: I4b381e5210e69669f972326202fdcc5a2c9c923b Signed-off-by: Vratko Polak <vrpolak@cisco.com>
Diffstat (limited to 'resources/libraries/python/autogen/Regenerator.py')
-rw-r--r--resources/libraries/python/autogen/Regenerator.py64
1 files changed, 44 insertions, 20 deletions
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index 3011b06897..4474996ef1 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -17,6 +17,7 @@ TODO: How can we check each suite id is unique,
when currently the suite generation is run on each directory separately?
"""
+import copy
import sys
from glob import glob
@@ -116,6 +117,39 @@ def check_suite_tag(suite_tag, prolog):
raise ValueError(f"Suite tag found {found} times for {suite_tag}")
+def filter_and_edit_kwargs_for_astf(suite_id, kwargs):
+ """Return possibly edited kwargs, or None if to be skipped.
+
+ This is a code block used in few places.
+ Kwargs is (a copy of) one item from tc_kwargs_list.
+ Currently, the editable field is frame_size,
+ to be increased to for tests with data (not just CPS).
+
+ :param suite_id: Suite ID.
+ :param kwargs: Key-value pairs used to construct one testcase.
+ :type suite_id: str
+ :type tc_kwargs_list: dict
+ :returns: Edited kwargs.
+ :rtype Optional[dict]
+ """
+ if u"-cps-" in suite_id:
+ # Contrary to UDP, there is no place to affect frame size
+ # in TCP CPS tests. Actual frames are close to min size.
+ # UDP uses the min value too, for fairer comparison to TCP.
+ if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES:
+ return None
+ elif (u"-pps-" in suite_id or u"-tput-" in suite_id):
+ if u"imix" in str(kwargs[u"frame_size"]).lower():
+ # ASTF does not support IMIX (yet).
+ return None
+ if kwargs[u"frame_size"] in MIN_FRAME_SIZE_VALUES:
+ # Minimal (TRex) TCP data frame is 80B for IPv4.
+ # In future, we may want to have also IPv6 TCP.
+ # UDP uses the same value, for fairer comparison to TCP.
+ kwargs[u"frame_size"] = 100
+ return kwargs
+
+
def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list):
"""Add default testcases to file.
@@ -130,7 +164,9 @@ def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list):
:type file_out: file
:type tc_kwargs_list: dict
"""
- for kwargs in tc_kwargs_list:
+ for kwas in tc_kwargs_list:
+ # We may edit framesize for ASTF, the copy should be local.
+ kwargs = copy.deepcopy(kwas)
# TODO: Is there a better way to disable some combinations?
emit = True
if kwargs[u"frame_size"] == 9000:
@@ -156,14 +192,8 @@ def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list):
emit = False
if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES:
emit = False
- if (
- u"-cps-" in suite_id
- or u"-pps-" in suite_id
- or u"-tput-" in suite_id
- ):
- if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES:
- emit = False
- if emit:
+ kwargs = filter_and_edit_kwargs_for_astf(suite_id, kwargs)
+ if emit and kwargs is not None:
file_out.write(testcase.generate(**kwargs))
@@ -207,17 +237,11 @@ def add_trex_testcases(testcase, suite_id, file_out, tc_kwargs_list):
:type file_out: file
:type tc_kwargs_list: dict
"""
- for kwargs in tc_kwargs_list:
- # TODO: Is there a better way to disable some combinations?
- emit = True
- if (
- u"-cps-" in suite_id
- or u"-pps-" in suite_id
- or u"-tput-" in suite_id
- ):
- if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES:
- emit = False
- if emit:
+ for kwas in tc_kwargs_list:
+ # We may edit framesize for ASTF, the copy should be local.
+ kwargs = copy.deepcopy(kwas)
+ kwargs = filter_and_edit_kwargs_for_astf(suite_id, kwargs)
+ if kwargs is not None:
file_out.write(testcase.generate(**kwargs))