From 1daa6fdc0bae284dee1b61f34534e59b60b7526a Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Mon, 25 Apr 2022 10:22:05 +0200 Subject: feat(astf): Support framesizes for ASTF - No support for IMIX. + Fix a bad bug in padding (most ASTF profiles had wrong frame sizes). + Fix a big typo in TCP PPS profiles (s->c was not data, just RST). + Control transaction size via ASTF_N_DATA_FRAMES env variable. - Default value 5 leads to transactions smaller than before. + It ensures transaction is one burst (per direction) even for jumbo. + Edit autogen to set supported frame sizes based on suite id. + Both TCP and UDP use the same values: + 64B for CPS (exact for UDP, nominal for TCP). + 100B, 1518B and 9000B for TPUT and PPS. - TCP TPUT achievable minimum is 70B. + Used 100B to leave room for possible IPv6 ASTF tests. + Separate function for code reused by vpp and trex tests. - I do not really like the new "copy and edit" approach added here. + But it is a quick edit, better autogen refactor is low priority. + Consider both established and transitory sessions as valid. - Mostly for compatibility with 2202 behavior and to avoid ramp-ups. - Assuming both session states have similar enough VPP CPU overhead. + Added a TODO to investigate and maybe reconsider later. + Update the state timeout value to 240s. + That is the default for TCP (for transitory state). - UDP could keep using 300s. + But I prefer UDP and TCP to behave as similarly as possible. + Use TRex tunables to get the exact frame size (for data packets). - It is not clear why the recipe for MSS has to be this complicated. + Move code away from profile init, as frame size is not known there. + Change internal profile API, so values related to MSS are passed. + Lower ramp-up rate for TCP TPUT tests. + Because without lower rate, jumbo fails on packet loss in ramp-up. + UDP TPUT ramp-up rate also lowered (just to keep suites more similar). + Distinguish one-direction and aggregated average frame size. + Update keyword documentation where the distiction matters. + One-direction is needed for turning bandwidth limit to TPS limit. + Aggregated is needed for correct NDRPDR bandwidth result value. - TCP TPUT will always be few percent below bidirectional maximum. + That is unavoidable, as one direction sends more control packets. + Add runtime consistency checks so future refactors are safer. + Fail if padding requested would be negative. + Fail if suite claims unexpected values for packets per transaction. + Edit the 4 types of ASTF profiles to keep them similar to each other. + Move UDP TPUT limit value from a field back to direct argument. + Stop pretending first UDP packet is not data. + Apply small improvements where convenient. + Replace "aggregate" with "aggregated" where possible. + To lower probability of any future typos in variable names. + Avoid calling Set Numeric Frame Sizes twice. + Code formatting, keyword documentation, code comments, ... + Add TODOs for less important code quality improvements. - Postpone updating of methodology pages to a subsequent change. Change-Id: I4b381e5210e69669f972326202fdcc5a2c9c923b Signed-off-by: Vratko Polak --- resources/libraries/python/autogen/Regenerator.py | 64 ++++++++++++++++------- 1 file changed, 44 insertions(+), 20 deletions(-) (limited to 'resources/libraries/python/autogen/Regenerator.py') diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py index 3011b06897..4474996ef1 100644 --- a/resources/libraries/python/autogen/Regenerator.py +++ b/resources/libraries/python/autogen/Regenerator.py @@ -17,6 +17,7 @@ TODO: How can we check each suite id is unique, when currently the suite generation is run on each directory separately? """ +import copy import sys from glob import glob @@ -116,6 +117,39 @@ def check_suite_tag(suite_tag, prolog): raise ValueError(f"Suite tag found {found} times for {suite_tag}") +def filter_and_edit_kwargs_for_astf(suite_id, kwargs): + """Return possibly edited kwargs, or None if to be skipped. + + This is a code block used in few places. + Kwargs is (a copy of) one item from tc_kwargs_list. + Currently, the editable field is frame_size, + to be increased to for tests with data (not just CPS). + + :param suite_id: Suite ID. + :param kwargs: Key-value pairs used to construct one testcase. + :type suite_id: str + :type tc_kwargs_list: dict + :returns: Edited kwargs. + :rtype Optional[dict] + """ + if u"-cps-" in suite_id: + # Contrary to UDP, there is no place to affect frame size + # in TCP CPS tests. Actual frames are close to min size. + # UDP uses the min value too, for fairer comparison to TCP. + if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES: + return None + elif (u"-pps-" in suite_id or u"-tput-" in suite_id): + if u"imix" in str(kwargs[u"frame_size"]).lower(): + # ASTF does not support IMIX (yet). + return None + if kwargs[u"frame_size"] in MIN_FRAME_SIZE_VALUES: + # Minimal (TRex) TCP data frame is 80B for IPv4. + # In future, we may want to have also IPv6 TCP. + # UDP uses the same value, for fairer comparison to TCP. + kwargs[u"frame_size"] = 100 + return kwargs + + def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list): """Add default testcases to file. @@ -130,7 +164,9 @@ def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list): :type file_out: file :type tc_kwargs_list: dict """ - for kwargs in tc_kwargs_list: + for kwas in tc_kwargs_list: + # We may edit framesize for ASTF, the copy should be local. + kwargs = copy.deepcopy(kwas) # TODO: Is there a better way to disable some combinations? emit = True if kwargs[u"frame_size"] == 9000: @@ -156,14 +192,8 @@ def add_default_testcases(testcase, iface, suite_id, file_out, tc_kwargs_list): emit = False if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES: emit = False - if ( - u"-cps-" in suite_id - or u"-pps-" in suite_id - or u"-tput-" in suite_id - ): - if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES: - emit = False - if emit: + kwargs = filter_and_edit_kwargs_for_astf(suite_id, kwargs) + if emit and kwargs is not None: file_out.write(testcase.generate(**kwargs)) @@ -207,17 +237,11 @@ def add_trex_testcases(testcase, suite_id, file_out, tc_kwargs_list): :type file_out: file :type tc_kwargs_list: dict """ - for kwargs in tc_kwargs_list: - # TODO: Is there a better way to disable some combinations? - emit = True - if ( - u"-cps-" in suite_id - or u"-pps-" in suite_id - or u"-tput-" in suite_id - ): - if kwargs[u"frame_size"] not in MIN_FRAME_SIZE_VALUES: - emit = False - if emit: + for kwas in tc_kwargs_list: + # We may edit framesize for ASTF, the copy should be local. + kwargs = copy.deepcopy(kwas) + kwargs = filter_and_edit_kwargs_for_astf(suite_id, kwargs) + if kwargs is not None: file_out.write(testcase.generate(**kwargs)) -- cgit 1.2.3-korg