diff options
Diffstat (limited to 'resources')
| -rw-r--r-- | resources/libraries/python/HoststackUtil.py | 91 | ||||
| -rw-r--r-- | resources/libraries/python/autogen/Regenerator.py | 67 | ||||
| -rw-r--r-- | resources/libraries/python/autogen/Testcase.py | 31 | ||||
| -rw-r--r-- | resources/libraries/robot/hoststack/hoststack.robot | 28 | ||||
| -rw-r--r-- | resources/tools/wrk/wrk.py | 4 |
5 files changed, 88 insertions, 133 deletions
diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py index dde5cf60d0..d184b5af6d 100644 --- a/resources/libraries/python/HoststackUtil.py +++ b/resources/libraries/python/HoststackUtil.py @@ -12,6 +12,7 @@ # limitations under the License. """Host Stack util library.""" +import json from time import sleep from robot.api import logger @@ -249,10 +250,16 @@ class HoststackUtil(): sleep(1) @staticmethod - def analyze_hoststack_test_program_output(node, role, nsim_attr, - program): + def analyze_hoststack_test_program_output( + node, role, nsim_attr, program): """Gather HostStack test program output and check for errors. + The [defer_fail] return bool is used instead of failing immediately + to allow the analysis of both the client and server instances of + the test program for debugging a test failure. When [defer_fail] + is true, then the string returned is debug output instead of + JSON formatted test program results. + :param node: DUT node. :param role: Role (client|server) of test program. :param nsim_attr: Network Simulation Attributes. @@ -262,7 +269,8 @@ class HoststackUtil(): :type role: str :type nsim_attr: dict :type program: dict - :returns: tuple of no results bool and test program results. + :returns: tuple of [defer_fail] bool and either JSON formatted hoststack + test program output or failure debug output. :rtype: bool, str :raises RuntimeError: If node subtype is not a DUT. """ @@ -277,7 +285,6 @@ class HoststackUtil(): program_stdout, program_stderr = \ HoststackUtil.get_hoststack_test_program_logs(node, program) - no_results = False env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u"" program_cmd = f"{env_vars}{program_name} {program[u'args']}" test_results = f"Test Results of '{program_cmd}':\n" @@ -296,50 +303,50 @@ class HoststackUtil(): f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \ f"pkts/drop\n" - if u"error" in program_stderr.lower(): - test_results += f"ERROR DETECTED:\n{program_stderr}" - raise RuntimeError(test_results) - if program_stdout: - bad_test_results = False - if program[u"name"] == u"vpp_echo": - if u"JSON stats" in program_stdout: - test_results += program_stdout - # TODO: Decode vpp_echo output when JSON format is correct. - # json_start = program_stdout.find(u"{") - # vpp_echo_results = json.loads(program_stdout[json_start:]) - if u'"has_failed": "0"' not in program_stdout: - bad_test_results = True - else: - test_results += u"Invalid test data output!\n" + \ - program_stdout - bad_test_results = True - else: - test_results += program_stdout - if bad_test_results: - raise RuntimeError(test_results) - else: - no_results = True - test_results += f"\nNo {program} test data retrieved!\n" - cmd = u"ls -l /tmp/*.log" - ls_stdout, _ = exec_cmd_no_error(node, cmd, sudo=True) - test_results += f"{ls_stdout}\n" - # TODO: Incorporate show error stats into results analysis - host = node[u"host"] test_results += \ - f"\n{role} VPP 'show errors' on host {host}:\n" \ + f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \ f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n" - return no_results, test_results + if u"error" in program_stderr.lower(): + test_results += f"ERROR DETECTED:\n{program_stderr}" + return (True, test_results) + if not program_stdout: + test_results += f"\nNo {program} test data retrieved!\n" + ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log", + sudo=True) + test_results += f"{ls_stdout}\n" + return (True, test_results) + if program[u"name"] == u"vpp_echo": + if u"JSON stats" in program_stdout and \ + u'"has_failed": "0"' in program_stdout: + json_start = program_stdout.find(u"{") + #TODO: Fix parsing once vpp_echo produces valid + # JSON output. Truncate for now. + json_end = program_stdout.find(u',\n "closing"') + json_results = f"{program_stdout[json_start:json_end]}\n}}" + program_json = json.loads(json_results) + else: + test_results += u"Invalid test data output!\n" + program_stdout + return (True, test_results) + elif program[u"name"] == u"iperf3": + test_results += program_stdout + iperf3_json = json.loads(program_stdout) + program_json = iperf3_json[u"intervals"][0][u"sum"] + else: + test_results += u"Unknown HostStack Test Program!\n" + \ + program_stdout + return (True, program_stdout) + return (False, json.dumps(program_json)) @staticmethod - def no_hoststack_test_program_results(server_no_results, client_no_results): - """Return True if no HostStack test program output was gathered. + def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail): + """Return True if either HostStack test program fail was deferred. - :param server_no_results: server no results value. - :param client_no_results: client no results value. - :type server_no_results: bool - :type client_no_results: bool + :param server_defer_fail: server no results value. + :param client_defer_fail: client no results value. + :type server_defer_fail: bool + :type client_defer_fail: bool :rtype: bool """ - return server_no_results and client_no_results + return server_defer_fail and client_defer_fail diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py index 5cd8bcccb7..d40ae7c9fc 100644 --- a/resources/libraries/python/autogen/Regenerator.py +++ b/resources/libraries/python/autogen/Regenerator.py @@ -440,50 +440,24 @@ class Regenerator: min_frame_size = PROTOCOL_TO_MIN_FRAME_SIZE[protocol] default_kwargs_list = [ - {u"frame_size": min_frame_size, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": min_frame_size, u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": min_frame_size, u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 1518, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 1518, u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 1518, u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 9000, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 9000, u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 9000, u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"} + {u"frame_size": min_frame_size, u"phy_cores": 1}, + {u"frame_size": min_frame_size, u"phy_cores": 2}, + {u"frame_size": min_frame_size, u"phy_cores": 4}, + {u"frame_size": 1518, u"phy_cores": 1}, + {u"frame_size": 1518, u"phy_cores": 2}, + {u"frame_size": 1518, u"phy_cores": 4}, + {u"frame_size": 9000, u"phy_cores": 1}, + {u"frame_size": 9000, u"phy_cores": 2}, + {u"frame_size": 9000, u"phy_cores": 4}, + {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1}, + {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2}, + {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4} ] - hoststack_wrk_kwargs_list = [ - {u"frame_size": 0, u"phy_cores": i, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"} for i in (1, 2, 4) + hs_wrk_kwargs_list = [ + {u"frame_size": 0, u"phy_cores": i} for i in (1, 2, 4) ] - hoststack_iperf3_kwargs_list = [ - {u"frame_size": 0, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 0, u"phy_cores": 1, u"clients": 1, - u"streams": 10, u"bytes_str": u"1G"}, - ] - hoststack_quic_kwargs_list = [ - {u"phy_cores": 1, u"frame_size": 0, u"clients": 1, - u"streams": 1, u"bytes_str": u"10G"}, - {u"phy_cores": 1, u"frame_size": 0, u"clients": 1, - u"streams": 10, u"bytes_str": u"1G"}, - {u"phy_cores": 1, u"frame_size": 0, u"clients": 10, - u"streams": 1, u"bytes_str": u"1G"}, - {u"phy_cores": 1, u"frame_size": 0, u"clients": 10, - u"streams": 10, u"bytes_str": u"100M"}, + hs_bps_kwargs_list = [ + {u"frame_size": 0, u"phy_cores": 1}, ] for in_filename in glob(pattern): @@ -510,14 +484,9 @@ class Regenerator: elif in_filename.endswith(u"-reconf.robot"): write_reconf_files(in_filename, in_prolog, default_kwargs_list) elif in_filename[-10:] in (u"-cps.robot", u"-rps.robot"): - write_tcp_files(in_filename, in_prolog, - hoststack_wrk_kwargs_list) + write_tcp_files(in_filename, in_prolog, hs_wrk_kwargs_list) elif in_filename[-10:] in (u"-bps.robot"): - if u"ldpreload-iperf3" in in_filename: - hoststack_kwargs_list = hoststack_iperf3_kwargs_list - else: - hoststack_kwargs_list = hoststack_quic_kwargs_list - write_tcp_files(in_filename, in_prolog, hoststack_kwargs_list) + write_tcp_files(in_filename, in_prolog, hs_bps_kwargs_list) else: raise RuntimeError( f"Error in {in_filename}: non-primary suite type found." diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py index c540d8ccec..d419b2993b 100644 --- a/resources/libraries/python/autogen/Testcase.py +++ b/resources/libraries/python/autogen/Testcase.py @@ -34,8 +34,7 @@ class Testcase: """ self.template = Template(template_string) - def generate(self, num, frame_size, phy_cores, clients, streams, - bytes_str): + def generate(self, num, frame_size, phy_cores): """Return string of test case code with placeholders filled. Fail if there are placeholders left unfilled. @@ -44,16 +43,9 @@ class Testcase: :param num: Test case number. Example value: 4. :param frame_size: Imix string or numeric frame size. Example: 74. :param phy_cores: Number of physical cores to use. Example: 2. - :param clients: Number of clients used by test program. Example: 4. - :param streams: Number of streams used by test program. Example: 10. - :param bytes_str: Size in bytes of stream sent by test program. - Example: 1G :type num: int :type frame_size: str or int :type phy_cores: int or str - :type clients: int - :type streams: int - :type bytes_str: str :returns: Filled template, usable as test case code. :rtype: str """ @@ -75,11 +67,6 @@ class Testcase: u"cores_num": f"${{{cores_num:d}}}", u"cores_str": phy_cores, u"tc_num": f"tc{num:02d}", - u"clients_num": f"${{{clients:d}}}", - u"clients_str": str(clients), - u"streams_num": f"${{{streams:d}}}", - u"streams_str": str(streams), - u"bytes_str": bytes_str, } ) return self.template.substitute(subst_dict) @@ -122,17 +109,9 @@ class Testcase: | | [Tags] | ${{cores_str}}C | | phy_cores=${{cores_num}} ''' - elif u"iperf3" in suite_id: - template_string = f''' -| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id} -| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM -| | phy_cores=${{cores_num}} | clients=${{clients_num}}''' - template_string += f" | streams=${{streams_num}}\n" else: - template_string = f''' -| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id} -| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM -| | phy_cores=${{cores_num}} | clients=${{clients_num}}''' - template_string += f" | streams=${{streams_num}}" \ - f" | bytes=${{bytes_str}}\n" + template_string = \ + f"\n| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id[:-4]}" \ + f"-{suite_id[-3:]}\n" \ + f"| | [Tags] | ${{cores_str}}C\n| | phy_cores=${{cores_num}}\n" return cls(template_string) diff --git a/resources/libraries/robot/hoststack/hoststack.robot b/resources/libraries/robot/hoststack/hoststack.robot index 16a390aeb8..c6bc7dea06 100644 --- a/resources/libraries/robot/hoststack/hoststack.robot +++ b/resources/libraries/robot/hoststack/hoststack.robot @@ -91,7 +91,7 @@ | ... | vcl_config=vcl_iperf3.conf | ... | ld_preload=${True} | ... | transparent_tls=${False} -| ... | json=${False} +| ... | json=${True} | ... | ip_version=${4} | &{iperf3_client_attr}= | ... | role=client @@ -101,7 +101,7 @@ | ... | vcl_config=vcl_iperf3.conf | ... | ld_preload=${True} | ... | transparent_tls=${False} -| ... | json=${False} +| ... | json=${True} | ... | ip_version=${4} | ... | ip_address=${EMPTY} | ... | parallel=${1} @@ -464,9 +464,9 @@ | | [Documentation] | | ... | Configure IP address on the port, set it up and start the specified | | ... | HostStack test programs on the DUTs. Gather test program -| | ... | output and append test results in message. -| | ... | Return boolean indicating when no results were available from -| | ... | both the server and client test programs. +| | ... | output and append JSON formatted test data in message. +| | ... | Return boolean indicating there was a defered failure of either the +| | ... | server and/or client test programs. | | | | Set To Dictionary | ${vpp_echo_server_attr} | uri_ip4_addr | | ... | ${dut2_if1_ip4_addr} @@ -492,25 +492,25 @@ | | ... | ${vpp_echo_client_attr}[namespace] | ${core_list} | | ... | ${vpp_echo_client_attr}[cfg_vpp_feature] | ${vpp_echo_client} | | When Hoststack Test Program Finished | ${dut1} | ${client_pid} -| | ${client_no_results} | ${client_output}= +| | ${client_defer_fail} | ${client_output}= | | ... | Analyze hoststack test program output | ${dut1} | Client | | ... | ${vpp_nsim_attr} | ${vpp_echo_client} | | Then Set test message | ${client_output} | | And Hoststack Test Program Finished | ${dut2} | ${server_pid} -| | ${server_no_results} | ${server_output}= +| | ${server_defer_fail} | ${server_output}= | | ... | Analyze hoststack test program output | ${dut2} | Server | | ... | ${vpp_nsim_attr} | ${vpp_echo_server} | | Set test message | ${server_output} | append=True -| | Run Keyword And Return | No Hoststack Test Program Results -| | ... | ${server_no_results} | ${client_no_results} +| | Run Keyword And Return | Hoststack Test Program Defer Fail +| | ... | ${server_defer_fail} | ${client_defer_fail} | Get Test Results From Hoststack Iperf3 Test | | [Documentation] | | ... | Configure IP address on the port, set it up and start the specified | | ... | HostStack test programs on the DUTs. Gather test program -| | ... | output and append test results in message. -| | ... | Return boolean indicating when no results were available from -| | ... | both the server and client test programs. +| | ... | output and append JSON formatted test data in message. +| | ... | Return boolean indicating there was a defered failure of either the +| | ... | server and/or client test programs. | | | | Set To Dictionary | ${iperf3_client_attr} | ip_address | | ... | ${dut2_if1_ip4_addr} @@ -534,8 +534,8 @@ | | ... | ${iperf3_client_attr}[namespace] | ${core_list} | | ... | ${iperf3_client_attr}[cfg_vpp_feature] | ${iperf3_client} | | When Hoststack Test Program Finished | ${dut1} | ${client_pid} -| | ${client_no_results} | ${client_output}= +| | ${client_defer_fail} | ${client_output}= | | ... | Analyze hoststack test program output | ${dut1} | Client | | ... | ${vpp_nsim_attr} | ${iperf3_client} | | Then Set test message | ${client_output} -| | Return From Keyword | ${client_no_results} +| | Return From Keyword | ${client_defer_fail} diff --git a/resources/tools/wrk/wrk.py b/resources/tools/wrk/wrk.py index 381e9b9da0..85d18d4a9c 100644 --- a/resources/tools/wrk/wrk.py +++ b/resources/tools/wrk/wrk.py @@ -186,12 +186,12 @@ def run_wrk(tg_node, profile_name, tg_numa, test_type, warm_up=False): if test_type == u"cps": log_msg += u"Connections/sec: Avg / Stdev / Max / +/- Stdev\n" for item in stats[u"rps-stats-lst"]: - log_msg += f"{0} / {1} / {2} / {3}\n".format(*item) + log_msg += u" / ".join(map(str, item)) + u"\n" log_msg += f"Total cps: {stats[u'rps-sum']}cps\n" elif test_type == u"rps": log_msg += u"Requests/sec: Avg / Stdev / Max / +/- Stdev\n" for item in stats[u"rps-stats-lst"]: - log_msg += f"{0} / {1} / {2} / {3}\n".format(*item) + log_msg += u" / ".join(map(str, item)) + u"\n" log_msg += f"Total rps: {stats[u'rps-sum']}rps\n" elif test_type == u"bw": log_msg += f"Transfer/sec: {stats[u'bw-sum']}Bps" |