diff options
15 files changed, 435 insertions, 179 deletions
diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py index fe2034a680..a03a5a66df 100644 --- a/resources/libraries/python/HoststackUtil.py +++ b/resources/libraries/python/HoststackUtil.py @@ -12,6 +12,7 @@ # limitations under the License. """Host Stack util library.""" +import json from time import sleep from robot.api import logger @@ -249,10 +250,16 @@ class HoststackUtil(): sleep(1) @staticmethod - def analyze_hoststack_test_program_output(node, role, nsim_attr, - program): + def analyze_hoststack_test_program_output( + node, role, nsim_attr, program): """Gather HostStack test program output and check for errors. + The [defer_fail] return bool is used instead of failing immediately + to allow the analysis of both the client and server instances of + the test program for debugging a test failure. When [defer_fail] + is true, then the string returned is debug output instead of + JSON formatted test program results. + :param node: DUT node. :param role: Role (client|server) of test program. :param nsim_attr: Network Simulation Attributes. @@ -262,7 +269,8 @@ class HoststackUtil(): :type role: str :type nsim_attr: dict :type program: dict - :returns: tuple of no results bool and test program results. + :returns: tuple of [defer_fail] bool and either JSON formatted hoststack + test program output or failure debug output. :rtype: bool, str :raises RuntimeError: If node subtype is not a DUT. """ @@ -277,7 +285,6 @@ class HoststackUtil(): program_stdout, program_stderr = \ HoststackUtil.get_hoststack_test_program_logs(node, program) - no_results = False env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u"" program_cmd = f"{env_vars}{program_name} {program[u'args']}" test_results = f"Test Results of '{program_cmd}':\n" @@ -296,50 +303,50 @@ class HoststackUtil(): f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \ f"pkts/drop\n" - if u"error" in program_stderr.lower(): - test_results += f"ERROR DETECTED:\n{program_stderr}" - raise RuntimeError(test_results) - if program_stdout: - bad_test_results = False - if program[u"name"] == u"vpp_echo": - if u"JSON stats" in program_stdout: - test_results += program_stdout - # TODO: Decode vpp_echo output when JSON format is correct. - # json_start = program_stdout.find(u"{") - # vpp_echo_results = json.loads(program_stdout[json_start:]) - if u'"has_failed": "0"' not in program_stdout: - bad_test_results = True - else: - test_results += u"Invalid test data output!\n" + \ - program_stdout - bad_test_results = True - else: - test_results += program_stdout - if bad_test_results: - raise RuntimeError(test_results) - else: - no_results = True - test_results += f"\nNo {program} test data retrieved!\n" - cmd = u"ls -l /tmp/*.log" - ls_stdout, _ = exec_cmd_no_error(node, cmd, sudo=True) - test_results += f"{ls_stdout}\n" - # TODO: Incorporate show error stats into results analysis - host = node[u"host"] test_results += \ - f"\n{role} VPP 'show errors' on host {host}:\n" \ + f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \ f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n" - return no_results, test_results + if u"error" in program_stderr.lower(): + test_results += f"ERROR DETECTED:\n{program_stderr}" + return (True, test_results) + if not program_stdout: + test_results += f"\nNo {program} test data retrieved!\n" + ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log", + sudo=True) + test_results += f"{ls_stdout}\n" + return (True, test_results) + if program[u"name"] == u"vpp_echo": + if u"JSON stats" in program_stdout and \ + u'"has_failed": "0"' in program_stdout: + json_start = program_stdout.find(u"{") + #TODO: Fix parsing once vpp_echo produces valid + # JSON output. Truncate for now. + json_end = program_stdout.find(u',\n "closing"') + json_results = f"{program_stdout[json_start:json_end]}\n}}" + program_json = json.loads(json_results) + else: + test_results += u"Invalid test data output!\n" + program_stdout + return (True, test_results) + elif program[u"name"] == u"iperf3": + test_results += program_stdout + iperf3_json = json.loads(program_stdout) + program_json = iperf3_json[u"intervals"][0][u"sum"] + else: + test_results += u"Unknown HostStack Test Program!\n" + \ + program_stdout + return (True, program_stdout) + return (False, json.dumps(program_json)) @staticmethod - def no_hoststack_test_program_results(server_no_results, client_no_results): - """Return True if no HostStack test program output was gathered. + def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail): + """Return True if either HostStack test program fail was deferred. - :param server_no_results: server no results value. - :param client_no_results: client no results value. - :type server_no_results: bool - :type client_no_results: bool + :param server_defer_fail: server no results value. + :param client_defer_fail: client no results value. + :type server_defer_fail: bool + :type client_defer_fail: bool :rtype: bool """ - return server_no_results and client_no_results + return server_defer_fail and client_defer_fail diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py index 5cd8bcccb7..d40ae7c9fc 100644 --- a/resources/libraries/python/autogen/Regenerator.py +++ b/resources/libraries/python/autogen/Regenerator.py @@ -440,50 +440,24 @@ class Regenerator: min_frame_size = PROTOCOL_TO_MIN_FRAME_SIZE[protocol] default_kwargs_list = [ - {u"frame_size": min_frame_size, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": min_frame_size, u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": min_frame_size, u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 1518, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 1518, u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 1518, u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 9000, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 9000, u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 9000, u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"} + {u"frame_size": min_frame_size, u"phy_cores": 1}, + {u"frame_size": min_frame_size, u"phy_cores": 2}, + {u"frame_size": min_frame_size, u"phy_cores": 4}, + {u"frame_size": 1518, u"phy_cores": 1}, + {u"frame_size": 1518, u"phy_cores": 2}, + {u"frame_size": 1518, u"phy_cores": 4}, + {u"frame_size": 9000, u"phy_cores": 1}, + {u"frame_size": 9000, u"phy_cores": 2}, + {u"frame_size": 9000, u"phy_cores": 4}, + {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1}, + {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2}, + {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4} ] - hoststack_wrk_kwargs_list = [ - {u"frame_size": 0, u"phy_cores": i, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"} for i in (1, 2, 4) + hs_wrk_kwargs_list = [ + {u"frame_size": 0, u"phy_cores": i} for i in (1, 2, 4) ] - hoststack_iperf3_kwargs_list = [ - {u"frame_size": 0, u"phy_cores": 1, u"clients": 1, - u"streams": 1, u"bytes_str": u"1G"}, - {u"frame_size": 0, u"phy_cores": 1, u"clients": 1, - u"streams": 10, u"bytes_str": u"1G"}, - ] - hoststack_quic_kwargs_list = [ - {u"phy_cores": 1, u"frame_size": 0, u"clients": 1, - u"streams": 1, u"bytes_str": u"10G"}, - {u"phy_cores": 1, u"frame_size": 0, u"clients": 1, - u"streams": 10, u"bytes_str": u"1G"}, - {u"phy_cores": 1, u"frame_size": 0, u"clients": 10, - u"streams": 1, u"bytes_str": u"1G"}, - {u"phy_cores": 1, u"frame_size": 0, u"clients": 10, - u"streams": 10, u"bytes_str": u"100M"}, + hs_bps_kwargs_list = [ + {u"frame_size": 0, u"phy_cores": 1}, ] for in_filename in glob(pattern): @@ -510,14 +484,9 @@ class Regenerator: elif in_filename.endswith(u"-reconf.robot"): write_reconf_files(in_filename, in_prolog, default_kwargs_list) elif in_filename[-10:] in (u"-cps.robot", u"-rps.robot"): - write_tcp_files(in_filename, in_prolog, - hoststack_wrk_kwargs_list) + write_tcp_files(in_filename, in_prolog, hs_wrk_kwargs_list) elif in_filename[-10:] in (u"-bps.robot"): - if u"ldpreload-iperf3" in in_filename: - hoststack_kwargs_list = hoststack_iperf3_kwargs_list - else: - hoststack_kwargs_list = hoststack_quic_kwargs_list - write_tcp_files(in_filename, in_prolog, hoststack_kwargs_list) + write_tcp_files(in_filename, in_prolog, hs_bps_kwargs_list) else: raise RuntimeError( f"Error in {in_filename}: non-primary suite type found." diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py index c540d8ccec..d419b2993b 100644 --- a/resources/libraries/python/autogen/Testcase.py +++ b/resources/libraries/python/autogen/Testcase.py @@ -34,8 +34,7 @@ class Testcase: """ self.template = Template(template_string) - def generate(self, num, frame_size, phy_cores, clients, streams, - bytes_str): + def generate(self, num, frame_size, phy_cores): """Return string of test case code with placeholders filled. Fail if there are placeholders left unfilled. @@ -44,16 +43,9 @@ class Testcase: :param num: Test case number. Example value: 4. :param frame_size: Imix string or numeric frame size. Example: 74. :param phy_cores: Number of physical cores to use. Example: 2. - :param clients: Number of clients used by test program. Example: 4. - :param streams: Number of streams used by test program. Example: 10. - :param bytes_str: Size in bytes of stream sent by test program. - Example: 1G :type num: int :type frame_size: str or int :type phy_cores: int or str - :type clients: int - :type streams: int - :type bytes_str: str :returns: Filled template, usable as test case code. :rtype: str """ @@ -75,11 +67,6 @@ class Testcase: u"cores_num": f"${{{cores_num:d}}}", u"cores_str": phy_cores, u"tc_num": f"tc{num:02d}", - u"clients_num": f"${{{clients:d}}}", - u"clients_str": str(clients), - u"streams_num": f"${{{streams:d}}}", - u"streams_str": str(streams), - u"bytes_str": bytes_str, } ) return self.template.substitute(subst_dict) @@ -122,17 +109,9 @@ class Testcase: | | [Tags] | ${{cores_str}}C | | phy_cores=${{cores_num}} ''' - elif u"iperf3" in suite_id: - template_string = f''' -| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id} -| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM -| | phy_cores=${{cores_num}} | clients=${{clients_num}}''' - template_string += f" | streams=${{streams_num}}\n" else: - template_string = f''' -| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id} -| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM -| | phy_cores=${{cores_num}} | clients=${{clients_num}}''' - template_string += f" | streams=${{streams_num}}" \ - f" | bytes=${{bytes_str}}\n" + template_string = \ + f"\n| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id[:-4]}" \ + f"-{suite_id[-3:]}\n" \ + f"| | [Tags] | ${{cores_str}}C\n| | phy_cores=${{cores_num}}\n" return cls(template_string) diff --git a/resources/libraries/robot/hoststack/hoststack.robot b/resources/libraries/robot/hoststack/hoststack.robot index 16a390aeb8..c6bc7dea06 100644 --- a/resources/libraries/robot/hoststack/hoststack.robot +++ b/resources/libraries/robot/hoststack/hoststack.robot @@ -91,7 +91,7 @@ | ... | vcl_config=vcl_iperf3.conf | ... | ld_preload=${True} | ... | transparent_tls=${False} -| ... | json=${False} +| ... | json=${True} | ... | ip_version=${4} | &{iperf3_client_attr}= | ... | role=client @@ -101,7 +101,7 @@ | ... | vcl_config=vcl_iperf3.conf | ... | ld_preload=${True} | ... | transparent_tls=${False} -| ... | json=${False} +| ... | json=${True} | ... | ip_version=${4} | ... | ip_address=${EMPTY} | ... | parallel=${1} @@ -464,9 +464,9 @@ | | [Documentation] | | ... | Configure IP address on the port, set it up and start the specified | | ... | HostStack test programs on the DUTs. Gather test program -| | ... | output and append test results in message. -| | ... | Return boolean indicating when no results were available from -| | ... | both the server and client test programs. +| | ... | output and append JSON formatted test data in message. +| | ... | Return boolean indicating there was a defered failure of either the +| | ... | server and/or client test programs. | | | | Set To Dictionary | ${vpp_echo_server_attr} | uri_ip4_addr | | ... | ${dut2_if1_ip4_addr} @@ -492,25 +492,25 @@ | | ... | ${vpp_echo_client_attr}[namespace] | ${core_list} | | ... | ${vpp_echo_client_attr}[cfg_vpp_feature] | ${vpp_echo_client} | | When Hoststack Test Program Finished | ${dut1} | ${client_pid} -| | ${client_no_results} | ${client_output}= +| | ${client_defer_fail} | ${client_output}= | | ... | Analyze hoststack test program output | ${dut1} | Client | | ... | ${vpp_nsim_attr} | ${vpp_echo_client} | | Then Set test message | ${client_output} | | And Hoststack Test Program Finished | ${dut2} | ${server_pid} -| | ${server_no_results} | ${server_output}= +| | ${server_defer_fail} | ${server_output}= | | ... | Analyze hoststack test program output | ${dut2} | Server | | ... | ${vpp_nsim_attr} | ${vpp_echo_server} | | Set test message | ${server_output} | append=True -| | Run Keyword And Return | No Hoststack Test Program Results -| | ... | ${server_no_results} | ${client_no_results} +| | Run Keyword And Return | Hoststack Test Program Defer Fail +| | ... | ${server_defer_fail} | ${client_defer_fail} | Get Test Results From Hoststack Iperf3 Test | | [Documentation] | | ... | Configure IP address on the port, set it up and start the specified | | ... | HostStack test programs on the DUTs. Gather test program -| | ... | output and append test results in message. -| | ... | Return boolean indicating when no results were available from -| | ... | both the server and client test programs. +| | ... | output and append JSON formatted test data in message. +| | ... | Return boolean indicating there was a defered failure of either the +| | ... | server and/or client test programs. | | | | Set To Dictionary | ${iperf3_client_attr} | ip_address | | ... | ${dut2_if1_ip4_addr} @@ -534,8 +534,8 @@ | | ... | ${iperf3_client_attr}[namespace] | ${core_list} | | ... | ${iperf3_client_attr}[cfg_vpp_feature] | ${iperf3_client} | | When Hoststack Test Program Finished | ${dut1} | ${client_pid} -| | ${client_no_results} | ${client_output}= +| | ${client_defer_fail} | ${client_output}= | | ... | Analyze hoststack test program output | ${dut1} | Client | | ... | ${vpp_nsim_attr} | ${iperf3_client} | | Then Set test message | ${client_output} -| | Return From Keyword | ${client_no_results} +| | Return From Keyword | ${client_defer_fail} diff --git a/resources/tools/wrk/wrk.py b/resources/tools/wrk/wrk.py index 381e9b9da0..85d18d4a9c 100644 --- a/resources/tools/wrk/wrk.py +++ b/resources/tools/wrk/wrk.py @@ -186,12 +186,12 @@ def run_wrk(tg_node, profile_name, tg_numa, test_type, warm_up=False): if test_type == u"cps": log_msg += u"Connections/sec: Avg / Stdev / Max / +/- Stdev\n" for item in stats[u"rps-stats-lst"]: - log_msg += f"{0} / {1} / {2} / {3}\n".format(*item) + log_msg += u" / ".join(map(str, item)) + u"\n" log_msg += f"Total cps: {stats[u'rps-sum']}cps\n" elif test_type == u"rps": log_msg += u"Requests/sec: Avg / Stdev / Max / +/- Stdev\n" for item in stats[u"rps-stats-lst"]: - log_msg += f"{0} / {1} / {2} / {3}\n".format(*item) + log_msg += u" / ".join(map(str, item)) + u"\n" log_msg += f"Total rps: {stats[u'rps-sum']}rps\n" elif test_type == u"bw": log_msg += f"Transfer/sec: {stats[u'bw-sum']}Bps" diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-ldpreload-iperf3-bps.robot new file mode 100644 index 0000000000..f2c7165059 --- /dev/null +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-ldpreload-iperf3-bps.robot @@ -0,0 +1,60 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.HoststackUtil +| Resource | resources/libraries/robot/shared/default.robot +| Resource | resources/libraries/robot/hoststack/hoststack.robot +| +| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV +| ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK +| ... | LDPRELOAD | IPERF3 | 1CLIENT | 1STREAM | 9000B +| ... | eth-ip4tcpbase-ldpreload-iperf3 +| +| Suite Setup | Setup suite single link no tg +| Suite Teardown | Tear down suite | hoststack +| Test Setup | Setup test +| Test Teardown | Tear down test +| +| Test Template | Local template +| +| Documentation | *Iperf3 client -> Iperf3 server throughput. +| +| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology +| ... | with single link between nodes. +| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-TCP +| ... | *[Cfg] DUT configuration:* +| ... | *[Ref] Applicable standard specifications:* + +*** Variables *** +| @{plugins_to_enable}= | dpdk_plugin.so +| ${nic_name}= | Intel-X710 +| ${nic_driver}= | vfio-pci +| ${overhead}= | ${0} +| ${frame_size}= | ${9000} +| ${crypto_type}= | ${None} + +*** Keywords *** +| Local template +| | [Arguments] | ${phy_cores} +| | +| | Set Test Variable | ${dpdk_no_tx_checksum_offload} | ${False} +| | Set VPP Hoststack Attributes | phy_cores=${phy_cores} +| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack Iperf3 Test Program + +*** Test Cases *** +| tc01-9000B-1c-eth-ip4tcpbase-ldpreload-iperf3-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps.robot new file mode 100644 index 0000000000..2eaed06173 --- /dev/null +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps.robot @@ -0,0 +1,62 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.HoststackUtil +| Resource | resources/libraries/robot/shared/default.robot +| Resource | resources/libraries/robot/hoststack/hoststack.robot +| +| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV +| ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK +| ... | NSIM | LDPRELOAD | IPERF3 | 1CLIENT | 1STREAM | 9000B +| ... | eth-ip4tcpbase-nsim-ldpreload-iperf3 +| +| Suite Setup | Setup suite single link no tg +| Suite Teardown | Tear down suite +| Test Setup | Setup test +| Test Teardown | Tear down test +| +| Test Template | Local template +| +| Documentation | *Iperf3 client -> Iperf3 server throughput. +| +| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology +| ... | with single link between nodes. +| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-TCP +| ... | *[Cfg] DUT configuration:* +| ... | *[Ref] Applicable standard specifications:* + +*** Variables *** +| @{plugins_to_enable}= | dpdk_plugin.so | nsim_plugin.so +| ${nic_name}= | Intel-X710 +| ${nic_driver}= | vfio-pci +| ${overhead}= | ${0} +| ${frame_size}= | ${9000} +| ${crypto_type}= | ${None} +| ${pkts_per_drop}= | ${100} + +*** Keywords *** +| Local template +| | [Arguments] | ${phy_cores} +| | +| | Set VPP Hoststack Attributes | phy_cores=${phy_cores} +| | Set VPP NSIM Attributes | output_feature_enable=${True} | +| | ... | packets_per_drop=${pkts_per_drop} +| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack Iperf3 Test Program + +*** Test Cases *** +| tc01-9000B-1c-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps.robot index a907c61e69..7d314bb1c6 100644 --- a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-ldpreload-iperf3-bps.robot +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps.robot @@ -18,7 +18,8 @@ | | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK -| ... | LDPRELOAD | IPERF3 | eth-ip4tcp-ldpreload-iperf3 +| ... | LDPRELOAD | IPERF3 | 1CLIENT | 10STREAM | 9000B +| ... | eth-ip4tcpscale1cl10s-ldpreload-iperf3 | | Suite Setup | Setup suite single link no tg | Suite Teardown | Tear down suite | hoststack @@ -42,23 +43,20 @@ | ${overhead}= | ${0} | ${frame_size}= | ${9000} | ${crypto_type}= | ${None} +| ${streams} | ${10} *** Keywords *** | Local template -| | [Arguments] | ${phy_cores} | ${clients} | ${streams} +| | [Arguments] | ${phy_cores} | | | | Set Test Variable | ${dpdk_no_tx_checksum_offload} | ${False} | | Set VPP Hoststack Attributes | phy_cores=${phy_cores} | | Set Iperf3 Client Attributes | parallel=${streams} -| | ${no_results}= | Get Test Results From Hoststack Iperf3 Test -| | Run Keyword If | ${no_results}==True | FAIL -| | ... | No Test Results From Iperf3 client +| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack Iperf3 Test Program *** Test Cases *** -| tc01-9000B-1c-eth-ip4tcp-ldpreload-iperf3-bps -| | [Tags] | 1C | 1CLIENT | 1STREAM -| | phy_cores=${1} | clients=${1} | streams=${1} - -| tc02-9000B-1c-eth-ip4tcp-ldpreload-iperf3-bps -| | [Tags] | 1C | 1CLIENT | 10STREAM -| | phy_cores=${1} | clients=${1} | streams=${10} +| tc01-9000B-1c-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-nsim-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps.robot index 0a98ee3acd..036414ffd7 100644 --- a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-nsim-ldpreload-iperf3-bps.robot +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps.robot @@ -18,7 +18,8 @@ | | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK -| ... | NSIM | LDPRELOAD | IPERF3 | eth-ip4tcp-nsim-ldpreload-iperf3 +| ... | NSIM | LDPRELOAD | IPERF3 | 1CLIENT | 10STREAM | 9000B +| ... | eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3 | | Suite Setup | Setup suite single link no tg | Suite Teardown | Tear down suite @@ -43,24 +44,21 @@ | ${frame_size}= | ${9000} | ${crypto_type}= | ${None} | ${pkts_per_drop}= | ${100} +| ${streams}= | ${10} *** Keywords *** | Local template -| | [Arguments] | ${phy_cores} | ${clients} | ${streams} +| | [Arguments] | ${phy_cores} | | | | Set VPP Hoststack Attributes | phy_cores=${phy_cores} | | Set Iperf3 Client Attributes | parallel=${streams} | | Set VPP NSIM Attributes | output_feature_enable=${True} | | | ... | packets_per_drop=${pkts_per_drop} -| | ${no_results}= | Get Test Results From Hoststack Iperf3 Test -| | Run Keyword If | ${no_results}==True | FAIL -| | ... | No Test Results From Iperf3 client +| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack Iperf3 Test Program *** Test Cases *** -| tc01-9000B-1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps -| | [Tags] | 1C | 1CLIENT | 1STREAM -| | phy_cores=${1} | clients=${1} | streams=${1} - -| tc02-9000B-1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps -| | [Tags] | 1C | 1CLIENT | 10STREAM -| | phy_cores=${1} | clients=${1} | streams=${10} +| tc01-9000B-1c-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicbase-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicbase-vppecho-bps.robot new file mode 100644 index 0000000000..564844e623 --- /dev/null +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicbase-vppecho-bps.robot @@ -0,0 +1,61 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.HoststackUtil +| Resource | resources/libraries/robot/shared/default.robot +| Resource | resources/libraries/robot/hoststack/hoststack.robot +| +| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV +| ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO +| ... | 1CLIENT | 1STREAM | HOSTSTACK | 9000B | eth-ip4udpquicbase-vppecho +| +| Suite Setup | Setup suite single link no tg +| Suite Teardown | Tear down suite +| Test Setup | Setup test +| Test Teardown | Tear down test +| +| Test Template | Local template +| +| Documentation | *QUIC Unidirectional Echo Client -> Echo Server throughput. +| +| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology +| ... | with single link between nodes. +| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-UDP-QUIC +| ... | *[Cfg] DUT configuration:* +| ... | *[Ref] Applicable standard specifications:* + +*** Variables *** +| @{plugins_to_enable}= | dpdk_plugin.so | quic_plugin.so +| ${nic_name}= | Intel-X710 +| ${nic_driver}= | vfio-pci +| ${overhead}= | ${0} +| ${frame_size}= | ${9000} +| ${crypto_type}= | ${None} +| ${bytes}= | 10G + +*** Keywords *** +| Local template +| | [Arguments] | ${phy_cores} +| | +| | Set VPP Hoststack Attributes | phy_cores=${phy_cores} +| | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | rx_bytes=${bytes} +| | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | tx_bytes=${bytes} +| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack VPP Echo Test Program + +*** Test Cases *** +| tc01-9000B-1c-eth-ip4udpquicbase-vppecho-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquic-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl10s-vppecho-bps.robot index a93afa974a..f4e5c146fb 100644 --- a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquic-vppecho-bps.robot +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl10s-vppecho-bps.robot @@ -18,7 +18,8 @@ | | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO -| ... | HOSTSTACK | eth-ip4udpquic-vppecho +| ... | HOSTSTACK | 10CLIENT | 10STREAM | 9000B +| ... | eth-ip4udpquicscale10cl10s-vppecho | | Suite Setup | Setup suite single link no tg | Suite Teardown | Tear down suite @@ -42,33 +43,24 @@ | ${overhead}= | ${0} | ${frame_size}= | ${9000} | ${crypto_type}= | ${None} +| ${clients}= | ${10} +| ${streams}= | ${10} +| ${bytes}= | 100M *** Keywords *** | Local template -| | [Arguments] | ${phy_cores} | ${clients} | ${streams} | ${bytes} +| | [Arguments] | ${phy_cores} | | | | Set VPP Hoststack Attributes | phy_cores=${phy_cores} | | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | nclients=${clients} | | ... | quic_streams=${streams} | rx_bytes=${bytes} | | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | nclients=${clients} | | ... | quic_streams=${streams} | tx_bytes=${bytes} -| | ${no_results}= | Get Test Results From Hoststack VPP Echo Test -| | Run Keyword If | ${no_results}==True | FAIL -| | ... | No Test Results From External Hoststack Apps +| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack VPP Echo Test Program *** Test Cases *** -| tc01-9000B-1c-eth-ip4udpquic-vppecho-bps -| | [Tags] | 1C | 1CLIENT | 1STREAM -| | phy_cores=${1} | clients=${1} | streams=${1} | bytes=10G - -| tc02-9000B-1c-eth-ip4udpquic-vppecho-bps -| | [Tags] | 1C | 1CLIENT | 10STREAM -| | phy_cores=${1} | clients=${1} | streams=${10} | bytes=1G - -| tc03-9000B-1c-eth-ip4udpquic-vppecho-bps -| | [Tags] | 1C | 10CLIENT | 1STREAM -| | phy_cores=${1} | clients=${10} | streams=${1} | bytes=1G - -| tc04-9000B-1c-eth-ip4udpquic-vppecho-bps -| | [Tags] | 1C | 10CLIENT | 10STREAM -| | phy_cores=${1} | clients=${10} | streams=${10} | bytes=100M +| tc01-9000B-1c-eth-ip4udpquicscale10cl10s-vppecho-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl1s-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl1s-vppecho-bps.robot new file mode 100644 index 0000000000..e160f0562f --- /dev/null +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl1s-vppecho-bps.robot @@ -0,0 +1,65 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.HoststackUtil +| Resource | resources/libraries/robot/shared/default.robot +| Resource | resources/libraries/robot/hoststack/hoststack.robot +| +| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV +| ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO +| ... | HOSTSTACK | 10CLIENT | 1STREAM | 9000B +| ... | eth-ip4udpquicscale10cl1s-vppecho +| +| Suite Setup | Setup suite single link no tg +| Suite Teardown | Tear down suite +| Test Setup | Setup test +| Test Teardown | Tear down test +| +| Test Template | Local template +| +| Documentation | *QUIC Unidirectional Echo Client -> Echo Server throughput. +| +| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology +| ... | with single link between nodes. +| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-UDP-QUIC +| ... | *[Cfg] DUT configuration:* +| ... | *[Ref] Applicable standard specifications:* + +*** Variables *** +| @{plugins_to_enable}= | dpdk_plugin.so | quic_plugin.so +| ${nic_name}= | Intel-X710 +| ${nic_driver}= | vfio-pci +| ${overhead}= | ${0} +| ${frame_size}= | ${9000} +| ${crypto_type}= | ${None} +| ${clients}= | ${10} +| ${bytes}= | 1G + +*** Keywords *** +| Local template +| | [Arguments] | ${phy_cores} +| | +| | Set VPP Hoststack Attributes | phy_cores=${phy_cores} +| | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | nclients=${clients} +| | ... | rx_bytes=${bytes} +| | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | nclients=${clients} +| | ... | tx_bytes=${bytes} +| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack VPP Echo Test Program + +*** Test Cases *** +| tc01-9000B-1c-eth-ip4udpquicscale10cl1s-vppecho-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale1cl10s-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale1cl10s-vppecho-bps.robot new file mode 100644 index 0000000000..d03e247df2 --- /dev/null +++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale1cl10s-vppecho-bps.robot @@ -0,0 +1,65 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.HoststackUtil +| Resource | resources/libraries/robot/shared/default.robot +| Resource | resources/libraries/robot/hoststack/hoststack.robot +| +| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV +| ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO +| ... | HOSTSTACK | 1CLIENT | 10STREAM | 9000B +| ... | eth-ip4udpquicscale1cl10s-vppecho +| +| Suite Setup | Setup suite single link no tg +| Suite Teardown | Tear down suite +| Test Setup | Setup test +| Test Teardown | Tear down test +| +| Test Template | Local template +| +| Documentation | *QUIC Unidirectional Echo Client -> Echo Server throughput. +| +| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology +| ... | with single link between nodes. +| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-UDP-QUIC +| ... | *[Cfg] DUT configuration:* +| ... | *[Ref] Applicable standard specifications:* + +*** Variables *** +| @{plugins_to_enable}= | dpdk_plugin.so | quic_plugin.so +| ${nic_name}= | Intel-X710 +| ${nic_driver}= | vfio-pci +| ${overhead}= | ${0} +| ${frame_size}= | ${9000} +| ${crypto_type}= | ${None} +| ${streams}= | ${10} +| ${bytes}= | 1G + +*** Keywords *** +| Local template +| | [Arguments] | ${phy_cores} +| | +| | Set VPP Hoststack Attributes | phy_cores=${phy_cores} +| | Set VPP Echo Server Attributes | cfg_vpp_feature=quic +| | ... | quic_streams=${streams} | rx_bytes=${bytes} +| | Set VPP Echo Client Attributes | cfg_vpp_feature=quic +| | ... | quic_streams=${streams} | tx_bytes=${bytes} +| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test +| | Run Keyword If | ${defer_fail}==True | FAIL +| | ... | Defered Failure From Hoststack VPP Echo Test Program + +*** Test Cases *** +| tc01-9000B-1c-eth-ip4udpquicscale1cl10s-vppecho-bps +| | [Tags] | 1C +| | phy_cores=${1} diff --git a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot index d7a7e6cf88..6b4f3c30ab 100644 --- a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot +++ b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot @@ -19,7 +19,7 @@ | Resource | resources/libraries/robot/hoststack/tcp_setup.robot | | Force Tags | 2_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV -| ... | HTTP | TCP | TCP_CPS | NIC_Intel-X710 | DRV_VFIO_PCI +| ... | HTTP | TCP | TCP_CPS | NIC_Intel-X710 | DRV_VFIO_PCI | IMIX | ... | eth-ip4tcphttp-wrk8u8c50con | | Suite Setup | Setup suite single link | wrk diff --git a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot index 27469bb189..b9b2226855 100644 --- a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot +++ b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot @@ -19,7 +19,7 @@ | Resource | resources/libraries/robot/hoststack/tcp_setup.robot | | Force Tags | 2_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV -| ... | HTTP | TCP | TCP_RPS | NIC_Intel-X710 | DRV_VFIO_PCI +| ... | HTTP | TCP | TCP_RPS | NIC_Intel-X710 | DRV_VFIO_PCI | IMIX | ... | eth-ip4tcphttp-wrk8u8c50con | | Suite Setup | Setup suite single link | wrk |