aboutsummaryrefslogtreecommitdiffstats
path: root/resources
diff options
context:
space:
mode:
authorDave Wallace <dwallacelf@gmail.com>2020-02-11 00:57:10 +0000
committerTibor Frank <tifrank@cisco.com>2020-03-03 10:06:52 +0000
commit5570bf3ab49301201dd7607bb4f8de67fd8f16dc (patch)
tree4154400992c4899ab4c083e8ed66e160784cbcbf /resources
parent827ed3c36c2d789dc3fd0f41cc17de1587416103 (diff)
perf: Clean up Hoststack tests
- Update test names with clients/streams - Convert test results to JSON output * iperf3 results include bits_per_second * vpp_echo results include both client and server output which includes time in seconds and rx_data/tx_data in bytes which can be used to calculate the average bits per second. Tx and Rx data will always be the same: BPS = (client tx_data * 8) / ((client time + server time) / 2) - Fix WRK test results data formatting errors Change-Id: Ie2aeb665e3cc0739b16f97ba2628eebe6e041d22 Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
Diffstat (limited to 'resources')
-rw-r--r--resources/libraries/python/HoststackUtil.py91
-rw-r--r--resources/libraries/python/autogen/Regenerator.py67
-rw-r--r--resources/libraries/python/autogen/Testcase.py31
-rw-r--r--resources/libraries/robot/hoststack/hoststack.robot28
-rw-r--r--resources/tools/wrk/wrk.py4
5 files changed, 88 insertions, 133 deletions
diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py
index dde5cf60d0..d184b5af6d 100644
--- a/resources/libraries/python/HoststackUtil.py
+++ b/resources/libraries/python/HoststackUtil.py
@@ -12,6 +12,7 @@
# limitations under the License.
"""Host Stack util library."""
+import json
from time import sleep
from robot.api import logger
@@ -249,10 +250,16 @@ class HoststackUtil():
sleep(1)
@staticmethod
- def analyze_hoststack_test_program_output(node, role, nsim_attr,
- program):
+ def analyze_hoststack_test_program_output(
+ node, role, nsim_attr, program):
"""Gather HostStack test program output and check for errors.
+ The [defer_fail] return bool is used instead of failing immediately
+ to allow the analysis of both the client and server instances of
+ the test program for debugging a test failure. When [defer_fail]
+ is true, then the string returned is debug output instead of
+ JSON formatted test program results.
+
:param node: DUT node.
:param role: Role (client|server) of test program.
:param nsim_attr: Network Simulation Attributes.
@@ -262,7 +269,8 @@ class HoststackUtil():
:type role: str
:type nsim_attr: dict
:type program: dict
- :returns: tuple of no results bool and test program results.
+ :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
+ test program output or failure debug output.
:rtype: bool, str
:raises RuntimeError: If node subtype is not a DUT.
"""
@@ -277,7 +285,6 @@ class HoststackUtil():
program_stdout, program_stderr = \
HoststackUtil.get_hoststack_test_program_logs(node, program)
- no_results = False
env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
program_cmd = f"{env_vars}{program_name} {program[u'args']}"
test_results = f"Test Results of '{program_cmd}':\n"
@@ -296,50 +303,50 @@ class HoststackUtil():
f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
f"pkts/drop\n"
- if u"error" in program_stderr.lower():
- test_results += f"ERROR DETECTED:\n{program_stderr}"
- raise RuntimeError(test_results)
- if program_stdout:
- bad_test_results = False
- if program[u"name"] == u"vpp_echo":
- if u"JSON stats" in program_stdout:
- test_results += program_stdout
- # TODO: Decode vpp_echo output when JSON format is correct.
- # json_start = program_stdout.find(u"{")
- # vpp_echo_results = json.loads(program_stdout[json_start:])
- if u'"has_failed": "0"' not in program_stdout:
- bad_test_results = True
- else:
- test_results += u"Invalid test data output!\n" + \
- program_stdout
- bad_test_results = True
- else:
- test_results += program_stdout
- if bad_test_results:
- raise RuntimeError(test_results)
- else:
- no_results = True
- test_results += f"\nNo {program} test data retrieved!\n"
- cmd = u"ls -l /tmp/*.log"
- ls_stdout, _ = exec_cmd_no_error(node, cmd, sudo=True)
- test_results += f"{ls_stdout}\n"
-
# TODO: Incorporate show error stats into results analysis
- host = node[u"host"]
test_results += \
- f"\n{role} VPP 'show errors' on host {host}:\n" \
+ f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
- return no_results, test_results
+ if u"error" in program_stderr.lower():
+ test_results += f"ERROR DETECTED:\n{program_stderr}"
+ return (True, test_results)
+ if not program_stdout:
+ test_results += f"\nNo {program} test data retrieved!\n"
+ ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
+ sudo=True)
+ test_results += f"{ls_stdout}\n"
+ return (True, test_results)
+ if program[u"name"] == u"vpp_echo":
+ if u"JSON stats" in program_stdout and \
+ u'"has_failed": "0"' in program_stdout:
+ json_start = program_stdout.find(u"{")
+ #TODO: Fix parsing once vpp_echo produces valid
+ # JSON output. Truncate for now.
+ json_end = program_stdout.find(u',\n "closing"')
+ json_results = f"{program_stdout[json_start:json_end]}\n}}"
+ program_json = json.loads(json_results)
+ else:
+ test_results += u"Invalid test data output!\n" + program_stdout
+ return (True, test_results)
+ elif program[u"name"] == u"iperf3":
+ test_results += program_stdout
+ iperf3_json = json.loads(program_stdout)
+ program_json = iperf3_json[u"intervals"][0][u"sum"]
+ else:
+ test_results += u"Unknown HostStack Test Program!\n" + \
+ program_stdout
+ return (True, program_stdout)
+ return (False, json.dumps(program_json))
@staticmethod
- def no_hoststack_test_program_results(server_no_results, client_no_results):
- """Return True if no HostStack test program output was gathered.
+ def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
+ """Return True if either HostStack test program fail was deferred.
- :param server_no_results: server no results value.
- :param client_no_results: client no results value.
- :type server_no_results: bool
- :type client_no_results: bool
+ :param server_defer_fail: server no results value.
+ :param client_defer_fail: client no results value.
+ :type server_defer_fail: bool
+ :type client_defer_fail: bool
:rtype: bool
"""
- return server_no_results and client_no_results
+ return server_defer_fail and client_defer_fail
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index 5cd8bcccb7..d40ae7c9fc 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -440,50 +440,24 @@ class Regenerator:
min_frame_size = PROTOCOL_TO_MIN_FRAME_SIZE[protocol]
default_kwargs_list = [
- {u"frame_size": min_frame_size, u"phy_cores": 1, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": min_frame_size, u"phy_cores": 2, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": min_frame_size, u"phy_cores": 4, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 1518, u"phy_cores": 1, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 1518, u"phy_cores": 2, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 1518, u"phy_cores": 4, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 9000, u"phy_cores": 1, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 9000, u"phy_cores": 2, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 9000, u"phy_cores": 4, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"}
+ {u"frame_size": min_frame_size, u"phy_cores": 1},
+ {u"frame_size": min_frame_size, u"phy_cores": 2},
+ {u"frame_size": min_frame_size, u"phy_cores": 4},
+ {u"frame_size": 1518, u"phy_cores": 1},
+ {u"frame_size": 1518, u"phy_cores": 2},
+ {u"frame_size": 1518, u"phy_cores": 4},
+ {u"frame_size": 9000, u"phy_cores": 1},
+ {u"frame_size": 9000, u"phy_cores": 2},
+ {u"frame_size": 9000, u"phy_cores": 4},
+ {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1},
+ {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2},
+ {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4}
]
- hoststack_wrk_kwargs_list = [
- {u"frame_size": 0, u"phy_cores": i, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"} for i in (1, 2, 4)
+ hs_wrk_kwargs_list = [
+ {u"frame_size": 0, u"phy_cores": i} for i in (1, 2, 4)
]
- hoststack_iperf3_kwargs_list = [
- {u"frame_size": 0, u"phy_cores": 1, u"clients": 1,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"frame_size": 0, u"phy_cores": 1, u"clients": 1,
- u"streams": 10, u"bytes_str": u"1G"},
- ]
- hoststack_quic_kwargs_list = [
- {u"phy_cores": 1, u"frame_size": 0, u"clients": 1,
- u"streams": 1, u"bytes_str": u"10G"},
- {u"phy_cores": 1, u"frame_size": 0, u"clients": 1,
- u"streams": 10, u"bytes_str": u"1G"},
- {u"phy_cores": 1, u"frame_size": 0, u"clients": 10,
- u"streams": 1, u"bytes_str": u"1G"},
- {u"phy_cores": 1, u"frame_size": 0, u"clients": 10,
- u"streams": 10, u"bytes_str": u"100M"},
+ hs_bps_kwargs_list = [
+ {u"frame_size": 0, u"phy_cores": 1},
]
for in_filename in glob(pattern):
@@ -510,14 +484,9 @@ class Regenerator:
elif in_filename.endswith(u"-reconf.robot"):
write_reconf_files(in_filename, in_prolog, default_kwargs_list)
elif in_filename[-10:] in (u"-cps.robot", u"-rps.robot"):
- write_tcp_files(in_filename, in_prolog,
- hoststack_wrk_kwargs_list)
+ write_tcp_files(in_filename, in_prolog, hs_wrk_kwargs_list)
elif in_filename[-10:] in (u"-bps.robot"):
- if u"ldpreload-iperf3" in in_filename:
- hoststack_kwargs_list = hoststack_iperf3_kwargs_list
- else:
- hoststack_kwargs_list = hoststack_quic_kwargs_list
- write_tcp_files(in_filename, in_prolog, hoststack_kwargs_list)
+ write_tcp_files(in_filename, in_prolog, hs_bps_kwargs_list)
else:
raise RuntimeError(
f"Error in {in_filename}: non-primary suite type found."
diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py
index c540d8ccec..d419b2993b 100644
--- a/resources/libraries/python/autogen/Testcase.py
+++ b/resources/libraries/python/autogen/Testcase.py
@@ -34,8 +34,7 @@ class Testcase:
"""
self.template = Template(template_string)
- def generate(self, num, frame_size, phy_cores, clients, streams,
- bytes_str):
+ def generate(self, num, frame_size, phy_cores):
"""Return string of test case code with placeholders filled.
Fail if there are placeholders left unfilled.
@@ -44,16 +43,9 @@ class Testcase:
:param num: Test case number. Example value: 4.
:param frame_size: Imix string or numeric frame size. Example: 74.
:param phy_cores: Number of physical cores to use. Example: 2.
- :param clients: Number of clients used by test program. Example: 4.
- :param streams: Number of streams used by test program. Example: 10.
- :param bytes_str: Size in bytes of stream sent by test program.
- Example: 1G
:type num: int
:type frame_size: str or int
:type phy_cores: int or str
- :type clients: int
- :type streams: int
- :type bytes_str: str
:returns: Filled template, usable as test case code.
:rtype: str
"""
@@ -75,11 +67,6 @@ class Testcase:
u"cores_num": f"${{{cores_num:d}}}",
u"cores_str": phy_cores,
u"tc_num": f"tc{num:02d}",
- u"clients_num": f"${{{clients:d}}}",
- u"clients_str": str(clients),
- u"streams_num": f"${{{streams:d}}}",
- u"streams_str": str(streams),
- u"bytes_str": bytes_str,
}
)
return self.template.substitute(subst_dict)
@@ -122,17 +109,9 @@ class Testcase:
| | [Tags] | ${{cores_str}}C
| | phy_cores=${{cores_num}}
'''
- elif u"iperf3" in suite_id:
- template_string = f'''
-| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id}
-| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM
-| | phy_cores=${{cores_num}} | clients=${{clients_num}}'''
- template_string += f" | streams=${{streams_num}}\n"
else:
- template_string = f'''
-| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id}
-| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM
-| | phy_cores=${{cores_num}} | clients=${{clients_num}}'''
- template_string += f" | streams=${{streams_num}}" \
- f" | bytes=${{bytes_str}}\n"
+ template_string = \
+ f"\n| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id[:-4]}" \
+ f"-{suite_id[-3:]}\n" \
+ f"| | [Tags] | ${{cores_str}}C\n| | phy_cores=${{cores_num}}\n"
return cls(template_string)
diff --git a/resources/libraries/robot/hoststack/hoststack.robot b/resources/libraries/robot/hoststack/hoststack.robot
index 16a390aeb8..c6bc7dea06 100644
--- a/resources/libraries/robot/hoststack/hoststack.robot
+++ b/resources/libraries/robot/hoststack/hoststack.robot
@@ -91,7 +91,7 @@
| ... | vcl_config=vcl_iperf3.conf
| ... | ld_preload=${True}
| ... | transparent_tls=${False}
-| ... | json=${False}
+| ... | json=${True}
| ... | ip_version=${4}
| &{iperf3_client_attr}=
| ... | role=client
@@ -101,7 +101,7 @@
| ... | vcl_config=vcl_iperf3.conf
| ... | ld_preload=${True}
| ... | transparent_tls=${False}
-| ... | json=${False}
+| ... | json=${True}
| ... | ip_version=${4}
| ... | ip_address=${EMPTY}
| ... | parallel=${1}
@@ -464,9 +464,9 @@
| | [Documentation]
| | ... | Configure IP address on the port, set it up and start the specified
| | ... | HostStack test programs on the DUTs. Gather test program
-| | ... | output and append test results in message.
-| | ... | Return boolean indicating when no results were available from
-| | ... | both the server and client test programs.
+| | ... | output and append JSON formatted test data in message.
+| | ... | Return boolean indicating there was a defered failure of either the
+| | ... | server and/or client test programs.
| |
| | Set To Dictionary | ${vpp_echo_server_attr} | uri_ip4_addr
| | ... | ${dut2_if1_ip4_addr}
@@ -492,25 +492,25 @@
| | ... | ${vpp_echo_client_attr}[namespace] | ${core_list}
| | ... | ${vpp_echo_client_attr}[cfg_vpp_feature] | ${vpp_echo_client}
| | When Hoststack Test Program Finished | ${dut1} | ${client_pid}
-| | ${client_no_results} | ${client_output}=
+| | ${client_defer_fail} | ${client_output}=
| | ... | Analyze hoststack test program output | ${dut1} | Client
| | ... | ${vpp_nsim_attr} | ${vpp_echo_client}
| | Then Set test message | ${client_output}
| | And Hoststack Test Program Finished | ${dut2} | ${server_pid}
-| | ${server_no_results} | ${server_output}=
+| | ${server_defer_fail} | ${server_output}=
| | ... | Analyze hoststack test program output | ${dut2} | Server
| | ... | ${vpp_nsim_attr} | ${vpp_echo_server}
| | Set test message | ${server_output} | append=True
-| | Run Keyword And Return | No Hoststack Test Program Results
-| | ... | ${server_no_results} | ${client_no_results}
+| | Run Keyword And Return | Hoststack Test Program Defer Fail
+| | ... | ${server_defer_fail} | ${client_defer_fail}
| Get Test Results From Hoststack Iperf3 Test
| | [Documentation]
| | ... | Configure IP address on the port, set it up and start the specified
| | ... | HostStack test programs on the DUTs. Gather test program
-| | ... | output and append test results in message.
-| | ... | Return boolean indicating when no results were available from
-| | ... | both the server and client test programs.
+| | ... | output and append JSON formatted test data in message.
+| | ... | Return boolean indicating there was a defered failure of either the
+| | ... | server and/or client test programs.
| |
| | Set To Dictionary | ${iperf3_client_attr} | ip_address
| | ... | ${dut2_if1_ip4_addr}
@@ -534,8 +534,8 @@
| | ... | ${iperf3_client_attr}[namespace] | ${core_list}
| | ... | ${iperf3_client_attr}[cfg_vpp_feature] | ${iperf3_client}
| | When Hoststack Test Program Finished | ${dut1} | ${client_pid}
-| | ${client_no_results} | ${client_output}=
+| | ${client_defer_fail} | ${client_output}=
| | ... | Analyze hoststack test program output | ${dut1} | Client
| | ... | ${vpp_nsim_attr} | ${iperf3_client}
| | Then Set test message | ${client_output}
-| | Return From Keyword | ${client_no_results}
+| | Return From Keyword | ${client_defer_fail}
diff --git a/resources/tools/wrk/wrk.py b/resources/tools/wrk/wrk.py
index 381e9b9da0..85d18d4a9c 100644
--- a/resources/tools/wrk/wrk.py
+++ b/resources/tools/wrk/wrk.py
@@ -186,12 +186,12 @@ def run_wrk(tg_node, profile_name, tg_numa, test_type, warm_up=False):
if test_type == u"cps":
log_msg += u"Connections/sec: Avg / Stdev / Max / +/- Stdev\n"
for item in stats[u"rps-stats-lst"]:
- log_msg += f"{0} / {1} / {2} / {3}\n".format(*item)
+ log_msg += u" / ".join(map(str, item)) + u"\n"
log_msg += f"Total cps: {stats[u'rps-sum']}cps\n"
elif test_type == u"rps":
log_msg += u"Requests/sec: Avg / Stdev / Max / +/- Stdev\n"
for item in stats[u"rps-stats-lst"]:
- log_msg += f"{0} / {1} / {2} / {3}\n".format(*item)
+ log_msg += u" / ".join(map(str, item)) + u"\n"
log_msg += f"Total rps: {stats[u'rps-sum']}rps\n"
elif test_type == u"bw":
log_msg += f"Transfer/sec: {stats[u'bw-sum']}Bps"