From 84ab8bd624aa016988fc9f56e5a07e9ec07128b5 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Tue, 25 Feb 2020 10:11:10 +0100 Subject: Report: Hoststack methodology Change-Id: I105e1d4823df42522bff1af50d1bb173cd84d958 Signed-off-by: Tibor Frank --- .../dpdk_performance_tests/csit_release_notes.rst | 2 +- docs/report/introduction/methodology.rst | 4 +- .../methodology_hoststack_testing/index.rst | 8 ++++ .../methodology_http_tcp_with_wrk.rst | 39 ++++++++++++++++++ .../methodology_quic_with_vppecho.rst | 46 ++++++++++++++++++++++ .../methodology_tcp_with_iperf3.rst | 43 ++++++++++++++++++++ .../introduction/methodology_http_tcp_with_wrk.rst | 39 ------------------ .../introduction/methodology_quic_with_vppecho.rst | 44 --------------------- .../introduction/methodology_tcp_with_iperf3.rst | 41 ------------------- docs/report/introduction/physical_testbeds.rst | 8 ++-- .../vpp_performance_tests/csit_release_notes.rst | 4 +- resources/tools/presentation/input_data_parser.py | 12 +++--- 12 files changed, 150 insertions(+), 140 deletions(-) create mode 100644 docs/report/introduction/methodology_hoststack_testing/index.rst create mode 100644 docs/report/introduction/methodology_hoststack_testing/methodology_http_tcp_with_wrk.rst create mode 100644 docs/report/introduction/methodology_hoststack_testing/methodology_quic_with_vppecho.rst create mode 100644 docs/report/introduction/methodology_hoststack_testing/methodology_tcp_with_iperf3.rst delete mode 100644 docs/report/introduction/methodology_http_tcp_with_wrk.rst delete mode 100644 docs/report/introduction/methodology_quic_with_vppecho.rst delete mode 100644 docs/report/introduction/methodology_tcp_with_iperf3.rst diff --git a/docs/report/dpdk_performance_tests/csit_release_notes.rst b/docs/report/dpdk_performance_tests/csit_release_notes.rst index 7f190ab201..b6cec015be 100644 --- a/docs/report/dpdk_performance_tests/csit_release_notes.rst +++ b/docs/report/dpdk_performance_tests/csit_release_notes.rst @@ -30,7 +30,7 @@ Changes in |csit-release| - **New Intel Xeon Cascadelake Testbeds**: Added performance tests for 2-Node-Cascadelake (2n-clx) testbeds with x710, xxv710 and - mcx556a-edat NIC cards. + cx556a-edat NIC cards. .. // Alternative Note for 1st Bullet when bad microcode Skx, Clx results are published diff --git a/docs/report/introduction/methodology.rst b/docs/report/introduction/methodology.rst index 107a6954c6..61752a4560 100644 --- a/docs/report/introduction/methodology.rst +++ b/docs/report/introduction/methodology.rst @@ -13,9 +13,7 @@ Test Methodology methodology_data_plane_throughput/index methodology_packet_latency methodology_multi_core_speedup - methodology_http_tcp_with_wrk - methodology_tcp_with_iperf3 - methodology_quic_with_vppecho + methodology_hoststack_testing/index methodology_reconf methodology_vpp_startup_settings methodology_kvm_vms_vhost_user diff --git a/docs/report/introduction/methodology_hoststack_testing/index.rst b/docs/report/introduction/methodology_hoststack_testing/index.rst new file mode 100644 index 0000000000..e7b5b79610 --- /dev/null +++ b/docs/report/introduction/methodology_hoststack_testing/index.rst @@ -0,0 +1,8 @@ +Hoststack Testing +----------------- + +.. toctree:: + + methodology_http_tcp_with_wrk + methodology_tcp_with_iperf3 + methodology_quic_with_vppecho diff --git a/docs/report/introduction/methodology_hoststack_testing/methodology_http_tcp_with_wrk.rst b/docs/report/introduction/methodology_hoststack_testing/methodology_http_tcp_with_wrk.rst new file mode 100644 index 0000000000..f5da5339a0 --- /dev/null +++ b/docs/report/introduction/methodology_hoststack_testing/methodology_http_tcp_with_wrk.rst @@ -0,0 +1,39 @@ +HTTP/TCP with WRK +^^^^^^^^^^^^^^^^^ + +`WRK HTTP benchmarking tool `_ is used for +TCP/IP and HTTP tests of VPP Host Stack and built-in static HTTP server. +WRK has been chosen as it is capable of generating significant TCP/IP +and HTTP loads by scaling number of threads across multi-core processors. + +This in turn enables high scale benchmarking of the VPP Host Stack TCP/IP +and HTTP service including HTTP TCP/IP Connections-Per-Second (CPS) and +HTTP Requests-Per-Second. + +The initial tests are designed as follows: + +- HTTP and TCP/IP Connections-Per-Second (CPS) + + - WRK configured to use 8 threads across 8 cores, 1 thread per core. + - Maximum of 50 concurrent connections across all WRK threads. + - Timeout for server responses set to 5 seconds. + - Test duration is 30 seconds. + - Expected HTTP test sequence: + + - Single HTTP GET Request sent per open connection. + - Connection close after valid HTTP reply. + - Resulting flow sequence - 8 packets: >Syn, Ack, >Req, + Fin, Ack. + +- HTTP Requests-Per-Second + + - WRK configured to use 8 threads across 8 cores, 1 thread per core. + - Maximum of 50 concurrent connections across all WRK threads. + - Timeout for server responses set to 5 seconds. + - Test duration is 30 seconds. + - Expected HTTP test sequence: + + - Multiple HTTP GET Requests sent in sequence per open connection. + - Connection close after set test duration time. + - Resulting flow sequence: >Syn, Ack, >Req[1], Req[n], Fin, Ack. diff --git a/docs/report/introduction/methodology_hoststack_testing/methodology_quic_with_vppecho.rst b/docs/report/introduction/methodology_hoststack_testing/methodology_quic_with_vppecho.rst new file mode 100644 index 0000000000..329b9a2964 --- /dev/null +++ b/docs/report/introduction/methodology_hoststack_testing/methodology_quic_with_vppecho.rst @@ -0,0 +1,46 @@ +Hoststack Throughput Testing over QUIC/UDP/IP with vpp_echo +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`vpp_echo performance testing tool `_ +is a bespoke performance test application which utilizes the 'native +HostStack APIs' to verify performance and correct handling of +connection/stream events with uni-directional and bi-directional +streams of data. + +Because iperf3 does not support the QUIC transport protocol, vpp_echo +is used for measuring the maximum attainable bandwidth of the VPP Host +Stack connection utilzing the QUIC transport protocol across two +instances of VPP running on separate DUT nodes. The QUIC transport +protocol supports multiple streams per connection and test cases +utilize different combinations of QUIC connections and number of +streams per connection. + +The test configuration is as follows: + +:: + + DUT1 Network DUT2 + [ vpp_echo-client -> VPP1 ]=======[ VPP2 -> vpp_echo-server] + N-streams/connection + +where, + + 1. vpp_echo server attaches to VPP2 and LISTENs on VPP2:TCP port 1234. + 2. vpp_echo client creates one or more connections to VPP1 and opens + one or more stream per connection to VPP2:TCP port 1234. + 3. vpp_echo client transmits a uni-directional stream as fast as the + VPP Host Stack allows to the vpp_echo server for the test duration. + 4. At the end of the test the vpp_echo client emits the goodput + measurements for all streams and the sum of all streams. + + Test cases include + + 1. 1 QUIC Connection with 1 Stream + 2. 1 QUIC connection with 10 Streams + 3. 10 QUIC connetions with 1 Stream + 4. 10 QUIC connections with 10 Streams + + with stream sizes to provide reasonable test durations. The VPP Host + Stack QUIC transport is configured to utilize the picotls encryption + library. In the future, tests utilizing addtional encryption + algorithms will be added. diff --git a/docs/report/introduction/methodology_hoststack_testing/methodology_tcp_with_iperf3.rst b/docs/report/introduction/methodology_hoststack_testing/methodology_tcp_with_iperf3.rst new file mode 100644 index 0000000000..1355a3cb21 --- /dev/null +++ b/docs/report/introduction/methodology_hoststack_testing/methodology_tcp_with_iperf3.rst @@ -0,0 +1,43 @@ +Hoststack Throughput Testing over TCP/IP with iperf3 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`iperf3 goodput measurement tool `_ +is used for measuring the maximum attainable goodput of the VPP Host +Stack connection across two instances of VPP running on separate DUT +nodes. iperf3 is a popular open source tool for active measurements +of the maximum achievable goodput on IP networks. + +Because iperf3 utilizes the POSIX socket interface APIs, the current +test configuration utilizes the LD_PRELOAD mechanism in the linux +kernel to connect iperf3 to the VPP Host Stack using the VPP +Communications Library (VCL) LD_PRELOAD library (libvcl_ldpreload.so). + +In the future, a forked version of iperf3 which has been modified to +directly use the VCL application APIs may be added to determine the +difference in performance of 'VCL Native' applications versus utilizing +LD_PRELOAD which inherently has more overhead and other limitations. + +The test configuration is as follows: + +:: + + DUT1 Network DUT2 + [ iperf3-client -> VPP1 ]=======[ VPP2 -> iperf3-server] + +where, + + 1. iperf3 server attaches to VPP2 and LISTENs on VPP2:TCP port 5201. + 2. iperf3 client attaches to VPP1 and opens one or more stream + connections to VPP2:TCP port 5201. + 3. iperf3 client transmits a uni-directional stream as fast as the + VPP Host Stack allows to the iperf3 server for the test duration. + 4. At the end of the test the iperf3 client emits the goodput + measurements for all streams and the sum of all streams. + + Test cases include 1 and 10 Streams with a 20 second test duration + with the VPP Host Stack configured to utilize the Cubic TCP + congestion algorithm. + + Note: iperf3 is single threaded, so it is expected that the 10 stream + test does not show any performance improvement due to + multi-thread/multi-core execution. diff --git a/docs/report/introduction/methodology_http_tcp_with_wrk.rst b/docs/report/introduction/methodology_http_tcp_with_wrk.rst deleted file mode 100644 index cd831b4481..0000000000 --- a/docs/report/introduction/methodology_http_tcp_with_wrk.rst +++ /dev/null @@ -1,39 +0,0 @@ -HTTP/TCP with WRK ------------------ - -`WRK HTTP benchmarking tool `_ is used for -TCP/IP and HTTP tests of VPP Host Stack and built-in static HTTP server. -WRK has been chosen as it is capable of generating significant TCP/IP -and HTTP loads by scaling number of threads across multi-core processors. - -This in turn enables high scale benchmarking of the VPP Host Stack TCP/IP -and HTTP service including HTTP TCP/IP Connections-Per-Second (CPS) and -HTTP Requests-Per-Second. - -The initial tests are designed as follows: - -- HTTP and TCP/IP Connections-Per-Second (CPS) - - - WRK configured to use 8 threads across 8 cores, 1 thread per core. - - Maximum of 50 concurrent connections across all WRK threads. - - Timeout for server responses set to 5 seconds. - - Test duration is 30 seconds. - - Expected HTTP test sequence: - - - Single HTTP GET Request sent per open connection. - - Connection close after valid HTTP reply. - - Resulting flow sequence - 8 packets: >Syn, Ack, >Req, - Fin, Ack. - -- HTTP Requests-Per-Second - - - WRK configured to use 8 threads across 8 cores, 1 thread per core. - - Maximum of 50 concurrent connections across all WRK threads. - - Timeout for server responses set to 5 seconds. - - Test duration is 30 seconds. - - Expected HTTP test sequence: - - - Multiple HTTP GET Requests sent in sequence per open connection. - - Connection close after set test duration time. - - Resulting flow sequence: >Syn, Ack, >Req[1], Req[n], Fin, Ack. diff --git a/docs/report/introduction/methodology_quic_with_vppecho.rst b/docs/report/introduction/methodology_quic_with_vppecho.rst deleted file mode 100644 index 5579fb5954..0000000000 --- a/docs/report/introduction/methodology_quic_with_vppecho.rst +++ /dev/null @@ -1,44 +0,0 @@ -Hoststack Throughput Testing over QUIC/UDP/IP with vpp_echo ------------------------------------------------------------ - -`vpp_echo performance testing tool `_ -is a bespoke performance test application which utilizes the 'native -HostStack APIs' to verify performance and correct handling of -connection/stream events with uni-directional and bi-directional -streams of data. - -Because iperf3 does not support the QUIC transport protocol, vpp_echo -is used for measuring the maximum attainable bandwidth of the VPP Host -Stack connection utilzing the QUIC transport protocol across two -instances of VPP running on separate DUT nodes. The QUIC transport -protocol supports multiple streams per connection and test cases -utilize different combinations of QUIC connections and number of -streams per connection. - -The test configuration is as follows: - - DUT1 Network DUT2 -[ vpp_echo-client -> VPP1 ]=======[ VPP2 -> vpp_echo-server] - N-streams/connection - -where, - - 1. vpp_echo server attaches to VPP2 and LISTENs on VPP2:TCP port 1234. - 2. vpp_echo client creates one or more connections to VPP1 and opens - one or more stream per connection to VPP2:TCP port 1234. - 3. vpp_echo client transmits a uni-directional stream as fast as the - VPP Host Stack allows to the vpp_echo server for the test duration. - 4. At the end of the test the vpp_echo client emits the goodput - measurements for all streams and the sum of all streams. - - Test cases include - - 1. 1 QUIC Connection with 1 Stream - 2. 1 QUIC connection with 10 Streams - 3. 10 QUIC connetions with 1 Stream - 4. 10 QUIC connections with 10 Streams - - with stream sizes to provide reasonable test durations. The VPP Host - Stack QUIC transport is configured to utilize the picotls encryption - library. In the future, tests utilizing addtional encryption - algorithms will be added. diff --git a/docs/report/introduction/methodology_tcp_with_iperf3.rst b/docs/report/introduction/methodology_tcp_with_iperf3.rst deleted file mode 100644 index 288da004a5..0000000000 --- a/docs/report/introduction/methodology_tcp_with_iperf3.rst +++ /dev/null @@ -1,41 +0,0 @@ -Hoststack Throughput Testing over TCP/IP with iperf3 ----------------------------------------------------- - -`iperf3 goodput measurement tool `_ -is used for measuring the maximum attainable goodput of the VPP Host -Stack connection across two instances of VPP running on separate DUT -nodes. iperf3 is a popular open source tool for active measurements -of the maximum achievable goodput on IP networks. - -Because iperf3 utilizes the POSIX socket interface APIs, the current -test configuration utilizes the LD_PRELOAD mechanism in the linux -kernel to connect iperf3 to the VPP Host Stack using the VPP -Communications Library (VCL) LD_PRELOAD library (libvcl_ldpreload.so). - -In the future, a forked version of iperf3 which has been modified to -directly use the VCL application APIs may be added to determine the -difference in performance of 'VCL Native' applications versus utilizing -LD_PRELOAD which inherently has more overhead and other limitations. - -The test configuration is as follows: - - DUT1 Network DUT2 -[ iperf3-client -> VPP1 ]=======[ VPP2 -> iperf3-server] - -where, - - 1. iperf3 server attaches to VPP2 and LISTENs on VPP2:TCP port 5201. - 2. iperf3 client attaches to VPP1 and opens one or more stream - connections to VPP2:TCP port 5201. - 3. iperf3 client transmits a uni-directional stream as fast as the - VPP Host Stack allows to the iperf3 server for the test duration. - 4. At the end of the test the iperf3 client emits the goodput - measurements for all streams and the sum of all streams. - - Test cases include 1 and 10 Streams with a 20 second test duration - with the VPP Host Stack configured to utilize the Cubic TCP - congestion algorithm. - - Note: iperf3 is single threaded, so it is expected that the 10 stream - test does not show any performance improvement due to - multi-thread/multi-core execution. diff --git a/docs/report/introduction/physical_testbeds.rst b/docs/report/introduction/physical_testbeds.rst index 159728d8ee..24061f9737 100644 --- a/docs/report/introduction/physical_testbeds.rst +++ b/docs/report/introduction/physical_testbeds.rst @@ -71,7 +71,7 @@ SUT servers are populated with the following NIC models: #. NIC-1: x710-DA4 4p10GE Intel. #. NIC-2: xxv710-DA2 2p25GE Intel. -#. NIC-3: mcx556a-edat ConnectX5 2p100GE Mellanox. (Only testbed t27, t28) +#. NIC-3: cx556a-edat ConnectX5 2p100GE Mellanox. (Only testbed t27, t28) #. NIC-4: empty, future expansion. #. NIC-5: empty, future expansion. #. NIC-6: empty, future expansion. @@ -81,7 +81,7 @@ NIC models: #. NIC-1: x710-DA4 4p10GE Intel. #. NIC-2: xxv710-DA2 2p25GE Intel. -#. NIC-3: mcx556a-edat ConnectX5 2p100GE Mellanox. (Only testbed t27, t28) +#. NIC-3: cx556a-edat ConnectX5 2p100GE Mellanox. (Only testbed t27, t28) #. NIC-4: empty, future expansion. #. NIC-5: empty, future expansion. #. NIC-6: x710-DA4 4p10GE Intel. (For self-tests.) @@ -118,7 +118,7 @@ SUT servers are populated with the following NIC models: #. NIC-1: x710-DA4 4p10GE Intel. #. NIC-2: xxv710-DA2 2p25GE Intel. -#. NIC-3: mcx556a-edat ConnectX5 2p100GE Mellanox. (Not used yet.) +#. NIC-3: cx556a-edat ConnectX5 2p100GE Mellanox. (Not used yet.) #. NIC-4: empty, future expansion. #. NIC-5: empty, future expansion. #. NIC-6: empty, future expansion. @@ -128,7 +128,7 @@ NIC models: #. NIC-1: x710-DA4 4p10GE Intel. #. NIC-2: xxv710-DA2 2p25GE Intel. -#. NIC-3: mcx556a-edat ConnectX5 2p100GE Mellanox. (Not used yet.) +#. NIC-3: cx556a-edat ConnectX5 2p100GE Mellanox. (Not used yet.) #. NIC-4: empty, future expansion. #. NIC-5: empty, future expansion. #. NIC-6: x710-DA4 4p10GE Intel. (For self-tests.) diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst index e01868075d..8da1f29608 100644 --- a/docs/report/vpp_performance_tests/csit_release_notes.rst +++ b/docs/report/vpp_performance_tests/csit_release_notes.rst @@ -37,7 +37,7 @@ Changes in |csit-release| generator. HDRH latency data presented in latency packet percentile graphs and in detailed results tables. - - **Mellanox MCX556A-EDAT tests**: Added tests with Mellanox + - **Mellanox CX556A-EDAT tests**: Added tests with Mellanox ConnectX5-2p100GE NICs in 2n-clx testbeds using VPP native rdma driver. @@ -78,7 +78,7 @@ Changes in |csit-release| - **New Intel Xeon Cascadelake Testbeds**: Added performance tests for 2-Node-Cascadelake (2n-clx) testbeds with x710, xxv710 and - mcx556a-edat NIC cards. + cx556a-edat NIC cards. #. PRESENTATION AND ANALYTICS LAYER diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index db1fc5a092..e4a7a0a180 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -368,12 +368,12 @@ class ExecutionChecker(ResultVisitor): groups = re.search(self.REGEX_MRR_MSG_INFO, msg) if not groups or groups.lastindex != 1: - return msg + return u"Test Failed." try: data = groups.group(1).split(u", ") except (AttributeError, IndexError, ValueError, KeyError): - return msg + return u"Test Failed." out_str = u"[" try: @@ -381,7 +381,7 @@ class ExecutionChecker(ResultVisitor): out_str += f"{(float(item) / 1e6):.2f}, " return out_str[:-2] + u"]" except (AttributeError, IndexError, ValueError, KeyError): - return msg + return u"Test Failed." def _get_data_from_perf_test_msg(self, msg): """Get info from message of NDRPDR performance tests. @@ -394,7 +394,7 @@ class ExecutionChecker(ResultVisitor): groups = re.search(self.REGEX_PERF_MSG_INFO, msg) if not groups or groups.lastindex != 10: - return msg + return u"Test Failed." try: data = { @@ -410,7 +410,7 @@ class ExecutionChecker(ResultVisitor): u"pdr_lat_10_2": groups.group(10), } except (AttributeError, IndexError, ValueError, KeyError): - return msg + return u"Test Failed." def _process_lat(in_str_1, in_str_2): """Extract min, avg, max values from latency string. @@ -485,7 +485,7 @@ class ExecutionChecker(ResultVisitor): return out_msg except (AttributeError, IndexError, ValueError, KeyError): - return msg + return u"Test Failed." def _get_testbed(self, msg): """Called when extraction of testbed IP is required. -- cgit 1.2.3-korg