diff options
9 files changed, 247 insertions, 10 deletions
diff --git a/docs/report/index.html.template b/docs/report/index.html.template index f6fcc76d15..64312906c2 100644 --- a/docs/report/index.html.template +++ b/docs/report/index.html.template @@ -22,13 +22,13 @@ CSIT-2001 vpp_performance_tests/packet_latency/index vpp_performance_tests/reconf_tests/index vpp_performance_tests/nf_service_density/index + vpp_performance_tests/hoststack_testing/index vpp_performance_tests/comparisons/index vpp_performance_tests/throughput_trending vpp_performance_tests/test_environment vpp_performance_tests/documentation/index .. - vpp_performance_tests/hoststack_testing/index vpp_performance_tests/soak_tests/index .. toctree:: diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst index b2199eda84..e07fc5339d 100644 --- a/docs/report/introduction/report_history.rst +++ b/docs/report/introduction/report_history.rst @@ -23,6 +23,9 @@ FD.io CSIT-2001 Report history and per .[ww] revision changes are listed below. | | | | | - 2n-clx | | | | +| | 3. Added Hoststack tests | +| | | +| | 4. Edited Test Methodology --> Hoststack Testing | | | | +----------------+------------------------------------------------------------+ | .09 | 1. Added data: | diff --git a/docs/report/vpp_performance_tests/hoststack_testing/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/index.rst index 46390b5dc5..fef315ac74 100644 --- a/docs/report/vpp_performance_tests/hoststack_testing/index.rst +++ b/docs/report/vpp_performance_tests/hoststack_testing/index.rst @@ -3,8 +3,8 @@ Hoststack Testing .. toctree:: - http_server_performance/index - -.. iperf3/index quic/index + +.. + http_server_performance/index diff --git a/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst index b871f582d5..83c9d6335c 100644 --- a/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst +++ b/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst @@ -1,2 +1,75 @@ + +.. raw:: latex + + \clearpage + +.. raw:: html + + <script type="text/javascript"> + + function getDocHeight(doc) { + doc = doc || document; + var body = doc.body, html = doc.documentElement; + var height = Math.max( body.scrollHeight, body.offsetHeight, + html.clientHeight, html.scrollHeight, html.offsetHeight ); + return height; + } + + function setIframeHeight(id) { + var ifrm = document.getElementById(id); + var doc = ifrm.contentDocument? ifrm.contentDocument: + ifrm.contentWindow.document; + ifrm.style.visibility = 'hidden'; + ifrm.style.height = "10px"; // reset to minimal height ... + // IE opt. for bing/msn needs a bit added or scrollbar appears + ifrm.style.height = getDocHeight( doc ) + 4 + "px"; + ifrm.style.visibility = 'visible'; + } + + </script> + TCP/IP with iperf3 ------------------- +~~~~~~~~~~~~~~~~~~ + +.. todo:: + Add introduction + +.. raw:: latex + + \clearpage + +9000b-1t1c-xl710-base-scale +--------------------------- + +.. raw:: html + + <iframe id="ifrm01" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps.html"></iframe> + +.. raw:: latex + + \begin{figure}[H] + \centering + \graphicspath{{../../_build/_static/vpp/}} + \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps} + \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps} + \end{figure} + +.. raw:: latex + + \clearpage + +9000b-1t1c-xl710-nsim-base-scale +-------------------------------- + +.. raw:: html + + <iframe id="ifrm02" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps.html"></iframe> + +.. raw:: latex + + \begin{figure}[H] + \centering + \graphicspath{{../../_build/_static/vpp/}} + \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps} + \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps} + \end{figure} diff --git a/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst index 9cf6befb61..09a89a91a7 100644 --- a/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst +++ b/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst @@ -1,2 +1,55 @@ + +.. raw:: latex + + \clearpage + +.. raw:: html + + <script type="text/javascript"> + + function getDocHeight(doc) { + doc = doc || document; + var body = doc.body, html = doc.documentElement; + var height = Math.max( body.scrollHeight, body.offsetHeight, + html.clientHeight, html.scrollHeight, html.offsetHeight ); + return height; + } + + function setIframeHeight(id) { + var ifrm = document.getElementById(id); + var doc = ifrm.contentDocument? ifrm.contentDocument: + ifrm.contentWindow.document; + ifrm.style.visibility = 'hidden'; + ifrm.style.height = "10px"; // reset to minimal height ... + // IE opt. for bing/msn needs a bit added or scrollbar appears + ifrm.style.height = getDocHeight( doc ) + 4 + "px"; + ifrm.style.visibility = 'visible'; + } + + </script> + QUIC(picotls)/UDP/IP with vpp_echo ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. todo:: + Add introduction + +.. raw:: latex + + \clearpage + +9000b-1t1c-xl710-base-scale +--------------------------- + +.. raw:: html + + <iframe id="ifrm01" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps.html"></iframe> + +.. raw:: latex + + \begin{figure}[H] + \centering + \graphicspath{{../../_build/_static/vpp/}} + \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps} + \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps} + \end{figure} diff --git a/docs/report/vpp_performance_tests/index.rst b/docs/report/vpp_performance_tests/index.rst index 36aed84c99..2ddf9c8d82 100644 --- a/docs/report/vpp_performance_tests/index.rst +++ b/docs/report/vpp_performance_tests/index.rst @@ -10,11 +10,11 @@ VPP Performance packet_latency/index reconf_tests/index nf_service_density/index + hoststack_testing/index comparisons/index throughput_trending test_environment documentation/index .. - hoststack_testing/index soak_tests/index diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py index 5c9698912f..40993cd477 100644 --- a/resources/tools/presentation/generator_plots.py +++ b/resources/tools/presentation/generator_plots.py @@ -622,13 +622,14 @@ def plot_perf_box_name(plot, input_data): f"{plot.get(u'title', u'')}." ) data = input_data.filter_tests_by_name( - plot, params=[u"throughput", u"parent", u"tags", u"type"]) + plot, params=[u"throughput", u"result", u"parent", u"tags", u"type"]) if data is None: logging.error(u"No data.") return # Prepare the data for the plot y_vals = OrderedDict() + test_type = u"" for job in data: for build in job: for test in build: @@ -639,13 +640,30 @@ def plot_perf_box_name(plot, input_data): u"-pdr" in plot.get(u"title", u"").lower()): y_vals[test[u"parent"]].\ append(test[u"throughput"][u"PDR"][u"LOWER"]) + test_type = u"NDRPDR" elif (test[u"type"] in (u"NDRPDR", ) and u"-ndr" in plot.get(u"title", u"").lower()): y_vals[test[u"parent"]]. \ append(test[u"throughput"][u"NDR"][u"LOWER"]) + test_type = u"NDRPDR" elif test[u"type"] in (u"SOAK", ): y_vals[test[u"parent"]].\ append(test[u"throughput"][u"LOWER"]) + test_type = u"SOAK" + elif test[u"type"] in (u"HOSTSTACK", ): + if u"LDPRELOAD" in test[u"tags"]: + y_vals[test[u"parent"]].append( + float(test[u"result"][u"bits_per_second"]) / 1e3 + ) + elif u"VPPECHO" in test[u"tags"]: + y_vals[test[u"parent"]].append( + (float(test[u"result"][u"client"][u"tx_data"]) + * 8 / 1e3) / + ((float(test[u"result"][u"client"][u"time"]) + + float(test[u"result"][u"server"][u"time"])) / + 2) + ) + test_type = u"HOSTSTACK" else: continue except (KeyError, TypeError): @@ -696,7 +714,10 @@ def plot_perf_box_name(plot, input_data): # Create plot layout = deepcopy(plot[u"layout"]) if layout.get(u"title", None): - layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}" + if test_type in (u"HOSTSTACK", ): + layout[u"title"] = f"<b>Bandwidth:</b> {layout[u'title']}" + else: + layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}" if y_max: layout[u"yaxis"][u"range"] = [0, max(y_max)] plpl = plgo.Figure(data=traces, layout=layout) diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index f7869d889a..c43d46014e 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -29,6 +29,7 @@ from os import remove from datetime import datetime as dt from datetime import timedelta from json import loads +from json.decoder import JSONDecodeError import hdrh.histogram import hdrh.codec @@ -871,6 +872,40 @@ class ExecutionChecker(ResultVisitor): return latency, u"FAIL" + @staticmethod + def _get_hoststack_data(msg, tags): + """Get data from the hoststack test message. + + :param msg: The test message to be parsed. + :param tags: Test tags. + :type msg: str + :type tags: list + :returns: Parsed data as a JSON dict and the status (PASS/FAIL). + :rtype: tuple(dict, str) + """ + result = dict() + status = u"FAIL" + + msg = msg.replace(u"'", u'"').replace(u" ", u"") + if u"LDPRELOAD" in tags: + try: + result = loads(msg) + status = u"PASS" + except JSONDecodeError: + pass + elif u"VPPECHO" in tags: + try: + msg_lst = msg.replace(u"}{", u"} {").split(u" ") + result = dict( + client=loads(msg_lst[0]), + server=loads(msg_lst[1]) + ) + status = u"PASS" + except (JSONDecodeError, IndexError): + pass + + return result, status + def visit_suite(self, suite): """Implements traversing through the suite and its direct children. @@ -1040,6 +1075,10 @@ class ExecutionChecker(ResultVisitor): test_result[u"type"] = u"SOAK" test_result[u"throughput"], test_result[u"status"] = \ self._get_plr_throughput(test.message) + elif u"HOSTSTACK" in tags: + test_result[u"type"] = u"HOSTSTACK" + test_result[u"result"], test_result[u"status"] = \ + self._get_hoststack_data(test.message, tags) elif u"TCP" in tags: test_result[u"type"] = u"TCP" groups = re.search(self.REGEX_TCP, test.message) @@ -1152,7 +1191,8 @@ class ExecutionChecker(ResultVisitor): test_kw.name.count(u"Show Runtime Counters On All Duts"): self._msg_type = u"test-show-runtime" self._sh_run_counter += 1 - elif test_kw.name.count(u"Install Dpdk Test") and not self._version: + elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \ + not self._version: self._msg_type = u"dpdk-version" else: return diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 46e5ce1ce1..c334f1aa9a 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -247,6 +247,11 @@ - 23 # NDRPDR sel - 24 # NDRPDR sel + plot-vpp-hoststack-3n-hsw: + csit-vpp-perf-verify-2001-3n-hsw: + - 87 # hoststack + - 88 # hoststack + plot-dpdk-hdrh-lat-3n-hsw: csit-dpdk-perf-verify-2001-3n-hsw: - 16 # NDRPDR sel @@ -2156,6 +2161,8 @@ - 82 # NDRPDR full - 84 # NDRPDR full - 85 # NDRPDR full + - 87 # hoststack + - 88 # hoststack # csit-vpp-perf-verify-1904-3n-skx: # - 7 # MRR sel @@ -5913,6 +5920,46 @@ ### P L O T S ### ################################################################################ +# Hoststack tests 3n-hsw + +- type: "plot" + title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3" + algorithm: "plot_perf_box_name" + output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps" + data: "plot-vpp-hoststack-3n-hsw" + include: + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpbase-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpbase-ldpreload-iperf3-bps" + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpscale1Cl10S-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps" + layout: + title: "3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3" + layout: "plot-hoststack" + +- type: "plot" + title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3" + algorithm: "plot_perf_box_name" + output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps" + data: "plot-vpp-hoststack-3n-hsw" + include: + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpbase-Nsim-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps" + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpscale1Cl10S-Nsim-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps" + layout: + title: "3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3" + layout: "plot-hoststack" + +- type: "plot" + title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho" + algorithm: "plot_perf_box_name" + output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps" + data: "plot-vpp-hoststack-3n-hsw" + include: + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicbase-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicbase-vppecho-bps" + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale1Cl10S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale1cl10s-vppecho-bps" + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale10Cl1S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale10cl1s-vppecho-bps" + - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale10Cl10S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale10cl10s-vppecho-bps" + layout: + title: "3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho" + layout: "plot-hoststack" + ## Plots VPP HTTP Server Performance #- type: "plot" # title: "VPP HTTP Server Performance" |