PAL: Process Hoststack data

Change-Id: Ib87ad83f3bbb7e71b51b91b9b3e8dc963a76a047
Signed-off-by: Tibor Frank <tifrank@cisco.com>

9 files changed, 247 insertions, 10 deletions

diff --git a/docs/report/index.html.template b/docs/report/index.html.template
index f6fcc76d15..64312906c2 100644
--- a/docs/report/index.html.template
+++ b/docs/report/index.html.template
@@ -22,13 +22,13 @@ CSIT-2001
     vpp_performance_tests/packet_latency/index
     vpp_performance_tests/reconf_tests/index
     vpp_performance_tests/nf_service_density/index
+    vpp_performance_tests/hoststack_testing/index
     vpp_performance_tests/comparisons/index
     vpp_performance_tests/throughput_trending
     vpp_performance_tests/test_environment
     vpp_performance_tests/documentation/index
 
 ..
-    vpp_performance_tests/hoststack_testing/index
     vpp_performance_tests/soak_tests/index
 
 .. toctree::
diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst
index b2199eda84..e07fc5339d 100644
--- a/docs/report/introduction/report_history.rst
+++ b/docs/report/introduction/report_history.rst
@@ -23,6 +23,9 @@ FD.io CSIT-2001 Report history and per .[ww] revision changes are listed below.
 |                |                                                            |
 |                |    - 2n-clx                                                |
 |                |                                                            |
+|                | 3. Added Hoststack tests                                   |
+|                |                                                            |
+|                | 4. Edited Test Methodology --> Hoststack Testing           |
 |                |                                                            |
 +----------------+------------------------------------------------------------+
 | .09            | 1. Added data:                                             |
diff --git a/docs/report/vpp_performance_tests/hoststack_testing/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/index.rst
index 46390b5dc5..fef315ac74 100644
--- a/docs/report/vpp_performance_tests/hoststack_testing/index.rst
+++ b/docs/report/vpp_performance_tests/hoststack_testing/index.rst
@@ -3,8 +3,8 @@ Hoststack Testing
 
 .. toctree::
 
-    http_server_performance/index
-
-..
     iperf3/index
     quic/index
+
+..
+    http_server_performance/index
diff --git a/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst
index b871f582d5..83c9d6335c 100644
--- a/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst
+++ b/docs/report/vpp_performance_tests/hoststack_testing/iperf3/index.rst
@@ -1,2 +1,75 @@
+
+.. raw:: latex
+
+    \clearpage
+
+.. raw:: html
+
+    <script type="text/javascript">
+
+        function getDocHeight(doc) {
+            doc = doc || document;
+            var body = doc.body, html = doc.documentElement;
+            var height = Math.max( body.scrollHeight, body.offsetHeight,
+                html.clientHeight, html.scrollHeight, html.offsetHeight );
+            return height;
+        }
+
+        function setIframeHeight(id) {
+            var ifrm = document.getElementById(id);
+            var doc = ifrm.contentDocument? ifrm.contentDocument:
+                ifrm.contentWindow.document;
+            ifrm.style.visibility = 'hidden';
+            ifrm.style.height = "10px"; // reset to minimal height ...
+            // IE opt. for bing/msn needs a bit added or scrollbar appears
+            ifrm.style.height = getDocHeight( doc ) + 4 + "px";
+            ifrm.style.visibility = 'visible';
+        }
+
+    </script>
+
 TCP/IP with iperf3
-------------------
+~~~~~~~~~~~~~~~~~~
+
+.. todo::
+    Add introduction
+
+.. raw:: latex
+
+    \clearpage
+
+9000b-1t1c-xl710-base-scale
+---------------------------
+
+.. raw:: html
+
+    <iframe id="ifrm01" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps.html"></iframe>
+
+.. raw:: latex
+
+    \begin{figure}[H]
+        \centering
+            \graphicspath{{../../_build/_static/vpp/}}
+            \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps}
+            \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps}
+    \end{figure}
+
+.. raw:: latex
+
+    \clearpage
+
+9000b-1t1c-xl710-nsim-base-scale
+--------------------------------
+
+.. raw:: html
+
+    <iframe id="ifrm02" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps.html"></iframe>
+
+.. raw:: latex
+
+    \begin{figure}[H]
+        \centering
+            \graphicspath{{../../_build/_static/vpp/}}
+            \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps}
+            \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps}
+    \end{figure}
diff --git a/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst b/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst
index 9cf6befb61..09a89a91a7 100644
--- a/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst
+++ b/docs/report/vpp_performance_tests/hoststack_testing/quic/index.rst
@@ -1,2 +1,55 @@
+
+.. raw:: latex
+
+    \clearpage
+
+.. raw:: html
+
+    <script type="text/javascript">
+
+        function getDocHeight(doc) {
+            doc = doc || document;
+            var body = doc.body, html = doc.documentElement;
+            var height = Math.max( body.scrollHeight, body.offsetHeight,
+                html.clientHeight, html.scrollHeight, html.offsetHeight );
+            return height;
+        }
+
+        function setIframeHeight(id) {
+            var ifrm = document.getElementById(id);
+            var doc = ifrm.contentDocument? ifrm.contentDocument:
+                ifrm.contentWindow.document;
+            ifrm.style.visibility = 'hidden';
+            ifrm.style.height = "10px"; // reset to minimal height ...
+            // IE opt. for bing/msn needs a bit added or scrollbar appears
+            ifrm.style.height = getDocHeight( doc ) + 4 + "px";
+            ifrm.style.visibility = 'visible';
+        }
+
+    </script>
+
 QUIC(picotls)/UDP/IP with vpp_echo
-----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. todo::
+    Add introduction
+
+.. raw:: latex
+
+    \clearpage
+
+9000b-1t1c-xl710-base-scale
+---------------------------
+
+.. raw:: html
+
+    <iframe id="ifrm01" onload="setIframeHeight(this.id)" width="700" frameborder="0" scrolling="no" src="../../../_static/vpp/3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps.html"></iframe>
+
+.. raw:: latex
+
+    \begin{figure}[H]
+        \centering
+            \graphicspath{{../../_build/_static/vpp/}}
+            \includegraphics[clip, trim=0cm 0cm 5cm 0cm, width=0.70\textwidth]{3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps}
+            \label{fig:3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps}
+    \end{figure}
diff --git a/docs/report/vpp_performance_tests/index.rst b/docs/report/vpp_performance_tests/index.rst
index 36aed84c99..2ddf9c8d82 100644
--- a/docs/report/vpp_performance_tests/index.rst
+++ b/docs/report/vpp_performance_tests/index.rst
@@ -10,11 +10,11 @@ VPP Performance
     packet_latency/index
     reconf_tests/index
     nf_service_density/index
+    hoststack_testing/index
     comparisons/index
     throughput_trending
     test_environment
     documentation/index
 
 ..
-    hoststack_testing/index
     soak_tests/index
diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 5c9698912f..40993cd477 100644
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -622,13 +622,14 @@ def plot_perf_box_name(plot, input_data):
         f"{plot.get(u'title', u'')}."
     )
     data = input_data.filter_tests_by_name(
-        plot, params=[u"throughput", u"parent", u"tags", u"type"])
+        plot, params=[u"throughput", u"result", u"parent", u"tags", u"type"])
     if data is None:
         logging.error(u"No data.")
         return
 
     # Prepare the data for the plot
     y_vals = OrderedDict()
+    test_type = u""
     for job in data:
         for build in job:
             for test in build:
@@ -639,13 +640,30 @@ def plot_perf_box_name(plot, input_data):
                             u"-pdr" in plot.get(u"title", u"").lower()):
                         y_vals[test[u"parent"]].\
                             append(test[u"throughput"][u"PDR"][u"LOWER"])
+                        test_type = u"NDRPDR"
                     elif (test[u"type"] in (u"NDRPDR", ) and
                           u"-ndr" in plot.get(u"title", u"").lower()):
                         y_vals[test[u"parent"]]. \
                             append(test[u"throughput"][u"NDR"][u"LOWER"])
+                        test_type = u"NDRPDR"
                     elif test[u"type"] in (u"SOAK", ):
                         y_vals[test[u"parent"]].\
                             append(test[u"throughput"][u"LOWER"])
+                        test_type = u"SOAK"
+                    elif test[u"type"] in (u"HOSTSTACK", ):
+                        if u"LDPRELOAD" in test[u"tags"]:
+                            y_vals[test[u"parent"]].append(
+                                float(test[u"result"][u"bits_per_second"]) / 1e3
+                            )
+                        elif u"VPPECHO" in test[u"tags"]:
+                            y_vals[test[u"parent"]].append(
+                                (float(test[u"result"][u"client"][u"tx_data"])
+                                 * 8 / 1e3) /
+                                ((float(test[u"result"][u"client"][u"time"]) +
+                                  float(test[u"result"][u"server"][u"time"])) /
+                                 2)
+                            )
+                        test_type = u"HOSTSTACK"
                     else:
                         continue
                 except (KeyError, TypeError):
@@ -696,7 +714,10 @@ def plot_perf_box_name(plot, input_data):
         # Create plot
         layout = deepcopy(plot[u"layout"])
         if layout.get(u"title", None):
-            layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"
+            if test_type in (u"HOSTSTACK", ):
+                layout[u"title"] = f"<b>Bandwidth:</b> {layout[u'title']}"
+            else:
+                layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"
         if y_max:
             layout[u"yaxis"][u"range"] = [0, max(y_max)]
         plpl = plgo.Figure(data=traces, layout=layout)
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index f7869d889a..c43d46014e 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -29,6 +29,7 @@ from os import remove
 from datetime import datetime as dt
 from datetime import timedelta
 from json import loads
+from json.decoder import JSONDecodeError
 
 import hdrh.histogram
 import hdrh.codec
@@ -871,6 +872,40 @@ class ExecutionChecker(ResultVisitor):
 
         return latency, u"FAIL"
 
+    @staticmethod
+    def _get_hoststack_data(msg, tags):
+        """Get data from the hoststack test message.
+
+        :param msg: The test message to be parsed.
+        :param tags: Test tags.
+        :type msg: str
+        :type tags: list
+        :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
+        :rtype: tuple(dict, str)
+        """
+        result = dict()
+        status = u"FAIL"
+
+        msg = msg.replace(u"'", u'"').replace(u" ", u"")
+        if u"LDPRELOAD" in tags:
+            try:
+                result = loads(msg)
+                status = u"PASS"
+            except JSONDecodeError:
+                pass
+        elif u"VPPECHO" in tags:
+            try:
+                msg_lst = msg.replace(u"}{", u"} {").split(u" ")
+                result = dict(
+                    client=loads(msg_lst[0]),
+                    server=loads(msg_lst[1])
+                )
+                status = u"PASS"
+            except (JSONDecodeError, IndexError):
+                pass
+
+        return result, status
+
     def visit_suite(self, suite):
         """Implements traversing through the suite and its direct children.
 
@@ -1040,6 +1075,10 @@ class ExecutionChecker(ResultVisitor):
                 test_result[u"type"] = u"SOAK"
                 test_result[u"throughput"], test_result[u"status"] = \
                     self._get_plr_throughput(test.message)
+            elif u"HOSTSTACK" in tags:
+                test_result[u"type"] = u"HOSTSTACK"
+                test_result[u"result"], test_result[u"status"] = \
+                    self._get_hoststack_data(test.message, tags)
             elif u"TCP" in tags:
                 test_result[u"type"] = u"TCP"
                 groups = re.search(self.REGEX_TCP, test.message)
@@ -1152,7 +1191,8 @@ class ExecutionChecker(ResultVisitor):
                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
             self._msg_type = u"test-show-runtime"
             self._sh_run_counter += 1
-        elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
+        elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
+                not self._version:
             self._msg_type = u"dpdk-version"
         else:
             return
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 46e5ce1ce1..c334f1aa9a 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -247,6 +247,11 @@
         - 23  # NDRPDR sel
         - 24  # NDRPDR sel
 
+    plot-vpp-hoststack-3n-hsw:
+      csit-vpp-perf-verify-2001-3n-hsw:
+        - 87  # hoststack
+        - 88  # hoststack
+
     plot-dpdk-hdrh-lat-3n-hsw:
       csit-dpdk-perf-verify-2001-3n-hsw:
         - 16  # NDRPDR sel
@@ -2156,6 +2161,8 @@
       - 82  # NDRPDR full
       - 84  # NDRPDR full
       - 85  # NDRPDR full
+      - 87  # hoststack
+      - 88  # hoststack
 
 #    csit-vpp-perf-verify-1904-3n-skx:
 #      - 7   # MRR sel
@@ -5913,6 +5920,46 @@
 ###                                P L O T S                                 ###
 ################################################################################
 
+# Hoststack tests 3n-hsw
+
+- type: "plot"
+  title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3"
+  algorithm: "plot_perf_box_name"
+  output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3-bps"
+  data: "plot-vpp-hoststack-3n-hsw"
+  include:
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpbase-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpbase-ldpreload-iperf3-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpscale1Cl10S-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps"
+  layout:
+    title: "3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-ldpreload-iperf3"
+    layout: "plot-hoststack"
+
+- type: "plot"
+  title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3"
+  algorithm: "plot_perf_box_name"
+  output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps"
+  data: "plot-vpp-hoststack-3n-hsw"
+  include:
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpbase-Nsim-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Tcpscale1Cl10S-Nsim-Ldpreload-Iperf3-Bps.9000B-1t1c-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps"
+  layout:
+    title: "3n-hsw-xl710-9000b-1t1c-eth-ip4tcp-nsim-ldpreload-iperf3"
+    layout: "plot-hoststack"
+
+- type: "plot"
+  title: "Bandwidth: 3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho"
+  algorithm: "plot_perf_box_name"
+  output-file: "{DIR[STATIC,VPP]}/3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho-bps"
+  data: "plot-vpp-hoststack-3n-hsw"
+  include:
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicbase-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicbase-vppecho-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale1Cl10S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale1cl10s-vppecho-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale10Cl1S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale10cl1s-vppecho-bps"
+    - "Tests.Vpp.Perf.Hoststack.40Ge2P1Xl710-Eth-Ip4Udpquicscale10Cl10S-Vppecho-Bps.9000B-1t1c-eth-ip4udpquicscale10cl10s-vppecho-bps"
+  layout:
+    title: "3n-hsw-xl710-9000b-1t1c-eth-ip4udpquic-vppecho"
+    layout: "plot-hoststack"
+
 ## Plots VPP HTTP Server Performance
 #- type: "plot"
 #  title: "VPP HTTP Server Performance"