Report: Add Soak tests

Change-Id: I8e5ae26209358d676d1beb758d97a3bfe3a22675
Signed-off-by: Tibor Frank <tifrank@cisco.com>
(cherry picked from commit 96c041ab6ab282ae07c10db1bd7383d63ab7def4)

6 files changed, 113 insertions, 61 deletions

diff --git a/docs/report/index.html.template b/docs/report/index.html.template
index 64312906c2..c89f7525c7 100644
--- a/docs/report/index.html.template
+++ b/docs/report/index.html.template
@@ -20,6 +20,7 @@ CSIT-2001
     vpp_performance_tests/packet_throughput_graphs/index
     vpp_performance_tests/throughput_speedup_multi_core/index
     vpp_performance_tests/packet_latency/index
+    vpp_performance_tests/soak_tests/index
     vpp_performance_tests/reconf_tests/index
     vpp_performance_tests/nf_service_density/index
     vpp_performance_tests/hoststack_testing/index
@@ -28,9 +29,6 @@ CSIT-2001
     vpp_performance_tests/test_environment
     vpp_performance_tests/documentation/index
 
-..
-    vpp_performance_tests/soak_tests/index
-
 .. toctree::
     :maxdepth: 2
     :caption: DPDK Performance
diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst
index 0bbd832ad1..bac932a876 100644
--- a/docs/report/introduction/report_history.rst
+++ b/docs/report/introduction/report_history.rst
@@ -29,6 +29,8 @@ FD.io CSIT-2001 Report history and per .[ww] revision changes are listed below.
 |                |                                                            |
 |                | 5. Added TCP/IP tests                                      |
 |                |                                                            |
+|                | 6. Added Soak tests                                        |
+|                |                                                            |
 +----------------+------------------------------------------------------------+
 | .09            | 1. Added data:                                             |
 |                |                                                            |
diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst
index 79b103e117..b384fb0f0c 100644
--- a/docs/report/vpp_performance_tests/comparisons/index.rst
+++ b/docs/report/vpp_performance_tests/comparisons/index.rst
@@ -8,23 +8,11 @@
 Comparisons
 ===========
 
-..
-    // Alternative Note for 1st Bullet when bad microcode Skx, Clx results are published
-    note::
-    VPP throughput drop in the current release in comparison to previous
-    releases **on Intel Xeon 2n-skx, 3n-skx and 2n-clx testbeds**: VPP
-    performance test data shows lower performance and behaviour
-    inconsistency of these systems following the recent upgrade of
-    processor microcode packages (skx ucode 0x2000064, clx ucode
-    0x500002c) as part of updating Ubuntu 18.04 LTS kernel version.
-    Tested VPP and DPDK applications (L3fwd) are affected. Skx and Clx
-    test data will be corrected in subsequent maintenance report
-    version(s) once the issue is resolved. See :ref:`vpp_known_issues`.
-
 .. toctree::
 
     current_vs_previous_release
     2n-clx_vs_3n-hsw_testbeds
+    soak_vs_ndr
 
 ..
     3n-skx_vs_3n-hsw_testbeds
diff --git a/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst b/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst
index 4e188eb61c..daf62cc0b7 100644
--- a/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst
+++ b/docs/report/vpp_performance_tests/comparisons/soak_vs_ndr.rst
@@ -1,4 +1,4 @@
-_`soak vs ndr comparison`:
+.. _`soak vs ndr comparison`:
 
 Soak Tests vs. NDR Tests
 ------------------------
@@ -10,11 +10,22 @@ in 1-core configurations.
 .. note::
 
     Test results have been generated by
-    `FD.io test executor vpp performance job 2n-skx`_
+    `FD.io test executor vpp performance job 2n-skx`_,
+    `FD.io test executor vpp performance job 2n-clx`_
     with RF result files csit-vpp-perf-|srelease|-\*.zip
     `archived here <../../_static/archive/>`_.
 
 Comparison tables in ASCII and CSV formats:
 
-  - `ASCII Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.txt>`_
-  - `CSV Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.csv>`_
+..
+    2n-skx
+    ~~~~~~
+
+      - `ASCII Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.txt>`_
+      - `CSV Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-skx.csv>`_
+
+2n-clx
+~~~~~~
+
+  - `ASCII Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-clx.txt>`_
+  - `CSV Soak vs. NDR comparison <../../_static/vpp/soak-vs-ndr-2n-clx.csv>`_
diff --git a/docs/report/vpp_performance_tests/index.rst b/docs/report/vpp_performance_tests/index.rst
index 2ddf9c8d82..22177eaf9e 100644
--- a/docs/report/vpp_performance_tests/index.rst
+++ b/docs/report/vpp_performance_tests/index.rst
@@ -8,6 +8,7 @@ VPP Performance
     packet_throughput_graphs/index
     throughput_speedup_multi_core/index
     packet_latency/index
+    soak_tests/index
     reconf_tests/index
     nf_service_density/index
     hoststack_testing/index
@@ -15,6 +16,3 @@ VPP Performance
     throughput_trending
     test_environment
     documentation/index
-
-..
-    soak_tests/index
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 14a53f067b..16bb42db84 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -441,6 +441,11 @@
         - 76  # Reconf
         - 77  # Reconf
 
+    plot-vpp-soak-2n-clx:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 93  # Soak
+        - 94  # Soak
+
     plot-dpdk-throughput-latency-2n-clx:
       csit-dpdk-perf-verify-2001-2n-clx:
         - 15  # NDRPDR sel
@@ -995,6 +1000,37 @@
         - 83  # SOAK
         - 86  # SOAK
 
+    vpp-soak-vs-ndr-2n-clx-ref:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 28  # NDRPDR sel
+        - 29  # NDRPDR sel
+        - 32  # NDRPDR sel
+        - 40  # NDRPDR sel
+        - 43  # NDRPDR sel
+        - 44  # NDRPDR sel
+        - 45  # NDRPDR sel
+        - 47  # NDRPDR sel
+        - 48  # NDRPDR sel
+        - 49  # NDRPDR sel
+    vpp-soak-vs-ndr-2n-clx-cmp:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 93  # SOAK
+        - 94  # Soak
+    vpp-soak-vs-ndr-2n-clx:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 28  # NDRPDR sel
+        - 29  # NDRPDR sel
+        - 32  # NDRPDR sel
+        - 40  # NDRPDR sel
+        - 43  # NDRPDR sel
+        - 44  # NDRPDR sel
+        - 45  # NDRPDR sel
+        - 47  # NDRPDR sel
+        - 48  # NDRPDR sel
+        - 49  # NDRPDR sel
+        - 93  # SOAK
+        - 94  # Soak
+
     vpp-performance-changes-3n-hsw-mrr-h1:
       csit-vpp-perf-verify-1904-3n-hsw:
         - 9   # MRR sel
@@ -2325,8 +2361,6 @@
 #      - 113 # NFV reconf sel
 #      - 114 # NFV reconf sel
 #      - 115 # NFV reconf sel
-#      - 83  # SOAK
-#      - 86  # SOAK
 #    csit-vpp-perf-verify-2001-2n-skx:
 #      - 21  # NDRPDR sel
 #      - 22  # NDRPDR sel
@@ -2386,6 +2420,8 @@
       - 89  # TCP
       - 90  # TCP
       - 91  # TCP
+      - 93  # Soak
+      - 94  # Soak
 
     csit-vpp-perf-verify-1908-3n-tsh:
       - 7   # NDRPDR sel
@@ -2675,7 +2711,7 @@
 ###                               T A B L E S                                ###
 ################################################################################
 
-## VPP Soak vs NDR
+## VPP Soak vs NDR - 2n-skx
 #- type: "table"
 #  title: "VPP Soak vs NDR 2n-skx"
 #  algorithm: "table_soak_vs_ndr"
@@ -2694,6 +2730,25 @@
 #    - "name"
 #    - "parent"
 
+# VPP Soak vs NDR - 2n-clx
+- type: "table"
+  title: "VPP Soak vs NDR 2n-clx"
+  algorithm: "table_soak_vs_ndr"
+  output-file: "{DIR[STATIC,VPP]}/soak-vs-ndr-2n-clx"
+  reference:
+    title: "NDR"
+    data: "vpp-soak-vs-ndr-2n-clx-ref"
+  compare:
+    title: "Soak"
+    data: "vpp-soak-vs-ndr-2n-clx-cmp"
+  data: "vpp-soak-vs-ndr-2n-clx"
+  include-tests: "NDR"
+  filter: "('NDRPDR' or 'SOAK') and not 'NF_DENSITY'"
+  parameters:
+    - "throughput"
+    - "name"
+    - "parent"
+
 ################################################################################
 
 ## VPP Compare NICs 3n-skx 2t1c ndr
@@ -6091,41 +6146,41 @@
 
 ################################################################################
 
-## Soak Test (PLRsearch), boxes
-#- type: "plot"
-#  title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)"
-#  algorithm: "plot_perf_box_name"
-#  output-file-type: ".html"
-#  output-file: "{DIR[STATIC,VPP]}/soak-test-1"
-#  data: "plot-vpp-soak-2n-skx"
-#  include:
-#    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Patch-Soak.64B-2t1c-avf-eth-l2patch-soak"
-#    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Xcbase-Soak.64B-2t1c-avf-eth-l2xcbase-soak"
-#    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-soak"
-#    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Patch-Soak.64B-2t1c-eth-l2patch-soak"
-#    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Xcbase-Soak.64B-2t1c-eth-l2xcbase-soak"
-#    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-eth-l2bdbasemaclrn-soak"
-#  layout:
-#    title: "30 Minutes Soak Test (PLRsearch)"
-#    layout: "plot-soak-throughput"
-#
-## Soak Test (PLRsearch), boxes
-#- type: "plot"
-#  title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)"
-#  algorithm: "plot_perf_box_name"
-#  output-file-type: ".html"
-#  output-file: "{DIR[STATIC,VPP]}/soak-test-2"
-#  data: "plot-vpp-soak-2n-skx"
-#  include:
-#    - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Vppl2Xc-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc-soak"
-#    - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-soak"
-#    - "Tests.Vpp.Perf.Container Memif.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Memif-1Dcr-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr-soak"
-#    - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Avf-Ethip4-Ip4Base-Soak.64B-2t1c-avf-ethip4-ip4base-soak"
-#    - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Ethip4-Ip4Base-Soak.64B-2t1c-ethip4-ip4base-soak"
-#    - "Tests.Vpp.Perf.Ip6.2N1L-25Ge2P1Xxv710-Ethip6-Ip6Base-Soak.78B-2t1c-ethip6-ip6base-soak"
-#  layout:
-#    title: "30 Minutes Soak Test (PLRsearch)"
-#    layout: "plot-soak-throughput"
+# Soak Test (PLRsearch), boxes - 2n-clx
+- type: "plot"
+  title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)"
+  algorithm: "plot_perf_box_name"
+  output-file-type: ".html"
+  output-file: "{DIR[STATIC,VPP]}/soak-test-1"
+  data: "plot-vpp-soak-2n-clx"
+  include:
+    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Xcbase-Soak.64B-2t1c-avf-eth-l2xcbase-soak"
+    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-soak"
+    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Xcbase-Soak.64B-2t1c-eth-l2xcbase-soak"
+    - "Tests.Vpp.Perf.L2.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Soak.64B-2t1c-eth-l2bdbasemaclrn-soak"
+    - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Avf-Ethip4-Ip4Base-Soak.64B-2t1c-avf-ethip4-ip4base-soak"
+    - "Tests.Vpp.Perf.Ip4.2N1L-25Ge2P1Xxv710-Ethip4-Ip4Base-Soak.64B-2t1c-ethip4-ip4base-soak"
+  layout:
+    title: "30 Minutes Soak Test (PLRsearch)"
+    layout: "plot-soak-throughput"
+
+# Soak Test (PLRsearch), boxes - 2n-clx
+- type: "plot"
+  title: "VPP Critical rate: 30 Minutes Soak Test (PLRsearch)"
+  algorithm: "plot_perf_box_name"
+  output-file-type: ".html"
+  output-file: "{DIR[STATIC,VPP]}/soak-test-2"
+  data: "plot-vpp-soak-2n-clx"
+  include:
+    - "Tests.Vpp.Perf.Ip6.2N1L-25Ge2P1Xxv710-Avf-Ethip6-Ip6Base-Soak.78B-2t1c-avf-ethip6-ip6base-soak"
+    - "Tests.Vpp.Perf.Ip6.2N1L-25Ge2P1Xxv710-Ethip6-Ip6Base-Soak.78B-2t1c-ethip6-ip6base-soak"
+    - "Tests.Vpp.Perf.Container Memif.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Eth-2Memif-1Dcr-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-eth-2memif-1dcr-soak"
+    - "Tests.Vpp.Perf.Container Memif.2N1L-25Ge2P1Xxv710-Eth-L2Bdbasemaclrn-Eth-2Memif-1Dcr-Soak.64B-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr-soak"
+    - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-soak"
+    - "Tests.Vpp.Perf.Vm Vhost.2N1L-25Ge2P1Xxv710-Avf-Eth-L2Bdbasemaclrn-Eth-2Vhostvr1024-1Vm-Vppl2Xc-Soak.64B-2t1c-avf-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc-soak"
+  layout:
+    title: "30 Minutes Soak Test (PLRsearch)"
+    layout: "plot-soak-throughput"
 
 ###############################################################################