PAL: Detailed test results

Change-Id: I75ebad7874602dbc136440a3b4c6f94b095728c4
Signed-off-by: Tibor Frank <tifrank@cisco.com>

7 files changed, 263 insertions, 11 deletions

diff --git a/docs/report/detailed_test_results/vpp_performance_results/index.rst b/docs/report/detailed_test_results/vpp_performance_results/index.rst
index 0459da69ab..b6cdd64a29 100644
--- a/docs/report/detailed_test_results/vpp_performance_results/index.rst
+++ b/docs/report/detailed_test_results/vpp_performance_results/index.rst
@@ -7,8 +7,8 @@ VPP Throughput
     ../vpp_performance_results_3n_tsh/index
     ../vpp_performance_results_2n_dnv/index
     ../vpp_performance_results_3n_dnv/index
-    ../vpp_performance_results_2n_clx/index
 
 ..
+    ../vpp_performance_results_2n_clx/index
     ../vpp_performance_results_2n_skx/index
     ../vpp_performance_results_3n_skx/index
diff --git a/docs/report/test_configuration/vpp_performance_configuration/index.rst b/docs/report/test_configuration/vpp_performance_configuration/index.rst
index 7b4b9ab460..7085c8951d 100644
--- a/docs/report/test_configuration/vpp_performance_configuration/index.rst
+++ b/docs/report/test_configuration/vpp_performance_configuration/index.rst
@@ -7,8 +7,8 @@ VPP Throughput
     ../vpp_performance_configuration_3n_tsh/index
     ../vpp_performance_configuration_2n_dnv/index
     ../vpp_performance_configuration_3n_dnv/index
-    ../vpp_performance_configuration_2n_clx/index
 
 ..
+    ../vpp_performance_configuration_2n_clx/index
     ../vpp_performance_configuration_2n_skx/index
     ../vpp_performance_configuration_3n_skx/index
diff --git a/docs/report/test_operational_data/vpp_performance_operational_data/index.rst b/docs/report/test_operational_data/vpp_performance_operational_data/index.rst
index 7eca0de3b6..e1c4748529 100644
--- a/docs/report/test_operational_data/vpp_performance_operational_data/index.rst
+++ b/docs/report/test_operational_data/vpp_performance_operational_data/index.rst
@@ -7,8 +7,8 @@ VPP Throughput
     ../vpp_performance_operational_data_3n_tsh/index
     ../vpp_performance_operational_data_2n_dnv/index
     ../vpp_performance_operational_data_3n_dnv/index
-    ../vpp_performance_operational_data_2n_clx/index
 
 ..
+    ../vpp_performance_operational_data_2n_clx/index
     ../vpp_performance_operational_data_2n_skx/index
     ../vpp_performance_operational_data_3n_skx/index
diff --git a/docs/report/vpp_performance_tests/comparisons/2n-clx_vs_3n-hsw_testbeds.rst b/docs/report/vpp_performance_tests/comparisons/2n-clx_vs_3n-hsw_testbeds.rst
new file mode 100644
index 0000000000..44f2eb91c9
--- /dev/null
+++ b/docs/report/vpp_performance_tests/comparisons/2n-clx_vs_3n-hsw_testbeds.rst
@@ -0,0 +1,55 @@
+2n-clx vs. 3n-Hsw Testbeds
+--------------------------
+
+Relative comparison of |vpp-release| packet throughput (NDR, PDR and
+MRR) is calculated for the same tests executed on 3-Node Skylake (3n-
+skx) and 3-Node Haswell (3n-hsw) physical testbed types, in 1-core,
+2-core and 4-core configurations.
+
+.. note::
+
+    Test results have been generated by
+    `FD.io test executor vpp performance job 3n-hsw`_ and
+    `FD.io test executor vpp performance job 2n-clx`_
+    with RF result
+    files csit-vpp-perf-|srelease|-\*.zip
+    `archived here <../../_static/archive/>`_.
+
+NDR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 1c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr.html>`_
+  - `HTML 2c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr.html>`_
+  - `ASCII 1c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr.txt>`_
+  - `ASCII 2c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr.txt>`_
+  - `CSV 1c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr.csv>`_
+  - `CSV 2c NDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr.csv>`_
+
+PDR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 1c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr.html>`_
+  - `HTML 2c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr.html>`_
+  - `ASCII 1c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr.txt>`_
+  - `ASCII 2c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr.txt>`_
+  - `CSV 1c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr.csv>`_
+  - `CSV 2c PDR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr.csv>`_
+
+MRR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+  - `HTML 1c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr.html>`_
+  - `HTML 2c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr.html>`_
+  - `HTML 4c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr.html>`_
+  - `ASCII 1c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr.txt>`_
+  - `ASCII 2c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr.txt>`_
+  - `ASCII 4c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr.txt>`_
+  - `CSV 1c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr.csv>`_
+  - `CSV 2c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr.csv>`_
+  - `CSV 4c MRR comparison <../../_static/vpp/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr.csv>`_
diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst
index 4511a3e578..79b103e117 100644
--- a/docs/report/vpp_performance_tests/comparisons/index.rst
+++ b/docs/report/vpp_performance_tests/comparisons/index.rst
@@ -24,6 +24,7 @@ Comparisons
 .. toctree::
 
     current_vs_previous_release
+    2n-clx_vs_3n-hsw_testbeds
 
 ..
     3n-skx_vs_3n-hsw_testbeds
diff --git a/resources/tools/presentation/generator_files.py b/resources/tools/presentation/generator_files.py
index 47683ea96b..e1ddef013d 100644
--- a/resources/tools/presentation/generator_files.py
+++ b/resources/tools/presentation/generator_files.py
@@ -133,6 +133,7 @@ def file_test_results(file_spec, input_data, frmt=u"rst"):
     if suites.empty:
         return
     suites = input_data.merge_data(suites)
+    suites.sort_index(inplace=True)
 
     file_name = u""
     for suite_longname, suite in suites.items():
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 49ecae7c3a..64b61fae9c 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -380,8 +380,9 @@
         - 29  # NDRPDR sel
         - 32  # NDRPDR sel
         - 40  # NDRPDR sel
+        - 43  # NDRPDR sel
         - 44  # NDRPDR sel
-
+        - 45  # NDRPDR sel
 
     plot-vpp-hdrh-lat-2n-clx:
       csit-vpp-perf-verify-2001-2n-clx:
@@ -394,8 +395,9 @@
         - 29  # NDRPDR sel
         - 32  # NDRPDR sel
         - 40  # NDRPDR sel
+        - 43  # NDRPDR sel
         - 44  # NDRPDR sel
-
+        - 45  # NDRPDR sel
 
     plot-vpp-nfv-2n-clx-mrr:
       csit-vpp-perf-verify-2001-2n-clx:
@@ -1210,12 +1212,39 @@
         - 29  # NDRPDR sel
         - 32  # NDRPDR sel
         - 40  # NDRPDR sel
+        - 43  # NDRPDR sel
         - 44  # NDRPDR sel
-
+        - 45  # NDRPDR sel
     vpp-compare-testbeds-skx-clx:
       - "vpp-compare-testbeds-skx-clx-ref"
       - "vpp-compare-testbeds-skx-clx-cmp"
 
+    vpp-compare-testbeds-hsw-clx-ref:
+      csit-vpp-perf-verify-2001-3n-hsw:
+        - 21  # NDRPDR sel
+        - 22  # NDRPDR sel
+        - 23  # NDRPDR sel
+        - 24  # NDRPDR sel
+        - 25  # NDRPDR sel
+        - 29  # NDRPDR sel
+        - 30  # NDRPDR sel
+        - 31  # NDRPDR sel
+        - 32  # NDRPDR sel
+        - 34  # NDRPDR sel
+    vpp-compare-testbeds-hsw-clx-cmp:
+      # TODO: Add the data
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 28  # NDRPDR sel
+        - 29  # NDRPDR sel
+        - 32  # NDRPDR sel
+        - 40  # NDRPDR sel
+        - 43  # NDRPDR sel
+        - 44  # NDRPDR sel
+        - 45  # NDRPDR sel
+    vpp-compare-testbeds-hsw-clx:
+      - "vpp-compare-testbeds-hsw-clx-ref"
+      - "vpp-compare-testbeds-hsw-clx-cmp"
+
     vpp-compare-testbeds-mrr-ref:
       csit-vpp-perf-verify-2001-3n-hsw:
         - 17  # MRR sel
@@ -1240,6 +1269,16 @@
       - "vpp-compare-testbeds-mrr-skx-clx-ref"
       - "vpp-compare-testbeds-mrr-skx-clx-cmp"
 
+    vpp-compare-testbeds-mrr-hsw-clx-ref:
+      csit-vpp-perf-verify-2001-3n-hsw:
+        - 17  # MRR sel
+    vpp-compare-testbeds-mrr-hsw-clx-cmp:
+      csit-vpp-perf-verify-2001-2n-clx:
+        - 39  # MRR
+    vpp-compare-testbeds-mrr-hsw-clx:
+      - "vpp-compare-testbeds-mrr-hsw-clx-ref"
+      - "vpp-compare-testbeds-mrr-hsw-clx-cmp"
+
     dpdk-compare-testbeds-ref:
       csit-dpdk-perf-verify-2001-3n-hsw:
         - 4   # NDRPDR sel
@@ -2174,7 +2213,9 @@
       - 29  # NDRPDR sel
       - 32  # NDRPDR sel
       - 40  # NDRPDR sel
+      - 43  # NDRPDR sel
       - 44  # NDRPDR sel
+      - 45  # NDRPDR sel
       - 39  # MRR
       - 34  # NDRPDR full
       - 35  # NDRPDR full
@@ -2463,8 +2504,8 @@
   format:
     html:
     - full
-    pdf:
-    - minimal
+#    pdf:
+#    - minimal
 
 ################################################################################
 ###                               T A B L E S                                ###
@@ -3489,7 +3530,7 @@
 #  - "parent"
 #  - "throughput"
 #  - "tags"
-#
+
 ## VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c pdr
 #- type: "table"
 #  title: "VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c pdr"
@@ -3569,7 +3610,95 @@
 #  - "parent"
 #  - "throughput"
 #  - "tags"
-#
+
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c ndr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c ndr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-1c-ndr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-hsw-clx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '1C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c ndr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c ndr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-2c-ndr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-hsw-clx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '2C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c pdr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c pdr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-1c-pdr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-hsw-clx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '1C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c pdr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c pdr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-2c-pdr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-hsw-clx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'NDRPDR' and '2C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
 ## VPP Comparison Across Topologies 3n-skx to 2n-skx 1c ndr
 #- type: "table"
 #  title: "VPP Comparison Across Topologies 3n-skx to 2n-skx 1c ndr"
@@ -4137,7 +4266,7 @@
 #  - "parent"
 #  - "result"
 #  - "tags"
-#
+
 ## VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c mrr
 #- type: "table"
 #  title: "VPP Comparison Across Testbeds 2n-skx to 2n-clx 2t1c mrr"
@@ -4255,6 +4384,72 @@
 #  - "parent"
 #  - "result"
 
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c mrr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 1c mrr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-1c-mrr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-mrr-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-mrr-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-mrr-hsw-clx"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'MRR' and '1C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c mrr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 2c mrr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-2c-mrr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-mrr-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-mrr-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-mrr-hsw-clx"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'MRR' and '2C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Comparison Across Testbeds 3n-hsw to 2n-clx 4c mrr
+- type: "table"
+  title: "VPP Comparison Across Testbeds 3n-hsw to 2n-clx 4c mrr"
+  algorithm: "table_perf_comparison_nic"
+  output-file: "{DIR[STATIC,VPP]}/performance-compare-testbeds-3n-hsw-2n-clx-4c-mrr"
+  reference:
+    title: "3n-hsw xl710"
+    data: "vpp-compare-testbeds-mrr-hsw-clx-ref"
+    nic: "NIC_Intel-XL710"
+  compare:
+    title: "2n-clx xxv710"
+    data: "vpp-compare-testbeds-mrr-hsw-clx-cmp"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-compare-testbeds-mrr-hsw-clx"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "('NIC_Intel-XXV710' or 'NIC_Intel-XL710') and 'MRR' and '4C'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
 # Detailed Test Results - VPP Performance Results 3n-hsw
 - type: "table"
   title: "Detailed Test Results - VPP Performance Results 3n-hsw"