Report 2005: Add data

Change-Id: I11c238327de71e75ca1ad099455f8daeb1ce996b
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 145 insertions, 257 deletions

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index d670800ae5..c1048fa64b 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -883,6 +883,7 @@
       csit-vpp-perf-report-iterative-2005-2n-clx:
         - 62  # NDRPDR sel env 4
         - 63  # NDRPDR sel env 4
+        - 64  # NDRPDR sel env 4
 
     vpp-performance-changes-2n-dnv-h1:
       csit-vpp-perf-verify-1908-2n-dnv:
@@ -1671,120 +1672,6 @@
       hoverlabel:
         namelength: -1
 
-#    plot-cps:
-#      titlefont:
-#        size: 18
-#      xaxis:
-#        title: "<b>Test Cases [Index]</b>"
-#        titlefont:
-#          size: 16
-#        autorange: True
-#        fixedrange: False
-#        gridcolor: "rgb(230, 230, 230)"
-#        linecolor: "rgb(220, 220, 220)"
-#        linewidth: 1
-#        showgrid: True
-#        showline: True
-#        showticklabels: True
-#        tickcolor: "rgb(220, 220, 220)"
-#        tickmode: "linear"
-#        tickfont:
-#          size: 16
-#        zeroline: False
-#      yaxis:
-#        title: "<b>Connections Per Second [cps]</b>"
-#        titlefont:
-#          size: 16
-#        gridcolor: "rgb(230, 230, 230)"
-#        hoverformat: ".4s"
-#        linecolor: "rgb(220, 220, 220)"
-#        linewidth: 1
-#        rangemode: "tozero"
-#        showgrid: True
-#        showline: True
-#        showticklabels: True
-#        tickcolor: "rgb(220, 220, 220)"
-#        tickformat: ".3s"
-#        tickfont:
-#          size: 16
-#        zeroline: False
-#      boxmode: "group"
-#      boxgroupgap: 0.5
-#      autosize: False
-#      margin:
-#        t: 50
-#        b: 20
-#        l: 80
-#        r: 20
-#      showlegend: True
-#      legend:
-#        orientation: "h"
-#        font:
-#          size: 16
-#      width: 700
-#      height: 800
-#      paper_bgcolor: "#fff"
-#      plot_bgcolor: "#fff"
-#      hoverlabel:
-#        namelength: -1
-#
-#    plot-rps:
-#      titlefont:
-#        size: 18
-#      xaxis:
-#        title: "<b>Test Cases [Index]</b>"
-#        titlefont:
-#          size: 16
-#        autorange: True
-#        fixedrange: False
-#        gridcolor: "rgb(230, 230, 230)"
-#        linecolor: "rgb(220, 220, 220)"
-#        linewidth: 1
-#        showgrid: True
-#        showline: True
-#        showticklabels: True
-#        tickcolor: "rgb(220, 220, 220)"
-#        tickmode: "linear"
-#        tickfont:
-#          size: 16
-#        zeroline: False
-#      yaxis:
-#        title: "<b>Requests Per Second [rps]</b>"
-#        titlefont:
-#          size: 16
-#        gridcolor: "rgb(230, 230, 230)"
-#        hoverformat: ".4s"
-#        linecolor: "rgb(220, 220, 220)"
-#        linewidth: 1
-#        rangemode: "tozero"
-#        showgrid: True
-#        showline: True
-#        showticklabels: True
-#        tickcolor: "rgb(230, 230, 230)"
-#        tickformat: ".3s"
-#        tickfont:
-#          size: 16
-#        zeroline: False
-#      boxmode: "group"
-#      boxgroupgap: 0.5
-#      autosize: False
-#      margin:
-#        t: 50
-#        b: 20
-#        l: 80
-#        r: 20
-#      showlegend: True
-#      legend:
-#        orientation: "h"
-#        font:
-#          size: 16
-#      width: 700
-#      height: 800
-#      paper_bgcolor: "#fff"
-#      plot_bgcolor: "#fff"
-#      hoverlabel:
-#        namelength: -1
-
     plot-hoststack:
       titlefont:
         size: 16
@@ -2486,6 +2373,7 @@
       - 51  # NDRPDR sel
       - 62  # NDRPDR sel env 4
       - 63  # NDRPDR sel env 4
+      - 64  # NDRPDR sel env 4
       - 12  # MRR
       - 42  # Reconf
       - 45  # Reconf
@@ -4087,149 +3975,149 @@
     - "1908, 2001, 2005 - VPP versions tested."
     - "Diff - difference between 2001 and 2005 VPP version."
 
-## VPP Performance Changes 3n-tsh 1t1c pdr
-#- type: "table"
-#  title: "VPP Performance Changes 3n-tsh 1t1c pdr"
-#  algorithm: "table_comparison"
-#  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr"
-#  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
-#  filter: "'1T1C'"
-#  columns:
-#    - title: "1908.1-2"
-#      data-set: "vpp-performance-changes-3n-tsh-h1"
-#    - title: "2001.0-2"
-#      data-set: "vpp-performance-changes-3n-tsh-ref"
-#    - title: "2005.0-3"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
-#    - title: "2005.0-4"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp"
-#  comparisons:
-#    - title: "Diff1"
-#      reference: 3
-#      compare: 4
-#    - title: "Diff2"
-#      reference: 2
-#      compare: 3
-##  rca:
-##    - title: "RCA1"
-##      data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml"
-##    - title: "RCA2"
-##      data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml"
-#  legend:
-#    - "Legend to Table"
-#    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
-#    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
-#    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
-##    - 'RCA1 - root cause analysis reference for Diff1.'
-##    - 'RCA2 - root cause analysis reference for Diff2.'
-#
-## VPP Performance Changes 3n-tsh 2t2c pdr
-#- type: "table"
-#  title: "VPP Performance Changes 3n-tsh 2t2c pdr"
-#  algorithm: "table_comparison"
-#  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr"
-#  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
-#  filter: "'2T2C'"
-#  columns:
-#    - title: "1908.1-2"
-#      data-set: "vpp-performance-changes-3n-tsh-h1"
-#    - title: "2001.0-2"
-#      data-set: "vpp-performance-changes-3n-tsh-ref"
-#    - title: "2005.0-3"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
-#    - title: "2005.0-4"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp"
-#  comparisons:
-#    - title: "Diff1"
-#      reference: 3
-#      compare: 4
-#    - title: "Diff2"
-#      reference: 2
-#      compare: 3
-##  rca:
-##    - title: "RCA1"
-##      data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml"
-##    - title: "RCA2"
-##      data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml"
-#  legend:
-#    - "Legend to Table"
-#    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
-#    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
-#    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
-##    - 'RCA1 - root cause analysis reference for Diff1.'
-##    - 'RCA2 - root cause analysis reference for Diff2.'
-#
-## VPP Performance Changes 3n-tsh 1t1c ndr
-#- type: "table"
-#  title: "VPP Performance Changes 3n-tsh 1t1c ndr"
-#  algorithm: "table_comparison"
-#  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr"
-#  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
-#  filter: "'1T1C'"
-#  columns:
-#    - title: "1908.1-2"
-#      data-set: "vpp-performance-changes-3n-tsh-h1"
-#    - title: "2001.0-2"
-#      data-set: "vpp-performance-changes-3n-tsh-ref"
-#    - title: "2005.0-3"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
-#    - title: "2005.0-4"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp"
-#  comparisons:
-#    - title: "Diff1"
-#      reference: 3
-#      compare: 4
-#    - title: "Diff2"
-#      reference: 2
-#      compare: 3
-##  rca:
-##    - title: "RCA1"
-##      data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml"
-##    - title: "RCA2"
-##      data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml"
-#  legend:
-#    - "Legend to Table"
-#    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
-#    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
-#    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
-##    - 'RCA1 - root cause analysis reference for Diff1.'
-##    - 'RCA2 - root cause analysis reference for Diff2.'
-#
-## VPP Performance Changes 3n-tsh 2t2c ndr
-#- type: "table"
-#  title: "VPP Performance Changes 3n-tsh 2t2c ndr"
-#  algorithm: "table_comparison"
-#  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr"
-#  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
-#  filter: "'2T2C'"
-#  columns:
-#    - title: "1908.1-2"
-#      data-set: "vpp-performance-changes-3n-tsh-h1"
-#    - title: "2001.0-2"
-#      data-set: "vpp-performance-changes-3n-tsh-ref"
-#    - title: "2005.0-3"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
-#    - title: "2005.0-4"
-#      data-set: "vpp-performance-changes-3n-tsh-cmp"
-#  comparisons:
-#    - title: "Diff1"
-#      reference: 3
-#      compare: 4
-#    - title: "Diff2"
-#      reference: 2
-#      compare: 3
-##  rca:
-##    - title: "RCA1"
-##      data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml"
-##    - title: "RCA2"
-##      data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml"
-#  legend:
-#    - "Legend to Table"
-#    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
-#    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
-#    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
-##    - 'RCA1 - root cause analysis reference for Diff1.'
-##    - 'RCA2 - root cause analysis reference for Diff2.'
+# VPP Performance Changes 3n-tsh 1t1c pdr
+- type: "table"
+  title: "VPP Performance Changes 3n-tsh 1t1c pdr"
+  algorithm: "table_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'1T1C'"
+  columns:
+    - title: "1908.1-2"
+      data-set: "vpp-performance-changes-3n-tsh-h1"
+    - title: "2001.0-2"
+      data-set: "vpp-performance-changes-3n-tsh-ref"
+    - title: "2005.0-3"
+      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+    - title: "2005.0-4"
+      data-set: "vpp-performance-changes-3n-tsh-cmp"
+  comparisons:
+    - title: "Diff1"
+      reference: 3
+      compare: 4
+    - title: "Diff2"
+      reference: 2
+      compare: 3
+#  rca:
+#    - title: "RCA1"
+#      data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml"
+#    - title: "RCA2"
+#      data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml"
+  legend:
+    - "Legend to Table"
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
+    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
+#    - 'RCA1 - root cause analysis reference for Diff1.'
+#    - 'RCA2 - root cause analysis reference for Diff2.'
+
+# VPP Performance Changes 3n-tsh 2t2c pdr
+- type: "table"
+  title: "VPP Performance Changes 3n-tsh 2t2c pdr"
+  algorithm: "table_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T2C'"
+  columns:
+    - title: "1908.1-2"
+      data-set: "vpp-performance-changes-3n-tsh-h1"
+    - title: "2001.0-2"
+      data-set: "vpp-performance-changes-3n-tsh-ref"
+    - title: "2005.0-3"
+      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+    - title: "2005.0-4"
+      data-set: "vpp-performance-changes-3n-tsh-cmp"
+  comparisons:
+    - title: "Diff1"
+      reference: 3
+      compare: 4
+    - title: "Diff2"
+      reference: 2
+      compare: 3
+#  rca:
+#    - title: "RCA1"
+#      data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml"
+#    - title: "RCA2"
+#      data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml"
+  legend:
+    - "Legend to Table"
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
+    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
+#    - 'RCA1 - root cause analysis reference for Diff1.'
+#    - 'RCA2 - root cause analysis reference for Diff2.'
+
+# VPP Performance Changes 3n-tsh 1t1c ndr
+- type: "table"
+  title: "VPP Performance Changes 3n-tsh 1t1c ndr"
+  algorithm: "table_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'1T1C'"
+  columns:
+    - title: "1908.1-2"
+      data-set: "vpp-performance-changes-3n-tsh-h1"
+    - title: "2001.0-2"
+      data-set: "vpp-performance-changes-3n-tsh-ref"
+    - title: "2005.0-3"
+      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+    - title: "2005.0-4"
+      data-set: "vpp-performance-changes-3n-tsh-cmp"
+  comparisons:
+    - title: "Diff1"
+      reference: 3
+      compare: 4
+    - title: "Diff2"
+      reference: 2
+      compare: 3
+#  rca:
+#    - title: "RCA1"
+#      data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml"
+#    - title: "RCA2"
+#      data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml"
+  legend:
+    - "Legend to Table"
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
+    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
+#    - 'RCA1 - root cause analysis reference for Diff1.'
+#    - 'RCA2 - root cause analysis reference for Diff2.'
+
+# VPP Performance Changes 3n-tsh 2t2c ndr
+- type: "table"
+  title: "VPP Performance Changes 3n-tsh 2t2c ndr"
+  algorithm: "table_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T2C'"
+  columns:
+    - title: "1908.1-2"
+      data-set: "vpp-performance-changes-3n-tsh-h1"
+    - title: "2001.0-2"
+      data-set: "vpp-performance-changes-3n-tsh-ref"
+    - title: "2005.0-3"
+      data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+    - title: "2005.0-4"
+      data-set: "vpp-performance-changes-3n-tsh-cmp"
+  comparisons:
+    - title: "Diff1"
+      reference: 3
+      compare: 4
+    - title: "Diff2"
+      reference: 2
+      compare: 3
+#  rca:
+#    - title: "RCA1"
+#      data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml"
+#    - title: "RCA2"
+#      data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml"
+  legend:
+    - "Legend to Table"
+    - 'YYMM.V-E - tested VPP version (YY year, MM month, V minor version) and CSIT test environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+    - 'Diff1 - difference between environment versions "3" and "4" for the same VPP version, 2005.0-3 relative to 2005.0-4. Version "3" was found to be incompatible with "2" due to different number of TRex workers used.'
+    - 'Diff2 - difference between the current and the previous VPP version in the compatible environment versions "2" and "4", 2005.0-4 relative to 2001.0-2.'
+#    - 'RCA1 - root cause analysis reference for Diff1.'
+#    - 'RCA2 - root cause analysis reference for Diff2.'
 
 # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr
 - type: "table"