aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2020-07-02 10:27:56 +0200
committerTibor Frank <tifrank@cisco.com>2020-07-02 08:29:36 +0000
commit6a8910fe0f5301c51fd558225d0041a75bee057d (patch)
treea1eb5c0b93badf0aa9e55c54c9ecd582b5bf38c9
parent9c1e867935e601947757049662a8db1a95fd6a26 (diff)
Report 2005: Add data
Change-Id: I65e1a386a29faa9c890a526b8d264f4eabcbf382 Signed-off-by: Tibor Frank <tifrank@cisco.com> (cherry picked from commit 9e06ee18ddcd7287162afd6848121e9a094fee57)
-rw-r--r--resources/tools/presentation/specification.yaml1268
1 files changed, 1102 insertions, 166 deletions
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 93f9f0ea13..ff8c10544a 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -851,6 +851,9 @@
- 44 # NDRPDR sel env 4
- 45 # NDRPDR sel env 4
- 47 # NDRPDR sel env 4
+ vpp-performance-changes-3n-skx-cmp-2001.0-4:
+ csit-vpp-perf-report-iterative-2005-3n-skx:
+ - 50 # NDRPDR sel 2001.0-4
vpp-performance-changes-2n-skx-h1:
csit-vpp-perf-verify-1908_1-2n-skx:
@@ -897,6 +900,9 @@
- 92 # NDRPDR sel env 4
- 93 # NDRPDR sel env 4
- 94 # NDRPDR sel env 4
+ vpp-performance-changes-2n-skx-cmp-2001.0-4:
+ csit-vpp-perf-report-iterative-2005-2n-skx:
+ - 100 # NDRPDR sel 2001.0-4
# vpp-performance-changes-nfv-2n-skx-ref:
# csit-vpp-perf-verify-2001-2n-skx:
@@ -944,6 +950,11 @@
- 68 # NDRPDR sel env 4
- 69 # NDRPDR sel env 4
- 70 # NDRPDR sel env 4
+ vpp-performance-changes-2n-clx-cmp-2001.0-4:
+ csit-vpp-perf-report-iterative-2005-2n-clx:
+ - 73 # NDRPDR sel 2001.0-4
+ - 74 # NDRPDR sel 2001.0-4
+ - 75 # NDRPDR sel 2001.0-4
vpp-performance-changes-2n-dnv-h1:
csit-vpp-perf-verify-1908-2n-dnv:
@@ -1056,6 +1067,9 @@
csit-vpp-perf-report-iterative-2005-3n-tsh:
- 23 # NDRPDR sel env 4
- 25 # NDRPDR sel env 4
+ vpp-performance-changes-3n-tsh-cmp-2001.0-4:
+ csit-vpp-perf-report-iterative-2005-3n-tsh:
+ - 28 # NDRPDR sel 2001.0-4
vpp-soak-vs-ndr-2n-skx-ref:
csit-vpp-perf-report-iterative-2005-2n-skx:
@@ -2332,6 +2346,7 @@
- 44 # NDRPDR sel env 4
- 45 # NDRPDR sel env 4
- 47 # NDRPDR sel env 4
+ - 50 # NDRPDR sel 2001.0-4
- 6 # MRR sel
- 49 # MRR sel env 4
csit-vpp-perf-report-coverage-2005-3n-skx:
@@ -2409,6 +2424,7 @@
- 92 # NDRPDR sel env 4
- 93 # NDRPDR sel env 4
- 94 # NDRPDR sel env 4
+ - 100 # NDRPDR sel 2001.0-4
- 42 # MRR sel
- 99 # MRR sel env 4
- 48 # Reconf
@@ -2479,6 +2495,9 @@
- 68 # NDRPDR sel env 4
- 69 # NDRPDR sel env 4
- 70 # NDRPDR sel env 4
+ - 73 # NDRPDR sel 2001.0-4
+ - 74 # NDRPDR sel 2001.0-4
+ - 75 # NDRPDR sel 2001.0-4
- 12 # MRR
- 72 # MRR env 4
- 42 # Reconf
@@ -2549,6 +2568,7 @@
- 21 # NDRPDR sel
- 23 # NDRPDR sel env 4
- 25 # NDRPDR sel env 4
+ - 28 # NDRPDR sel 2001.0-4
- 10 # MRR sel
- 27 # MRR sel env 4
@@ -3262,19 +3282,171 @@
data-set: "vpp-performance-changes-3n-hsw-ref"
- title: "2001.0-4"
data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4"
- - title: "2005.0-3"
- data-set: "vpp-performance-changes-3n-hsw-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-hsw-cmp-2"
comparisons:
- title: "2001.0-4 vs 2001.0-2"
reference: 2
compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-hsw 2t2c pdr
+- type: "table"
+ title: "VPP Performance Changes 3n-hsw 2t2c pdr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-hsw-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-hsw-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-hsw 1t1c ndr
+- type: "table"
+ title: "VPP Performance Changes 3n-hsw 1t1c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-hsw-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-hsw-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-hsw 2t2c ndr
+- type: "table"
+ title: "VPP Performance Changes 3n-hsw 2t2c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-hsw-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-hsw-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-hsw 1t1c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 3n-hsw 1t1c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-hsw-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-hsw-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4"
+ - title: "2005.0-3"
+ data-set: "vpp-performance-changes-3n-hsw-cmp"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-hsw-cmp-2"
+ comparisons:
- title: "2005.0-4 vs 2005.0-3"
reference: 4
compare: 5
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 5
# rca:
# - title: "RCA1"
@@ -3288,17 +3460,18 @@
- ''
- ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-hsw 2t2c pdr
+# VPP Performance Changes 3n-hsw 2t2c pdr-full
- type: "table"
- title: "VPP Performance Changes 3n-hsw 2t2c pdr"
+ title: "VPP Performance Changes 3n-hsw 2t2c pdr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr-full"
include-tests: "PDR" # "PDR" | "NDR" | "MRR"
filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
columns:
@@ -3313,14 +3486,14 @@
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-hsw-cmp-2"
comparisons:
- - title: "2001.0-4 vs 2001.0-2"
- reference: 2
- compare: 3
- title: "2005.0-4 vs 2005.0-3"
reference: 4
compare: 5
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 5
# rca:
# - title: "RCA1"
@@ -3334,17 +3507,18 @@
- ''
- ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-hsw 1t1c ndr
+# VPP Performance Changes 3n-hsw 1t1c ndr-full
- type: "table"
- title: "VPP Performance Changes 3n-hsw 1t1c ndr"
+ title: "VPP Performance Changes 3n-hsw 1t1c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
columns:
@@ -3359,14 +3533,14 @@
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-hsw-cmp-2"
comparisons:
- - title: "2001.0-4 vs 2001.0-2"
- reference: 2
- compare: 3
- title: "2005.0-4 vs 2005.0-3"
reference: 4
compare: 5
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 5
# rca:
# - title: "RCA1"
@@ -3378,19 +3552,20 @@
legend:
- 'Legend to Table:'
- ''
- - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.'
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-hsw 2t2c ndr
+# VPP Performance Changes 3n-hsw 2t2c ndr-full
- type: "table"
- title: "VPP Performance Changes 3n-hsw 2t2c ndr"
+ title: "VPP Performance Changes 3n-hsw 2t2c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
columns:
@@ -3405,14 +3580,14 @@
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-hsw-cmp-2"
comparisons:
- - title: "2001.0-4 vs 2001.0-2"
- reference: 2
- compare: 3
- title: "2005.0-4 vs 2005.0-3"
reference: 4
compare: 5
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 5
# rca:
# - title: "RCA1"
@@ -3424,13 +3599,14 @@
legend:
- 'Legend to Table:'
- ''
- - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.'
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 3n-skx 2t1c pdr
- type: "table"
@@ -3444,16 +3620,16 @@
data-set: "vpp-performance-changes-3n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-skx-ref"
- - title: "2005.0-3"
- data-set: "vpp-performance-changes-3n-skx-cmp"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-skx-cmp-2"
comparisons:
- - title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 4
# rca:
# - title: "RCA1"
@@ -3465,10 +3641,10 @@
- ''
- ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
-# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 3n-skx 4t2c pdr
- type: "table"
@@ -3482,37 +3658,207 @@
data-set: "vpp-performance-changes-3n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-skx 2t1c ndr
+- type: "table"
+ title: "VPP Performance Changes 3n-skx 2t1c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-skx 4t2c ndr
+- type: "table"
+ title: "VPP Performance Changes 3n-skx 4t2c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-skx 2t1c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 3n-skx 2t1c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-3n-skx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-skx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ compare: 5
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca3.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-skx 4t2c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 3n-skx 4t2c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
+ - title: "2005.0-3"
+ data-set: "vpp-performance-changes-3n-skx-cmp"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2"
+ comparisons:
+ - title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-skx 2t1c ndr
+# VPP Performance Changes 3n-skx 2t1c ndr-full
- type: "table"
- title: "VPP Performance Changes 3n-skx 2t1c ndr"
+ title: "VPP Performance Changes 3n-skx 2t1c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
@@ -3520,37 +3866,46 @@
data-set: "vpp-performance-changes-3n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-3n-skx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-skx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-skx 4t2c ndr
+# VPP Performance Changes 3n-skx 4t2c ndr-full
- type: "table"
- title: "VPP Performance Changes 3n-skx 4t2c ndr"
+ title: "VPP Performance Changes 3n-skx 4t2c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
@@ -3558,31 +3913,40 @@
data-set: "vpp-performance-changes-3n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-3n-skx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-skx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 2n-skx 2t1c pdr
- type: "table"
@@ -3596,16 +3960,16 @@
data-set: "vpp-performance-changes-2n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-skx-ref"
- - title: "2005.0-3"
- data-set: "vpp-performance-changes-2n-skx-cmp"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-skx-cmp-2"
comparisons:
- - title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 4
# rca:
# - title: "RCA1"
@@ -3617,10 +3981,10 @@
- ''
- ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
-# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 2n-skx 4t2c pdr
- type: "table"
@@ -3634,37 +3998,207 @@
data-set: "vpp-performance-changes-2n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-skx 2t1c ndr
+- type: "table"
+ title: "VPP Performance Changes 2n-skx 2t1c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-skx 4t2c ndr
+- type: "table"
+ title: "VPP Performance Changes 2n-skx 4t2c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-skx 2t1c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 2n-skx 2t1c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-skx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-skx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ compare: 5
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca3.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-skx 4t2c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 2n-skx 4t2c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-skx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
+ - title: "2005.0-3"
+ data-set: "vpp-performance-changes-2n-skx-cmp"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2"
+ comparisons:
+ - title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 2n-skx 2t1c ndr
+# VPP Performance Changes 2n-skx 2t1c ndr-full
- type: "table"
- title: "VPP Performance Changes 2n-skx 2t1c ndr"
+ title: "VPP Performance Changes 2n-skx 2t1c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
@@ -3672,37 +4206,46 @@
data-set: "vpp-performance-changes-2n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-skx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-skx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 2n-skx 4t2c ndr
+# VPP Performance Changes 2n-skx 4t2c ndr-full
- type: "table"
- title: "VPP Performance Changes 2n-skx 4t2c ndr"
+ title: "VPP Performance Changes 2n-skx 4t2c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
@@ -3710,31 +4253,40 @@
data-set: "vpp-performance-changes-2n-skx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-skx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-skx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-skx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 2n-clx-xxv710 2t1c pdr
- type: "table"
@@ -3744,141 +4296,337 @@
include-tests: "PDR" # "PDR" | "NDR" | "MRR"
filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-clx-xxv710 4t2c pdr
+- type: "table"
+ title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-clx-xxv710 2t1c ndr
+- type: "table"
+ title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-clx-xxv710 4t2c ndr
+- type: "table"
+ title: "VPP Performance Changes 2n-clx-xxv710 ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-clx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-clx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
compare: 3
- - title: "2005.0-4 vs 2001.0-2"
- reference: 1
- compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 2n-clx-xxv710 4t2c pdr
+# VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full
- type: "table"
- title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr"
+ title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr-full"
include-tests: "PDR" # "PDR" | "NDR" | "MRR"
filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-clx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-clx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
compare: 3
- - title: "2005.0-4 vs 2001.0-2"
- reference: 1
- compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 2n-clx-xxv710 2t1c ndr
+# VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full
- type: "table"
- title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr"
+ title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-clx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-clx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
compare: 3
- - title: "2005.0-4 vs 2001.0-2"
- reference: 1
- compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 2n-clx-xxv710 4t2c ndr
+# VPP Performance Changes 2n-clx-xxv710 4t2c ndr-full
- type: "table"
- title: "VPP Performance Changes 2n-clx-xxv710 ndr"
+ title: "VPP Performance Changes 2n-clx-xxv710 ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-2n-clx-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-2n-clx-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-2n-clx-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-2n-clx-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
compare: 3
- - title: "2005.0-4 vs 2001.0-2"
- reference: 1
- compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 2n-clx-x556 2t1c pdr
- type: "table"
@@ -4160,16 +4908,16 @@
data-set: "vpp-performance-changes-3n-tsh-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-tsh-ref"
- - title: "2005.0-3"
- data-set: "vpp-performance-changes-3n-tsh-cmp"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-tsh-cmp-2"
comparisons:
- - title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
compare: 4
# rca:
# - title: "RCA1"
@@ -4181,10 +4929,10 @@
- ''
- ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
-# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Performance Changes 3n-tsh 2t2c pdr
- type: "table"
@@ -4198,37 +4946,207 @@
data-set: "vpp-performance-changes-3n-tsh-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-tsh 1t1c ndr
+- type: "table"
+ title: "VPP Performance Changes 3n-tsh 1t1c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'1T1C'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-tsh-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-tsh 2t2c ndr
+- type: "table"
+ title: "VPP Performance Changes 3n-tsh 2t2c ndr"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T2C'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-tsh-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+ comparisons:
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 4
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-tsh 1t1c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 3n-tsh 1t1c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'1T1C'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-tsh-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-3n-tsh-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-tsh-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
+ reference: 2
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ compare: 5
+# rca:
+# - title: "RCA1"
+# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml"
+# - title: "RCA2"
+# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca3.yaml"
+ legend:
+ - 'Legend to Table:'
+ - ''
+ - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.'
+ - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
+ - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
+# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
+
+# VPP Performance Changes 3n-tsh 2t2c pdr-full
+- type: "table"
+ title: "VPP Performance Changes 3n-tsh 2t2c pdr-full"
+ algorithm: "table_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr-full"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T2C'"
+ columns:
+ - title: "1908.1-2"
+ data-set: "vpp-performance-changes-3n-tsh-h1"
+ - title: "2001.0-2"
+ data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
+ - title: "2005.0-3"
+ data-set: "vpp-performance-changes-3n-tsh-cmp"
+ - title: "2005.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2"
+ comparisons:
+ - title: "2005.0-4 vs 2005.0-3"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-tsh 1t1c ndr
+# VPP Performance Changes 3n-tsh 1t1c ndr-full
- type: "table"
- title: "VPP Performance Changes 3n-tsh 1t1c ndr"
+ title: "VPP Performance Changes 3n-tsh 1t1c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'1T1C'"
columns:
@@ -4236,37 +5154,46 @@
data-set: "vpp-performance-changes-3n-tsh-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-3n-tsh-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-tsh-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
-# VPP Performance Changes 3n-tsh 2t2c ndr
+# VPP Performance Changes 3n-tsh 2t2c ndr-full
- type: "table"
- title: "VPP Performance Changes 3n-tsh 2t2c ndr"
+ title: "VPP Performance Changes 3n-tsh 2t2c ndr-full"
algorithm: "table_comparison"
- output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr"
+ output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr-full"
include-tests: "NDR" # "PDR" | "NDR" | "MRR"
filter: "'2T2C'"
columns:
@@ -4274,31 +5201,40 @@
data-set: "vpp-performance-changes-3n-tsh-h1"
- title: "2001.0-2"
data-set: "vpp-performance-changes-3n-tsh-ref"
+ - title: "2001.0-4"
+ data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4"
- title: "2005.0-3"
data-set: "vpp-performance-changes-3n-tsh-cmp"
- title: "2005.0-4"
data-set: "vpp-performance-changes-3n-tsh-cmp-2"
comparisons:
- title: "2005.0-4 vs 2005.0-3"
- reference: 3
- compare: 4
- - title: "2005.0-4 vs 2001.0-2"
+ reference: 4
+ compare: 5
+ - title: "2001.0-4 vs 2001.0-2"
reference: 2
- compare: 4
+ compare: 3
+ - title: "2005.0-4 vs 2001.0-4"
+ reference: 3
+ compare: 5
# rca:
# - title: "RCA1"
# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml"
# - title: "RCA2"
# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml"
+# - title: "RCA3"
+# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca3.yaml"
legend:
- 'Legend to Table:'
- ''
- ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.'
- ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).'
- '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.'
- - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.'
+ - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.'
+ - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.'
# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.'
-# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.'
+# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.'
+# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.'
# VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr
- type: "table"