diff options
| -rw-r--r-- | resources/tools/presentation/specification.yaml | 1268 |
1 files changed, 1102 insertions, 166 deletions
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 93f9f0ea13..ff8c10544a 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -851,6 +851,9 @@ - 44 # NDRPDR sel env 4 - 45 # NDRPDR sel env 4 - 47 # NDRPDR sel env 4 + vpp-performance-changes-3n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-skx: + - 50 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-skx-h1: csit-vpp-perf-verify-1908_1-2n-skx: @@ -897,6 +900,9 @@ - 92 # NDRPDR sel env 4 - 93 # NDRPDR sel env 4 - 94 # NDRPDR sel env 4 + vpp-performance-changes-2n-skx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-skx: + - 100 # NDRPDR sel 2001.0-4 # vpp-performance-changes-nfv-2n-skx-ref: # csit-vpp-perf-verify-2001-2n-skx: @@ -944,6 +950,11 @@ - 68 # NDRPDR sel env 4 - 69 # NDRPDR sel env 4 - 70 # NDRPDR sel env 4 + vpp-performance-changes-2n-clx-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-2n-clx: + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 vpp-performance-changes-2n-dnv-h1: csit-vpp-perf-verify-1908-2n-dnv: @@ -1056,6 +1067,9 @@ csit-vpp-perf-report-iterative-2005-3n-tsh: - 23 # NDRPDR sel env 4 - 25 # NDRPDR sel env 4 + vpp-performance-changes-3n-tsh-cmp-2001.0-4: + csit-vpp-perf-report-iterative-2005-3n-tsh: + - 28 # NDRPDR sel 2001.0-4 vpp-soak-vs-ndr-2n-skx-ref: csit-vpp-perf-report-iterative-2005-2n-skx: @@ -2332,6 +2346,7 @@ - 44 # NDRPDR sel env 4 - 45 # NDRPDR sel env 4 - 47 # NDRPDR sel env 4 + - 50 # NDRPDR sel 2001.0-4 - 6 # MRR sel - 49 # MRR sel env 4 csit-vpp-perf-report-coverage-2005-3n-skx: @@ -2409,6 +2424,7 @@ - 92 # NDRPDR sel env 4 - 93 # NDRPDR sel env 4 - 94 # NDRPDR sel env 4 + - 100 # NDRPDR sel 2001.0-4 - 42 # MRR sel - 99 # MRR sel env 4 - 48 # Reconf @@ -2479,6 +2495,9 @@ - 68 # NDRPDR sel env 4 - 69 # NDRPDR sel env 4 - 70 # NDRPDR sel env 4 + - 73 # NDRPDR sel 2001.0-4 + - 74 # NDRPDR sel 2001.0-4 + - 75 # NDRPDR sel 2001.0-4 - 12 # MRR - 72 # MRR env 4 - 42 # Reconf @@ -2549,6 +2568,7 @@ - 21 # NDRPDR sel - 23 # NDRPDR sel env 4 - 25 # NDRPDR sel env 4 + - 28 # NDRPDR sel 2001.0-4 - 10 # MRR sel - 27 # MRR sel env 4 @@ -3262,19 +3282,171 @@ data-set: "vpp-performance-changes-3n-hsw-ref" - title: "2001.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-hsw-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c pdr +- type: "table" + title: "VPP Performance Changes 3n-hsw 2t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca1.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-hsw-2t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-hsw 1t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca2.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-1t1c-ndr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 2t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-hsw 2t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-hsw-2t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-hsw 1t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-hsw 1t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-hsw-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-hsw-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-hsw-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-hsw-cmp-2" + comparisons: - title: "2005.0-4 vs 2005.0-3" reference: 4 compare: 5 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 5 # rca: # - title: "RCA1" @@ -3288,17 +3460,18 @@ - '' - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-hsw 2t2c pdr +# VPP Performance Changes 3n-hsw 2t2c pdr-full - type: "table" - title: "VPP Performance Changes 3n-hsw 2t2c pdr" + title: "VPP Performance Changes 3n-hsw 2t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: @@ -3313,14 +3486,14 @@ - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2001.0-4 vs 2001.0-2" - reference: 2 - compare: 3 - title: "2005.0-4 vs 2005.0-3" reference: 4 compare: 5 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 5 # rca: # - title: "RCA1" @@ -3334,17 +3507,18 @@ - '' - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-hsw 1t1c ndr +# VPP Performance Changes 3n-hsw 1t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-hsw 1t1c ndr" + title: "VPP Performance Changes 3n-hsw 1t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: @@ -3359,14 +3533,14 @@ - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2001.0-4 vs 2001.0-2" - reference: 2 - compare: 3 - title: "2005.0-4 vs 2005.0-3" reference: 4 compare: 5 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 5 # rca: # - title: "RCA1" @@ -3378,19 +3552,20 @@ legend: - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-hsw 2t2c ndr +# VPP Performance Changes 3n-hsw 2t2c ndr-full - type: "table" - title: "VPP Performance Changes 3n-hsw 2t2c ndr" + title: "VPP Performance Changes 3n-hsw 2t2c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'" columns: @@ -3405,14 +3580,14 @@ - title: "2005.0-4" data-set: "vpp-performance-changes-3n-hsw-cmp-2" comparisons: - - title: "2001.0-4 vs 2001.0-2" - reference: 2 - compare: 3 - title: "2005.0-4 vs 2005.0-3" reference: 4 compare: 5 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 5 # rca: # - title: "RCA1" @@ -3424,13 +3599,14 @@ legend: - 'Legend to Table:' - '' - - ' Test Case: Test cases executed for: PDR throughput on 3n-hsw with xl710.' + - ' Test Case: Test cases executed for: NDR throughput on 3n-hsw with xl710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 2t1c pdr - type: "table" @@ -3444,16 +3620,16 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" @@ -3465,10 +3641,10 @@ - '' - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-skx 4t2c pdr - type: "table" @@ -3482,37 +3658,207 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-pdr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-pdr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: PDR throughput on 3n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-skx 2t1c ndr +# VPP Performance Changes 3n-skx 2t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-skx 2t1c ndr" + title: "VPP Performance Changes 3n-skx 2t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: @@ -3520,37 +3866,46 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-2t1c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-skx 4t2c ndr +# VPP Performance Changes 3n-skx 4t2c ndr-full - type: "table" - title: "VPP Performance Changes 3n-skx 4t2c ndr" + title: "VPP Performance Changes 3n-skx 4t2c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: @@ -3558,31 +3913,40 @@ data-set: "vpp-performance-changes-3n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-skx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-skx-4t2c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 3n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 2t1c pdr - type: "table" @@ -3596,16 +3960,16 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" @@ -3617,10 +3981,10 @@ - '' - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-skx 4t2c pdr - type: "table" @@ -3634,37 +3998,207 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c ndr +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c ndr +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-pdr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-skx 4t2c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-skx 4t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-skx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-2n-skx-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-skx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-pdr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: PDR throughput on 2n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-skx 2t1c ndr +# VPP Performance Changes 2n-skx 2t1c ndr-full - type: "table" - title: "VPP Performance Changes 2n-skx 2t1c ndr" + title: "VPP Performance Changes 2n-skx 2t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: @@ -3672,37 +4206,46 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-skx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-2t1c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-skx 4t2c ndr +# VPP Performance Changes 2n-skx 4t2c ndr-full - type: "table" - title: "VPP Performance Changes 2n-skx 4t2c ndr" + title: "VPP Performance Changes 2n-skx 4t2c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: @@ -3710,31 +4253,40 @@ data-set: "vpp-performance-changes-2n-skx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-skx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-skx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-skx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-skx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-skx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-skx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-skx-4t2c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 2n-skx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-xxv710 2t1c pdr - type: "table" @@ -3744,141 +4296,337 @@ include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 4t2c ndr +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full +- type: "table" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 - - title: "2005.0-4 vs 2001.0-2" - reference: 1 - compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-2t1c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-2t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-pdr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-clx-xxv710 4t2c pdr +# VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr" + title: "VPP Performance Changes 2n-clx-xxv710 4t2c pdr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-pdr-full" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 - - title: "2005.0-4 vs 2001.0-2" - reference: 1 - compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-4t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-4t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-pdr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: PDR throughput on 2n-clx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-clx-xxv710 2t1c ndr +# VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr" + title: "VPP Performance Changes 2n-clx-xxv710 2t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-2t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 - - title: "2005.0-4 vs 2001.0-2" - reference: 1 - compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-2t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-2t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-2t1c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 2n-clx-xxv710 4t2c ndr +# VPP Performance Changes 2n-clx-xxv710 4t2c ndr-full - type: "table" - title: "VPP Performance Changes 2n-clx-xxv710 ndr" + title: "VPP Performance Changes 2n-clx-xxv710 ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-clx-xxv710-4t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'4T2C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'" columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-2n-clx-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-2n-clx-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-2n-clx-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-2n-clx-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-2n-clx-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 compare: 3 - - title: "2005.0-4 vs 2001.0-2" - reference: 1 - compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-2n-clx-4t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-2n-clx-4t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-2n-clx-4t2c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 2n-clx with xxv710.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 2n-clx-x556 2t1c pdr - type: "table" @@ -4160,16 +4908,16 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" - - title: "2005.0-3" - data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + - title: "2001.0-4 vs 2001.0-2" reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 compare: 4 # rca: # - title: "RCA1" @@ -4181,10 +4929,10 @@ - '' - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' -# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Performance Changes 3n-tsh 2t2c pdr - type: "table" @@ -4198,37 +4946,207 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c ndr +- type: "table" + title: "VPP Performance Changes 3n-tsh 1t1c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c ndr +- type: "table" + title: "VPP Performance Changes 3n-tsh 2t2c ndr" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 4 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 1t1c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-tsh 1t1c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'1T1C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" + reference: 2 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + compare: 5 +# rca: +# - title: "RCA1" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca1.yaml" +# - title: "RCA2" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-pdr-rca3.yaml" + legend: + - 'Legend to Table:' + - '' + - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' + - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' + - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' +# - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' + +# VPP Performance Changes 3n-tsh 2t2c pdr-full +- type: "table" + title: "VPP Performance Changes 3n-tsh 2t2c pdr-full" + algorithm: "table_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-pdr-full" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T2C'" + columns: + - title: "1908.1-2" + data-set: "vpp-performance-changes-3n-tsh-h1" + - title: "2001.0-2" + data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" + - title: "2005.0-3" + data-set: "vpp-performance-changes-3n-tsh-cmp" + - title: "2005.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2" + comparisons: + - title: "2005.0-4 vs 2005.0-3" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-pdr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: PDR throughput on 3n-tsh with x520.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-tsh 1t1c ndr +# VPP Performance Changes 3n-tsh 1t1c ndr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 1t1c ndr" + title: "VPP Performance Changes 3n-tsh 1t1c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-1t1c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'1T1C'" columns: @@ -4236,37 +5154,46 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-1t1c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' -# VPP Performance Changes 3n-tsh 2t2c ndr +# VPP Performance Changes 3n-tsh 2t2c ndr-full - type: "table" - title: "VPP Performance Changes 3n-tsh 2t2c ndr" + title: "VPP Performance Changes 3n-tsh 2t2c ndr-full" algorithm: "table_comparison" - output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr" + output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-tsh-2t2c-ndr-full" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'2T2C'" columns: @@ -4274,31 +5201,40 @@ data-set: "vpp-performance-changes-3n-tsh-h1" - title: "2001.0-2" data-set: "vpp-performance-changes-3n-tsh-ref" + - title: "2001.0-4" + data-set: "vpp-performance-changes-3n-tsh-cmp-2001.0-4" - title: "2005.0-3" data-set: "vpp-performance-changes-3n-tsh-cmp" - title: "2005.0-4" data-set: "vpp-performance-changes-3n-tsh-cmp-2" comparisons: - title: "2005.0-4 vs 2005.0-3" - reference: 3 - compare: 4 - - title: "2005.0-4 vs 2001.0-2" + reference: 4 + compare: 5 + - title: "2001.0-4 vs 2001.0-2" reference: 2 - compare: 4 + compare: 3 + - title: "2005.0-4 vs 2001.0-4" + reference: 3 + compare: 5 # rca: # - title: "RCA1" # data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca1.yaml" # - title: "RCA2" # data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca2.yaml" +# - title: "RCA3" +# data: "rca/rca-2005-3n-tsh-2t2c-ndr-rca3.yaml" legend: - 'Legend to Table:' - '' - ' Test Case: Test cases executed for: NDR throughput on 3n-tsh with x520.' - ' YYMM.V-E: [Mpps +- StDev] results for VPP version (YY year, MM month, V minor version) tested in CSIT environment version (E environment version, "2" for 2020-03-27, "3" for 2020-06-11, "4" for 2020-06-24).' - '2005.0-4 vs 2005.0-3: [% +- StDev] Relative difference, latter is the base. Environment version "3" was found to be incompatible with "2" due to higher number of TRex workers used.' - - '2005.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base. Used as a reference to compare benchmark results of VPP current release v20.05.0 vs previous release v20.01.0.' + - '2001.0-4 vs 2001.0-2: [% +- StDev] Relative difference, latter is the base.' + - '2005.0-4 vs 2001.0-4: [% +- StDev] Relative difference, latter is the base.' # - ' RCA1: root cause analysis reference for 2005.0-4 vs 2005.0-3.' -# - ' RCA2: root cause analysis reference for 2005.0-4 vs 2001.0-2.' +# - ' RCA2: root cause analysis reference for 2001.0-4 vs 2001.0-2.' +# - ' RCA3: root cause analysis reference for 2005.0-4 vs 2001.0-4.' # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr - type: "table" |