From 1fc4b0592f127c83d3e4057dd96366e0a28f1c56 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Thu, 2 Aug 2018 15:34:44 +0200 Subject: CSIT-1208: Add new data to 1807 report Change-Id: I93c6625d537bfef3e955e171db92e5a639ffc22e Signed-off-by: Tibor Frank (cherry picked from commit b37edd789d18a621896b26a42048bd6929742e0a) --- .../dpdk_performance_tests/csit_release_notes.rst | 39 +++ .../vpp_performance_tests/csit_release_notes.rst | 7 +- .../throughput_speedup_multi_core/index.rst | 1 + resources/tools/presentation/specification.yaml | 274 +++++++++++++-------- 4 files changed, 219 insertions(+), 102 deletions(-) diff --git a/docs/report/dpdk_performance_tests/csit_release_notes.rst b/docs/report/dpdk_performance_tests/csit_release_notes.rst index 28e6a37768..d073c493fe 100644 --- a/docs/report/dpdk_performance_tests/csit_release_notes.rst +++ b/docs/report/dpdk_performance_tests/csit_release_notes.rst @@ -73,6 +73,27 @@ formats: files csit-dpdk-perf-|srelease|-\*.zip `archived here <../_static/archive/>`_. +MRR Changes +~~~~~~~~~~~ + +MRR small packet throughput changes between releases are available in a +CSV and pretty ASCII formats: + + - `csv format for 1t1c <../_static/dpdk/performance-changes-1t1c-mrr.csv>`_, + - `csv format for 2t2c <../_static/dpdk/performance-changes-2t2c-mrr.csv>`_, + - `csv format for 4t4c <../_static/dpdk/performance-changes-4t4c-mrr.csv>`_, + - `pretty ASCII format for 1t1c <../_static/dpdk/performance-changes-1t1c-mrr.txt>`_, + - `pretty ASCII format for 2t2c <../_static/dpdk/performance-changes-2t2c-mrr.txt>`_, + - `pretty ASCII format for 4t4c <../_static/dpdk/performance-changes-4t4c-mrr.txt>`_. + +.. note:: + + Test results have been generated by + `FD.io test executor dpdk performance job 3n-hsw`_, + with Robot Framework result + files csit-vpp-perf-|srelease|-\*.zip + `archived here <../_static/archive/>`_. + Comparison Across Testbeds -------------------------- @@ -115,6 +136,24 @@ formats: files csit-dpdk-perf-|srelease|-\*.zip `archived here <../_static/archive/>`_. +MRR Changes +~~~~~~~~~~~ + +MRR throughput changes between testbeds are available in a +CSV and pretty ASCII formats: + + - `csv format for mrr <../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.csv>`_, + - `pretty ASCII format for mrr <../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.txt>`_. + +.. note:: + + Test results have been generated by + `FD.io test executor dpdk performance job 3n-hsw`_ and + `FD.io test executor dpdk performance job 3n-skx`_ + with Robot Framework result + files csit-vpp-perf-|srelease|-\*.zip + `archived here <../_static/archive/>`_. + Known Issues ------------ diff --git a/docs/report/vpp_performance_tests/csit_release_notes.rst b/docs/report/vpp_performance_tests/csit_release_notes.rst index a2fb1c70b0..90058d3526 100644 --- a/docs/report/vpp_performance_tests/csit_release_notes.rst +++ b/docs/report/vpp_performance_tests/csit_release_notes.rst @@ -121,9 +121,10 @@ CSV and pretty ASCII formats: .. note:: Test results have been generated by - `FD.io test executor vpp mrr jobs `_ - with Robot Framework result files csit-vpp-perf-mrr-daily-master__*__output.xml.gz - `archived here `_. + `FD.io test executor vpp performance job 3n-hsw`_, + with Robot Framework result + files csit-vpp-perf-|srelease|-\*.zip + `archived here <../_static/archive/>`_. Comparison Across Testbeds -------------------------- diff --git a/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst b/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst index 2417ce35dd..2042eb6c33 100644 --- a/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst +++ b/docs/report/vpp_performance_tests/throughput_speedup_multi_core/index.rst @@ -33,3 +33,4 @@ threaded VPP configurations relative to 1-core configurations. container_memif container_orchestrated ipsec + vts diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 3295aecba1..83d5bc5383 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -125,10 +125,13 @@ - 31 # sel - 35 # sel # TODO: Add Data sources - # 1807 + # csit-vpp-perf-verify-1807-3n-hsw: + # - # NDRPDR sel + vpp-compare-testbeds: # TODO: Add Data sources - # HSW + # csit-vpp-perf-verify-1807-3n-hsw: + # - # NDRPDR sel csit-vpp-perf-1801-all: - 124 # sel - 127 # sel @@ -143,25 +146,23 @@ - 167 # sel - 172 # sel acl only # TODO: Add Data sources - # SKX - csit-vpp-perf-1804-all: - - 21 # sel - - 22 # sel - - 23 # sel - - 24 # sel - - 27 # sel - - 28 # sel - - 29 # sel - - 30 # sel - - 31 # sel - - 35 # sel + csit-vpp-perf-verify-1807-3n-skx: + - 33 # NDRPDR sel + - 35 # NDRPDR sel + - 37 # NDRPDR sel + - 38 # NDRPDR sel + - 39 # NDRPDR sel + vpp-compare-testbeds-mrr: # TODO: Add Data sources csit-vpp-perf-verify-1807-3n-hsw: - 9 # MRR sel # TODO: Add Data sources csit-vpp-perf-verify-1807-3n-skx: - - 21 # MRR sel + - 28 # MRR sel + - 32 # MRR sel + - 36 # MRR sel + - 42 # MRR sel vpp-performance-changes-mrr: csit-vpp-perf-check-1801: @@ -189,6 +190,7 @@ # TODO: Add Data sources csit-vpp-perf-verify-1807-3n-hsw: - 9 # MRR sel + vpp-perf-results-3n-hsw: # TODO: Add Data sources # csit-vpp-perf-verify-1807-3n-hsw: @@ -197,6 +199,7 @@ - 20 # full - 25 # full - 49 # full + vpp-perf-results-3n-skx: # TODO: Add Data sources # csit-vpp-perf-verify-1807-3n-skx: @@ -205,26 +208,25 @@ - 20 # full - 25 # full - 49 # full + vpp-perf-results-2n-skx: # TODO: Add Data sources - # csit-vpp-perf-verify-1807-2n-skx: - csit-vpp-perf-1804-all: - - 19 # full - - 20 # full - - 25 # full - - 49 # full + csit-vpp-perf-verify-1807-2n-skx: + - 32 # NDRPDR full + vpp-mrr-results-3n-hsw: - # TODO: Replace with full csit-vpp-perf-verify-1807-3n-hsw: - - 9 # MRR sel + - 8 # MRR full + vpp-mrr-results-3n-skx: - # TODO: Replace with full + # TODO: Replace with better results csit-vpp-perf-verify-1807-3n-skx: - - 21 # MRR sel + - 31 # MRR full + vpp-mrr-results-2n-skx: - # TODO: Replace with full csit-vpp-perf-verify-1807-2n-skx: - - 22 # MRR sel + - 27 # MRR full + plot-throughput-speedup-analysis-3n-hsw: # TODO: Add Data sources # csit-vpp-perf-verify-1807-3n-hsw: @@ -243,30 +245,30 @@ - 30 # sel - 31 # sel - 35 # sel + plot-throughput-speedup-analysis-3n-skx: # TODO: Add Data sources - # csit-vpp-perf-verify-1807-3n-skx: - csit-vpp-perf-1804-all: - - 19 # full - - 20 # full - - 25 # full - - 49 # full - - 21 # sel - - 22 # sel - - 23 # sel - - 24 # sel - - 27 # sel - - 28 # sel - - 29 # sel - - 30 # sel - - 31 # sel - - 35 # sel + csit-vpp-perf-verify-1807-3n-skx: + - 33 # NDRPDR sel + - 35 # NDRPDR sel + - 37 # NDRPDR sel + - 38 # NDRPDR sel + - 39 # NDRPDR sel + plot-throughput-speedup-analysis-2n-skx: # TODO: Add Data sources csit-vpp-perf-verify-1807-2n-skx: - - 18 # NDRPDR sel - - 19 # NDRPDR sel - - 20 # NDRPDR sel + - 26 # NDRPDR sel + - 28 # NDRPDR sel + - 29 # NDRPDR sel + - 30 # NDRPDR sel + - 34 # NDRPDR sel + - 35 # NDRPDR sel + - 36 # NDRPDR sel + - 39 # NDRPDR sel + - 40 # NDRPDR sel + - 42 # NDRPDR sel + plot-vpp-throughput-latency-3n-hsw: # TODO: Add Data sources # csit-vpp-perf-verify-1807-3n-hsw: @@ -285,29 +287,29 @@ - 30 # sel - 31 # sel - 35 # sel + plot-vpp-throughput-latency-3n-skx: # TODO: Add Data sources - # csit-vpp-perf-verify-1807-3n-skx: - csit-vpp-perf-1804-all: - - 19 # full - - 20 # full - - 25 # full - - 49 # full - - 21 # sel - - 22 # sel - - 23 # sel - - 24 # sel - - 27 # sel - - 28 # sel - - 29 # sel - - 30 # sel - - 31 # sel - - 35 # sel + csit-vpp-perf-verify-1807-3n-skx: + - 33 # NDRPDR sel + - 35 # NDRPDR sel + - 37 # NDRPDR sel + - 38 # NDRPDR sel + - 39 # NDRPDR sel + plot-vpp-throughput-latency-2n-skx: csit-vpp-perf-verify-1807-2n-skx: - - 18 # NDRPDR sel - - 19 # NDRPDR sel - - 20 # NDRPDR sel + - 26 # NDRPDR sel + - 28 # NDRPDR sel + - 29 # NDRPDR sel + - 30 # NDRPDR sel + - 34 # NDRPDR sel + - 35 # NDRPDR sel + - 36 # NDRPDR sel + - 39 # NDRPDR sel + - 40 # NDRPDR sel + - 42 # NDRPDR sel + plot-vpp-http-server-performance: # TODO: Add Data sources csit-vpp-perf-1804-all: @@ -327,6 +329,7 @@ csit-vpp-functional-1807-ubuntu1604-virl: # TODO: Add Data sources - "lastSuccessfulBuild" + vpp-func-results-centos: csit-vpp-functional-1807-centos7-virl: # TODO: Add Data sources @@ -334,13 +337,15 @@ # VPP Ligato ligato-perf-results-3n-hsw: - # TODO: Replace with full + # TODO: Replace with better results csit-ligato-perf-1807-all: - - 4 # sel + - 3 # full ? + plot-ligato-throughput-speedup-analysis-3n-hsw: # TODO: Add Data sources csit-ligato-perf-1807-all: - 4 # sel + plot-ligato-throughput-latency-3n-hsw: # TODO: Add Data sources csit-ligato-perf-1807-all: @@ -370,6 +375,7 @@ - 21 - 22 - 23 + dpdk-compare-testbeds: csit-dpdk-perf-verify-1807-3n-skx: - 1 @@ -393,15 +399,19 @@ - 21 - 22 - 23 + dpdk-perf-results-3n-hsw: csit-dpdk-perf-verify-1807-3n-hsw: - 12 + dpdk-perf-results-3n-skx: csit-dpdk-perf-verify-1807-3n-skx: - 12 + dpdk-perf-results-2n-skx: csit-dpdk-perf-verify-1807-2n-skx: - 10 + plot-dpdk-throughput-latency-3n-hsw: csit-dpdk-perf-verify-1807-3n-hsw: - 11 @@ -414,6 +424,7 @@ - 21 - 22 - 23 + plot-dpdk-throughput-latency-3n-skx: csit-dpdk-perf-verify-1807-3n-skx: - 1 @@ -426,6 +437,7 @@ - 10 - 11 - 12 + plot-dpdk-throughput-latency-2n-skx: csit-dpdk-perf-verify-1807-2n-skx: - 1 @@ -712,17 +724,40 @@ - 48 # wrk # TODO: Add Data sources csit-vpp-perf-verify-1807-3n-hsw: + - 8 # MRR full - 9 # MRR sel # TODO: Add Data sources csit-vpp-perf-verify-1807-3n-skx: - - 21 # MRR sel + - 31 # MRR full + - 28 # MRR sel + - 32 # MRR sel + - 36 # MRR sel + - 42 # MRR sel + - 33 # NDRPDR sel + - 35 # NDRPDR sel + - 37 # NDRPDR sel + - 38 # NDRPDR sel + - 39 # NDRPDR sel # TODO: Add Data sources csit-vpp-perf-verify-1807-2n-skx: - - 17 # MRR sel - - 18 # NDRPDR sel - - 19 # NDRPDR sel - - 20 # NDRPDR sel - - 22 # MRR sel + - 27 # MRR full + - 31 # MRR sel + - 33 # MRR sel + - 37 # MRR sel + - 38 # MRR sel + - 41 # MRR sel + - 43 # MRR sel + - 32 # NDRPDR full + - 26 # NDRPDR sel + - 28 # NDRPDR sel + - 29 # NDRPDR sel + - 30 # NDRPDR sel + - 34 # NDRPDR sel + - 35 # NDRPDR sel + - 36 # NDRPDR sel + - 39 # NDRPDR sel + - 40 # NDRPDR sel + - 42 # NDRPDR sel csit-vpp-perf-check-1801: - 3 # mrr - 4 # mrr @@ -760,6 +795,7 @@ - 14 # sel # TODO: Add Data sources csit-ligato-perf-1807-all: + - 3 # full ? - 4 # sel csit-dpdk-perf-1804-all: - 4 @@ -824,8 +860,8 @@ format: html: - full -# pdf: -# - minimal + pdf: + - minimal ################################################################################ ### T A B L E S ### @@ -873,6 +909,7 @@ title: "rls1807" data: # TODO: Add Data sources + # csit-vpp-perf-verify-1807-3n-hsw: csit-vpp-perf-1804-all: - 21 # sel - 22 # sel @@ -932,6 +969,7 @@ title: "rls1807" data: # TODO: Add Data sources + # csit-vpp-perf-verify-1807-3n-hsw: csit-vpp-perf-1804-all: - 21 # sel - 22 # sel @@ -991,6 +1029,7 @@ title: "rls1807" data: # TODO: Add Data sources + # csit-vpp-perf-verify-1807-3n-hsw: csit-vpp-perf-1804-all: - 21 # sel - 22 # sel @@ -1050,6 +1089,7 @@ title: "rls1807" data: # TODO: Add Data sources + # csit-vpp-perf-verify-1807-3n-hsw: csit-vpp-perf-1804-all: - 21 # sel - 22 # sel @@ -1078,6 +1118,7 @@ title: "3-Node Hsw" data: # TODO: Add Data sources + # csit-vpp-perf-verify-1807-3n-hsw: csit-vpp-perf-1801-all: - 124 # sel - 127 # sel @@ -1095,17 +1136,12 @@ title: "3-Node Skx" data: # TODO: Add Data sources - csit-vpp-perf-1804-all: - - 21 # sel - - 22 # sel - - 23 # sel - - 24 # sel - - 27 # sel - - 28 # sel - - 29 # sel - - 30 # sel - - 31 # sel - - 35 # sel + csit-vpp-perf-verify-1807-3n-skx: + - 33 # NDRPDR sel + - 35 # NDRPDR sel + - 37 # NDRPDR sel + - 38 # NDRPDR sel + - 39 # NDRPDR sel data: "vpp-compare-testbeds" include-tests: "NDR" # "PDR" | "NDR" | "MRR" filter: "'NIC_Intel-X710' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'" @@ -1123,6 +1159,7 @@ title: "3-Node Hsw" data: # TODO: Add Data sources + # csit-vpp-perf-verify-1807-3n-hsw: csit-vpp-perf-1801-all: - 124 # sel - 127 # sel @@ -1140,17 +1177,12 @@ title: "3-Node Skx" data: # TODO: Add Data sources - csit-vpp-perf-1804-all: - - 21 # sel - - 22 # sel - - 23 # sel - - 24 # sel - - 27 # sel - - 28 # sel - - 29 # sel - - 30 # sel - - 31 # sel - - 35 # sel + csit-vpp-perf-verify-1807-3n-skx: + - 33 # NDRPDR sel + - 35 # NDRPDR sel + - 37 # NDRPDR sel + - 38 # NDRPDR sel + - 39 # NDRPDR sel data: "vpp-compare-testbeds" include-tests: "PDR" # "PDR" | "NDR" | "MRR" filter: "'NIC_Intel-X710' and ('NDRPDR' or 'NDRPDRDISC')" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'" @@ -1319,14 +1351,17 @@ data: # TODO: Add Data sources csit-vpp-perf-verify-1807-3n-skx: - - 21 # MRR sel + - 28 # MRR sel + - 32 # MRR sel + - 36 # MRR sel + - 42 # MRR sel data: "vpp-compare-testbeds-mrr" include-tests: "MRR" # "PDR" | "NDR" | "MRR" filter: "'NIC_Intel-X710' and MRR" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'" parameters: - "name" - "parent" - - "throughput" + - "result" - type: "table" @@ -2033,6 +2068,47 @@ - "parent" - "throughput" +- + type: "table" + title: "DPDK Comparison Across Testbeds 3n-hsw to 3n-skx mrr" + algorithm: "table_performance_comparison" + output-file: "{DIR[STATIC,DPDK]}/performance-compare-testbeds-3n-hsw-3n-skx-mrr" + reference: + title: "3-Node Hsw" + data: + csit-dpdk-perf-verify-1807-3n-hsw: + - 11 + - 12 # full + - 14 + - 15 + - 16 + - 17 + - 20 + - 21 + - 22 + - 23 + compare: + title: "3-Node Skx" + data: + csit-dpdk-perf-verify-1807-3n-skx: + - 1 + - 2 + - 3 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + data: "dpdk-compare-testbeds" + include-tests: "MRR" # "PDR" | "NDR" | "MRR" + filter: "'NIC_Intel-X710' and MRR" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'" + parameters: + - "name" + - "parent" + - "throughput" + - type: "table" title: "Detailed Test Results - DPDK Performance Results 3n-hsw" -- cgit 1.2.3-korg