aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2018-11-19 14:56:32 +0100
committerTibor Frank <tifrank@cisco.com>2018-11-20 09:34:08 +0000
commit153bf776309870aa615f8217c3b438aad199f5ed (patch)
treeb070847eb5657f3b568473cc9c16d7ee03473378
parentada7009a3c826430e83a29c31bdf9a5973986ad9 (diff)
CSIT-1359: Add 3n-skx and 2n-skx comparisons to Report
- 3n-Skx vs. 2n-Skx Testbeds Change-Id: I63e1d34d4589bbb771961ed3c21f0271fee728ad Signed-off-by: Tibor Frank <tifrank@cisco.com>
-rw-r--r--docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst44
-rw-r--r--docs/report/dpdk_performance_tests/comparisons/index.rst4
-rw-r--r--docs/report/introduction/report_history.rst4
-rw-r--r--docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst43
-rw-r--r--docs/report/vpp_performance_tests/comparisons/index.rst1
-rw-r--r--resources/tools/presentation/generator_tables.py9
-rw-r--r--resources/tools/presentation/specification.yaml258
7 files changed, 360 insertions, 3 deletions
diff --git a/docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst b/docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst
new file mode 100644
index 0000000000..bafbb38e20
--- /dev/null
+++ b/docs/report/dpdk_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst
@@ -0,0 +1,44 @@
+
+.. _dpdk_compare_topologies_3n-Skx_vs_2n-Skx:
+
+3n-Skx vs. 2n-Skx Testbeds
+--------------------------
+
+Relative comparison of |dpdk-release| Testpmd and L3fwd packet
+throughput (NDR, PDR and MRR) is calculated for the same tests executed
+on 3-Node Skylake (3n-skx) and 2-Node Skylake (2n-skx) physical testbed
+types, in 1-core, 2-core and 4-core configurations.
+
+.. note::
+
+ Test results have been generated by
+ `FD.io test executor dpdk performance job 3n-skx`_ and
+ `FD.io test executor dpdk performance job 2n-skx`_
+ with RF result
+ files csit-dpdk-perf-|srelease|-\*.zip
+ `archived here <../../_static/archive/>`_.
+
+NDR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in ASCII and CSV formats:
+
+ - `ASCII NDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-ndr.txt>`_
+ - `CSV NDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-ndr.csv>`_
+
+PDR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in ASCII and CSV formats:
+
+ - `ASCII PDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-pdr.txt>`_
+ - `CSV PDR comparison <../../_static/dpdk/performance-compare-topologies-3n-skx-2n-skx-pdr.csv>`_
+
+..
+ MRR Comparison
+ ~~~~~~~~~~~~~~
+
+ Comparison tables in ASCII and CSV formats:
+
+ - `ASCII MRR comparison <../../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.txt>`_
+ - `CSV MRR comparison <../../_static/dpdk/performance-compare-testbeds-3n-hsw-3n-skx-mrr.csv>`_
diff --git a/docs/report/dpdk_performance_tests/comparisons/index.rst b/docs/report/dpdk_performance_tests/comparisons/index.rst
index ace3baf091..7a3cb44444 100644
--- a/docs/report/dpdk_performance_tests/comparisons/index.rst
+++ b/docs/report/dpdk_performance_tests/comparisons/index.rst
@@ -4,10 +4,10 @@
\clearpage
Comparisons
-=============
+===========
.. toctree::
current_vs_previous_release
3n-skx_vs_3n-hsw_testbeds
-
+ 3n-skx_vs_2n-skx_testbeds
diff --git a/docs/report/introduction/report_history.rst b/docs/report/introduction/report_history.rst
index eef74febda..7c1ecae9e2 100644
--- a/docs/report/introduction/report_history.rst
+++ b/docs/report/introduction/report_history.rst
@@ -20,6 +20,10 @@ Document History
| | :ref:`dpdk_compare_current_vs_previous_release` |
| | |
| | 6. Changed title of this chapter to "Document History" |
+| | 7. Added comparisons between topologies: |
+| | |
+| | a. VPP: :ref:`vpp_compare_topologies_3n-Skx_vs_2n-Skx` |
+| | b. DPDK: :ref:`dpdk_compare_topologies_3n-Skx_vs_2n-Skx` |
| | |
+---------+--------------------------------------------------------------------+
| .w46 | 1. dot1q KVM VMs vhost-user tests added to |
diff --git a/docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst b/docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst
new file mode 100644
index 0000000000..38de3b4a41
--- /dev/null
+++ b/docs/report/vpp_performance_tests/comparisons/3n-skx_vs_2n-skx_testbeds.rst
@@ -0,0 +1,43 @@
+
+.. _vpp_compare_topologies_3n-Skx_vs_2n-Skx:
+
+3n-Skx vs. 2n-Skx Testbeds
+--------------------------
+
+Relative comparison of |vpp-release| packet throughput (NDR, PDR and
+MRR) is calculated for the same tests executed on 3-Node Skylake (3n-
+skx) and 2-Node Skylake (2n-skx) physical testbed types, in 1-core,
+2-core and 4-core configurations.
+
+.. note::
+
+ Test results have been generated by
+ `FD.io test executor vpp performance job 3n-skx`_ and
+ `FD.io test executor vpp performance job 2n-skx`_
+ with RF result
+ files csit-vpp-perf-|srelease|-\*.zip
+ `archived here <../../_static/archive/>`_.
+
+NDR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in ASCII and CSV formats:
+
+ - `ASCII NDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-ndr.txt>`_
+ - `CSV NDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-ndr.csv>`_
+
+PDR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in ASCII and CSV formats:
+
+ - `ASCII PDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-pdr.txt>`_
+ - `CSV PDR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-pdr.csv>`_
+
+MRR Comparison
+~~~~~~~~~~~~~~
+
+Comparison tables in ASCII and CSV formats:
+
+ - `ASCII MRR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-mrr.txt>`_
+ - `CSV MRR comparison <../../_static/vpp/performance-compare-topologies-3n-skx-2n-skx-mrr.csv>`_
diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst
index a9179c6900..52367ca423 100644
--- a/docs/report/vpp_performance_tests/comparisons/index.rst
+++ b/docs/report/vpp_performance_tests/comparisons/index.rst
@@ -10,4 +10,5 @@ Comparisons
current_vs_previous_release
3n-skx_vs_3n-hsw_testbeds
+ 3n-skx_vs_2n-skx_testbeds
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index b24222c7f0..a3373db6d8 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -240,11 +240,14 @@ def table_performance_comparison(table, input_data):
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
+ if "across topologies" in table["title"].lower():
+ tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
"-".join(tst_data["name"].
split("-")[:-1]))
- if "comparison across testbeds" in table["title"].lower():
+ if "across testbeds" in table["title"].lower() or \
+ "across topologies" in table["title"].lower():
name = name.\
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
@@ -286,6 +289,8 @@ def table_performance_comparison(table, input_data):
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
+ if "across topologies" in table["title"].lower():
+ tst_name_mod = tst_name_mod.replace("2n1l-", "")
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
@@ -323,6 +328,8 @@ def table_performance_comparison(table, input_data):
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
+ if "across topologies" in table["title"].lower():
+ tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
continue
if tbl_dict[tst_name_mod].get("history", None) is None:
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 816f21134b..918f28069e 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -239,6 +239,42 @@
- 12 # MRR sel
- 13 # MRR sel
+ vpp-compare-topologies:
+ csit-vpp-perf-verify-1810-3n-skx:
+ - 10 # NDRPDR sel
+ - 14 # NDRPDR sel
+ - 15 # NDRPDR sel
+ - 16 # NDRPDR sel
+ - 17 # NDRPDR sel
+ - 19 # NDRPDR sel
+ - 20 # NDRPDR sel
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 25 # NDRPDR sel
+ csit-vpp-perf-verify-1810-2n-skx:
+ - 9 # NDRPDR sel
+ - 13 # NDRPDR sel
+ - 14 # NDRPDR sel
+ - 19 # NDRPDR sel
+ - 20 # NDRPDR sel
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 24 # NDRPDR sel
+ - 27 # NDRPDR sel
+
+ vpp-compare-topologies-mrr:
+ csit-vpp-perf-verify-1810-3n-skx:
+ - 9 # MRR sel
+ - 11 # MRR sel
+ - 12 # MRR sel
+ - 13 # MRR sel
+ csit-vpp-perf-verify-1810-2n-skx:
+ - 8 # MRR sel
+ - 10 # MRR sel
+ - 11 # MRR sel
+ - 12 # MRR sel
+
vpp-performance-changes-3n-hsw-mrr:
csit-vpp-perf-check-1804:
- 6 # mrr - sel
@@ -552,6 +588,30 @@
- 9 # NDRPDR sel
- 10 # NDRPDR sel
+ dpdk-compare-topologies:
+ csit-dpdk-perf-verify-1810-3n-skx:
+ - 3 # NDRPDR sel
+ - 4 # NDRPDR sel
+ - 6 # NDRPDR sel
+ - 7 # NDRPDR sel
+ - 8 # NDRPDR sel
+ - 9 # NDRPDR sel
+ - 10 # NDRPDR sel
+ - 11 # NDRPDR sel
+ - 12 # NDRPDR sel
+ - 13 # NDRPDR sel
+ csit-dpdk-perf-verify-1810-2n-skx:
+ - 3 # NDRPDR sel
+ - 4 # NDRPDR sel
+ - 5 # NDRPDR sel
+ - 7 # NDRPDR sel
+ - 8 # NDRPDR sel
+ - 9 # NDRPDR sel
+ - 10 # NDRPDR sel
+ - 11 # NDRPDR sel
+ - 12 # NDRPDR sel
+ - 13 # NDRPDR sel
+
dpdk-perf-results-3n-hsw:
csit-dpdk-perf-verify-1810-3n-hsw:
- 1
@@ -1874,6 +1934,90 @@
- "parent"
- "throughput"
+# VPP Comparison Across Topologies 3n-skx to 2n-skx ndr
+-
+ type: "table"
+ title: "VPP Comparison Across Topologies 3n-skx to 2n-skx ndr"
+ algorithm: "table_performance_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-compare-topologies-3n-skx-2n-skx-ndr"
+ reference:
+ title: "3-Node Skx"
+ data:
+ csit-vpp-perf-verify-1810-3n-skx:
+ - 10 # NDRPDR sel
+ - 14 # NDRPDR sel
+ - 15 # NDRPDR sel
+ - 16 # NDRPDR sel
+ - 17 # NDRPDR sel
+ - 19 # NDRPDR sel
+ - 20 # NDRPDR sel
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 25 # NDRPDR sel
+ compare:
+ title: "2-Node Skx"
+ data:
+ csit-vpp-perf-verify-1810-2n-skx:
+ - 9 # NDRPDR sel
+ - 13 # NDRPDR sel
+ - 14 # NDRPDR sel
+ - 19 # NDRPDR sel
+ - 20 # NDRPDR sel
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 24 # NDRPDR sel
+ - 27 # NDRPDR sel
+ data: "vpp-compare-topologies"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'NDRPDR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+
+# VPP Comparison Across Topologies 3n-skx to 2n-skx pdr
+-
+ type: "table"
+ title: "VPP Comparison Across Topologies 3n-skx to 2n-skx pdr"
+ algorithm: "table_performance_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-compare-topologies-3n-skx-2n-skx-pdr"
+ reference:
+ title: "3-Node Skx"
+ data:
+ csit-vpp-perf-verify-1810-3n-skx:
+ - 10 # NDRPDR sel
+ - 14 # NDRPDR sel
+ - 15 # NDRPDR sel
+ - 16 # NDRPDR sel
+ - 17 # NDRPDR sel
+ - 19 # NDRPDR sel
+ - 20 # NDRPDR sel
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 25 # NDRPDR sel
+ compare:
+ title: "2-Node Skx"
+ data:
+ csit-vpp-perf-verify-1810-2n-skx:
+ - 9 # NDRPDR sel
+ - 13 # NDRPDR sel
+ - 14 # NDRPDR sel
+ - 19 # NDRPDR sel
+ - 20 # NDRPDR sel
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 24 # NDRPDR sel
+ - 27 # NDRPDR sel
+ data: "vpp-compare-topologies"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'NDRPDR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+
# VPP Performance Changes 3n-hsw 1t1c MRR
-
type: "table"
@@ -2255,6 +2399,36 @@
- "parent"
- "result"
+# VPP Comparison Across Topologies 3n-skx to 2n-skx mrr
+-
+ type: "table"
+ title: "VPP Comparison Across Topologies 3n-skx to 2n-skx mrr"
+ algorithm: "table_performance_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-compare-topologies-3n-skx-2n-skx-mrr"
+ reference:
+ title: "3-Node Skx"
+ data:
+ csit-vpp-perf-verify-1810-3n-skx:
+ - 9 # MRR sel
+ - 11 # MRR sel
+ - 12 # MRR sel
+ - 13 # MRR sel
+ compare:
+ title: "2-Node Skx"
+ data:
+ csit-vpp-perf-verify-1810-2n-skx:
+ - 8 # MRR sel
+ - 10 # MRR sel
+ - 11 # MRR sel
+ - 12 # MRR sel
+ data: "vpp-compare-topologies-mrr"
+ include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+ filter: "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "result"
+
# Detailed Test Results - VPP Performance Results 3n-hsw
-
type: "table"
@@ -3192,6 +3366,90 @@
- "parent"
- "throughput"
+# DPDK Comparison Across Topologies 3n-skx to 2n-skx ndr
+-
+ type: "table"
+ title: "DPDK Comparison Across Topologies 3n-skx to 2n-skx ndr"
+ algorithm: "table_performance_comparison"
+ output-file: "{DIR[STATIC,DPDK]}/performance-compare-topologies-3n-skx-2n-skx-ndr"
+ reference:
+ title: "3-Node Skx"
+ data:
+ csit-dpdk-perf-verify-1810-3n-skx:
+ - 3 # NDRPDR sel
+ - 4 # NDRPDR sel
+ - 6 # NDRPDR sel
+ - 7 # NDRPDR sel
+ - 8 # NDRPDR sel
+ - 9 # NDRPDR sel
+ - 10 # NDRPDR sel
+ - 11 # NDRPDR sel
+ - 12 # NDRPDR sel
+ - 13 # NDRPDR sel
+ compare:
+ title: "2-Node Skx"
+ data:
+ csit-dpdk-perf-verify-1810-2n-skx:
+ - 3 # NDRPDR sel
+ - 4 # NDRPDR sel
+ - 5 # NDRPDR sel
+ - 7 # NDRPDR sel
+ - 8 # NDRPDR sel
+ - 9 # NDRPDR sel
+ - 10 # NDRPDR sel
+ - 11 # NDRPDR sel
+ - 12 # NDRPDR sel
+ - 13 # NDRPDR sel
+ data: "dpdk-compare-topologies"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+
+# DPDK Comparison Across Topologies 3n-skx to 2n-skx pdr
+-
+ type: "table"
+ title: "DPDK Comparison Across Topologies 3n-skx to 2n-skx pdr"
+ algorithm: "table_performance_comparison"
+ output-file: "{DIR[STATIC,DPDK]}/performance-compare-topologies-3n-skx-2n-skx-pdr"
+ reference:
+ title: "3-Node Skx"
+ data:
+ csit-dpdk-perf-verify-1810-3n-skx:
+ - 3 # NDRPDR sel
+ - 4 # NDRPDR sel
+ - 6 # NDRPDR sel
+ - 7 # NDRPDR sel
+ - 8 # NDRPDR sel
+ - 9 # NDRPDR sel
+ - 10 # NDRPDR sel
+ - 11 # NDRPDR sel
+ - 12 # NDRPDR sel
+ - 13 # NDRPDR sel
+ compare:
+ title: "2-Node Skx"
+ data:
+ csit-dpdk-perf-verify-1810-2n-skx:
+ - 3 # NDRPDR sel
+ - 4 # NDRPDR sel
+ - 5 # NDRPDR sel
+ - 7 # NDRPDR sel
+ - 8 # NDRPDR sel
+ - 9 # NDRPDR sel
+ - 10 # NDRPDR sel
+ - 11 # NDRPDR sel
+ - 12 # NDRPDR sel
+ - 13 # NDRPDR sel
+ data: "dpdk-compare-topologies"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+
# DPDK Comparison Across Testbeds 3n-hsw to 3n-skx pdr
-
type: "table"