aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2020-02-10 14:58:22 +0100
committerTibor Frank <tifrank@cisco.com>2020-02-11 07:36:15 +0100
commit7dc45304cd2e4e877c7c5b1885d242c3977e9245 (patch)
treec46d1739ac7f8351c4f5be845cbd3186391026c1
parentd7ef5191ec1a7561d6109723980d48b85953a9c7 (diff)
Report: Add ucode comparisom
Change-Id: If918806ba5bcda4d87aa2ea2ab4d2c676dc9670a Signed-off-by: Tibor Frank <tifrank@cisco.com>
-rw-r--r--docs/report/vpp_performance_tests/comparisons/bad_good.rst97
-rw-r--r--docs/report/vpp_performance_tests/comparisons/index.rst1
-rw-r--r--resources/tools/presentation/generator_plots.py6
-rw-r--r--resources/tools/presentation/specification.yaml314
4 files changed, 417 insertions, 1 deletions
diff --git a/docs/report/vpp_performance_tests/comparisons/bad_good.rst b/docs/report/vpp_performance_tests/comparisons/bad_good.rst
new file mode 100644
index 0000000000..5660fc6f2b
--- /dev/null
+++ b/docs/report/vpp_performance_tests/comparisons/bad_good.rst
@@ -0,0 +1,97 @@
+
+.. _vpp_bad_good:
+
+The Bad and the Good
+--------------------
+
+.. note::
+
+ Test results have been generated by
+
+ - `FD.io test executor vpp performance job 2n-skx`_,
+ - `FD.io test executor vpp performance job 3n-skx`_,
+ - `FD.io test executor vpp performance job 2n-clx`_,
+
+ with RF result files csit-vpp-perf-|srelease|-\*.zip
+ `archived here <../../_static/archive/>`_.
+
+2n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+ - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.html>`_
+ - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.html>`_
+ - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.txt>`_
+ - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.txt>`_
+ - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.csv>`_
+ - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+ - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.html>`_
+ - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.html>`_
+ - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.txt>`_
+ - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.txt>`_
+ - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.csv>`_
+ - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.csv>`_
+
+3n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+ - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.html>`_
+ - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.html>`_
+ - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.txt>`_
+ - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.txt>`_
+ - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.csv>`_
+ - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+ - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.html>`_
+ - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.html>`_
+ - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.txt>`_
+ - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.txt>`_
+ - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.csv>`_
+ - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.csv>`_
+
+2n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+ - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.html>`_
+ - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.html>`_
+ - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.txt>`_
+ - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.txt>`_
+ - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.csv>`_
+ - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in HTML, ASCII and CSV formats:
+
+ - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.html>`_
+ - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.html>`_
+ - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.txt>`_
+ - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.txt>`_
+ - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.csv>`_
+ - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.csv>`_
diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst
index ed8ce9fc90..7862a63557 100644
--- a/docs/report/vpp_performance_tests/comparisons/index.rst
+++ b/docs/report/vpp_performance_tests/comparisons/index.rst
@@ -16,3 +16,4 @@ Comparisons
3n-skx_vs_2n-skx_testbeds
nic_comparison
soak_vs_ndr
+ bad_good
diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 51f3097f56..b6a393968d 100644
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -205,7 +205,11 @@ def plot_hdrh_lat_by_percentile(plot, input_data):
params=[u"latency", u"throughput", u"parent", u"tags", u"type"]
)[0][0]
elif plot.get(u"filter", None):
- data = input_data.filter_data(plot, continue_on_error=True)
+ data = input_data.filter_data(
+ plot,
+ params=[u"latency", u"throughput", u"parent", u"tags", u"type"],
+ continue_on_error=True
+ )
else:
job = list(plot[u"data"].keys())[0]
build = str(plot[u"data"][job][0])
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 84a56cdf55..309f45ccbc 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -564,6 +564,75 @@
###########################################################################
# Tables (more than one data set)
+ # The GOOD and the BAD
+
+ vpp-performance-3n-skx-bad:
+ csit-vpp-perf-verify-2001-3n-skx:
+ - 5 # NDRPDR sel BAD
+ - 6 # NDRPDR sel BAD
+ - 8 # NDRPDR sel BAD
+ - 9 # NDRPDR sel BAD
+ vpp-performance-3n-skx-good:
+ csit-vpp-perf-verify-2001-3n-skx:
+ - 14 # NDRPDR sel
+ - 15 # NDRPDR sel
+ vpp-performance-3n-skx:
+ csit-vpp-perf-verify-2001-3n-skx:
+ - 5 # NDRPDR sel BAD
+ - 6 # NDRPDR sel BAD
+ - 8 # NDRPDR sel BAD
+ - 9 # NDRPDR sel BAD
+ - 14 # NDRPDR sel
+ - 15 # NDRPDR sel
+
+ vpp-performance-2n-skx-bad:
+ csit-vpp-perf-verify-2001-2n-skx:
+ - 6 # NDRPDR sel BAD
+ - 7 # NDRPDR sel BAD
+ - 10 # NDRPDR sel BAD
+ - 11 # NDRPDR sel BAD
+ - 14 # NDRPDR sel BAD
+ vpp-performance-2n-skx-good:
+ csit-vpp-perf-verify-2001-2n-skx:
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 26 # NDRPDR sel
+ vpp-performance-2n-skx:
+ csit-vpp-perf-verify-2001-2n-skx:
+ - 6 # NDRPDR sel BAD
+ - 7 # NDRPDR sel BAD
+ - 10 # NDRPDR sel BAD
+ - 11 # NDRPDR sel BAD
+ - 14 # NDRPDR sel BAD
+ - 21 # NDRPDR sel
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 26 # NDRPDR sel
+
+ vpp-performance-2n-clx-bad:
+ csit-vpp-perf-verify-2001-2n-clx:
+ - 6 # NDRPDR sel BAD
+ - 7 # NDRPDR sel BAD
+ - 8 # NDRPDR sel BAD
+ - 10 # NDRPDR sel BAD
+ - 11 # NDRPDR sel BAD
+ vpp-performance-2n-clx-good:
+ csit-vpp-perf-verify-2001-2n-clx:
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 24 # NDRPDR sel
+ vpp-performance-2n-clx:
+ csit-vpp-perf-verify-2001-2n-clx:
+ - 6 # NDRPDR sel BAD
+ - 7 # NDRPDR sel BAD
+ - 8 # NDRPDR sel BAD
+ - 10 # NDRPDR sel BAD
+ - 11 # NDRPDR sel BAD
+ - 22 # NDRPDR sel
+ - 23 # NDRPDR sel
+ - 24 # NDRPDR sel
+
# Performance changes
vpp-performance-changes-3n-hsw-h1:
@@ -1892,6 +1961,10 @@
- 67 # NDRPDR sel vhost
- 36 # MRR sel
csit-vpp-perf-verify-2001-3n-skx:
+ - 5 # NDRPDR sel BAD
+ - 6 # NDRPDR sel BAD
+ - 8 # NDRPDR sel BAD
+ - 9 # NDRPDR sel BAD
- 14 # NDRPDR sel
- 15 # NDRPDR sel
@@ -1942,6 +2015,11 @@
- 83 # SOAK
- 86 # SOAK
csit-vpp-perf-verify-2001-2n-skx:
+ - 6 # NDRPDR sel BAD
+ - 7 # NDRPDR sel BAD
+ - 10 # NDRPDR sel BAD
+ - 11 # NDRPDR sel BAD
+ - 14 # NDRPDR sel BAD
- 21 # NDRPDR sel
- 22 # NDRPDR sel
- 23 # NDRPDR sel
@@ -1950,6 +2028,11 @@
- 31 # NFV
csit-vpp-perf-verify-2001-2n-clx:
+ - 6 # NDRPDR sel BAD
+ - 7 # NDRPDR sel BAD
+ - 8 # NDRPDR sel BAD
+ - 10 # NDRPDR sel BAD
+ - 11 # NDRPDR sel BAD
- 22 # NDRPDR sel
- 23 # NDRPDR sel
- 24 # NDRPDR sel
@@ -2475,6 +2558,237 @@
- "tags"
################################################################################
+# The Bad and the good
+
+- type: "table"
+ title: "VPP Performance Comparison 3n-skx 2t1c pdr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-2t1c-pdr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-3n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-3n-skx-good"
+ data: "vpp-performance-3n-skx"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 3n-skx 4t2c pdr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-4t2c-pdr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-3n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-3n-skx-good"
+ data: "vpp-performance-3n-skx"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 3n-skx 2t1c ndr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-2t1c-ndr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-3n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-3n-skx-good"
+ data: "vpp-performance-3n-skx"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 3n-skx 4t2c ndr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-4t2c-ndr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-3n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-3n-skx-good"
+ data: "vpp-performance-3n-skx"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-skx 2t1c pdr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-2t1c-pdr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-skx-good"
+ data: "vpp-performance-2n-skx"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-skx 4t2c pdr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-4t2c-pdr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-skx-good"
+ data: "vpp-performance-2n-skx"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-skx 2t1c ndr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-2t1c-ndr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-skx-good"
+ data: "vpp-performance-2n-skx"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-skx 4t2c ndr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-4t2c-ndr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-skx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-skx-good"
+ data: "vpp-performance-2n-skx"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-clx 2t1c pdr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-2t1c-pdr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-clx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-clx-good"
+ data: "vpp-performance-2n-clx"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-clx 4t2c pdr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-4t2c-pdr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-clx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-clx-good"
+ data: "vpp-performance-2n-clx"
+ include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-clx 2t1c ndr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-2t1c-ndr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-clx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-clx-good"
+ data: "vpp-performance-2n-clx"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+- type: "table"
+ title: "VPP Performance Comparison 2n-clx 4t2c ndr"
+ algorithm: "table_perf_comparison"
+ output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-4t2c-ndr"
+ reference:
+ title: "The Bad"
+ data: "vpp-performance-2n-clx-bad"
+ compare:
+ title: "The Good"
+ data: "vpp-performance-2n-clx-good"
+ data: "vpp-performance-2n-clx"
+ include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+ filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'"
+ parameters:
+ - "name"
+ - "parent"
+ - "throughput"
+ - "tags"
+
+################################################################################
# VPP
# VPP Performance Changes 3n-hsw 1t1c pdr