diff options
4 files changed, 417 insertions, 1 deletions
diff --git a/docs/report/vpp_performance_tests/comparisons/bad_good.rst b/docs/report/vpp_performance_tests/comparisons/bad_good.rst new file mode 100644 index 0000000000..5660fc6f2b --- /dev/null +++ b/docs/report/vpp_performance_tests/comparisons/bad_good.rst @@ -0,0 +1,97 @@ + +.. _vpp_bad_good: + +The Bad and the Good +-------------------- + +.. note:: + + Test results have been generated by + + - `FD.io test executor vpp performance job 2n-skx`_, + - `FD.io test executor vpp performance job 3n-skx`_, + - `FD.io test executor vpp performance job 2n-clx`_, + + with RF result files csit-vpp-perf-|srelease|-\*.zip + `archived here <../../_static/archive/>`_. + +2n-skx +~~~~~~ + +NDR Comparison +`````````````` + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.html>`_ + - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.html>`_ + - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.txt>`_ + - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.txt>`_ + - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-ndr.csv>`_ + - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-ndr.csv>`_ + +PDR Comparison +`````````````` + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.html>`_ + - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.html>`_ + - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.txt>`_ + - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.txt>`_ + - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-2t1c-pdr.csv>`_ + - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-skx-4t2c-pdr.csv>`_ + +3n-skx +~~~~~~ + +NDR Comparison +`````````````` + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.html>`_ + - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.html>`_ + - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.txt>`_ + - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.txt>`_ + - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-ndr.csv>`_ + - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-ndr.csv>`_ + +PDR Comparison +`````````````` + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.html>`_ + - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.html>`_ + - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.txt>`_ + - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.txt>`_ + - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-2t1c-pdr.csv>`_ + - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-3n-skx-4t2c-pdr.csv>`_ + +2n-skx +~~~~~~ + +NDR Comparison +`````````````` + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.html>`_ + - `HTML 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.html>`_ + - `ASCII 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.txt>`_ + - `ASCII 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.txt>`_ + - `CSV 2t1c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-ndr.csv>`_ + - `CSV 4t2c NDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-ndr.csv>`_ + +PDR Comparison +`````````````` + +Comparison tables in HTML, ASCII and CSV formats: + + - `HTML 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.html>`_ + - `HTML 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.html>`_ + - `ASCII 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.txt>`_ + - `ASCII 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.txt>`_ + - `CSV 2t1c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-2t1c-pdr.csv>`_ + - `CSV 4t2c PDR comparison <../../_static/vpp/performance-bad-good-2n-clx-4t2c-pdr.csv>`_ diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst index ed8ce9fc90..7862a63557 100644 --- a/docs/report/vpp_performance_tests/comparisons/index.rst +++ b/docs/report/vpp_performance_tests/comparisons/index.rst @@ -16,3 +16,4 @@ Comparisons 3n-skx_vs_2n-skx_testbeds nic_comparison soak_vs_ndr + bad_good diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py index 51f3097f56..b6a393968d 100644 --- a/resources/tools/presentation/generator_plots.py +++ b/resources/tools/presentation/generator_plots.py @@ -205,7 +205,11 @@ def plot_hdrh_lat_by_percentile(plot, input_data): params=[u"latency", u"throughput", u"parent", u"tags", u"type"] )[0][0] elif plot.get(u"filter", None): - data = input_data.filter_data(plot, continue_on_error=True) + data = input_data.filter_data( + plot, + params=[u"latency", u"throughput", u"parent", u"tags", u"type"], + continue_on_error=True + ) else: job = list(plot[u"data"].keys())[0] build = str(plot[u"data"][job][0]) diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 84a56cdf55..309f45ccbc 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -564,6 +564,75 @@ ########################################################################### # Tables (more than one data set) + # The GOOD and the BAD + + vpp-performance-3n-skx-bad: + csit-vpp-perf-verify-2001-3n-skx: + - 5 # NDRPDR sel BAD + - 6 # NDRPDR sel BAD + - 8 # NDRPDR sel BAD + - 9 # NDRPDR sel BAD + vpp-performance-3n-skx-good: + csit-vpp-perf-verify-2001-3n-skx: + - 14 # NDRPDR sel + - 15 # NDRPDR sel + vpp-performance-3n-skx: + csit-vpp-perf-verify-2001-3n-skx: + - 5 # NDRPDR sel BAD + - 6 # NDRPDR sel BAD + - 8 # NDRPDR sel BAD + - 9 # NDRPDR sel BAD + - 14 # NDRPDR sel + - 15 # NDRPDR sel + + vpp-performance-2n-skx-bad: + csit-vpp-perf-verify-2001-2n-skx: + - 6 # NDRPDR sel BAD + - 7 # NDRPDR sel BAD + - 10 # NDRPDR sel BAD + - 11 # NDRPDR sel BAD + - 14 # NDRPDR sel BAD + vpp-performance-2n-skx-good: + csit-vpp-perf-verify-2001-2n-skx: + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 26 # NDRPDR sel + vpp-performance-2n-skx: + csit-vpp-perf-verify-2001-2n-skx: + - 6 # NDRPDR sel BAD + - 7 # NDRPDR sel BAD + - 10 # NDRPDR sel BAD + - 11 # NDRPDR sel BAD + - 14 # NDRPDR sel BAD + - 21 # NDRPDR sel + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 26 # NDRPDR sel + + vpp-performance-2n-clx-bad: + csit-vpp-perf-verify-2001-2n-clx: + - 6 # NDRPDR sel BAD + - 7 # NDRPDR sel BAD + - 8 # NDRPDR sel BAD + - 10 # NDRPDR sel BAD + - 11 # NDRPDR sel BAD + vpp-performance-2n-clx-good: + csit-vpp-perf-verify-2001-2n-clx: + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 24 # NDRPDR sel + vpp-performance-2n-clx: + csit-vpp-perf-verify-2001-2n-clx: + - 6 # NDRPDR sel BAD + - 7 # NDRPDR sel BAD + - 8 # NDRPDR sel BAD + - 10 # NDRPDR sel BAD + - 11 # NDRPDR sel BAD + - 22 # NDRPDR sel + - 23 # NDRPDR sel + - 24 # NDRPDR sel + # Performance changes vpp-performance-changes-3n-hsw-h1: @@ -1892,6 +1961,10 @@ - 67 # NDRPDR sel vhost - 36 # MRR sel csit-vpp-perf-verify-2001-3n-skx: + - 5 # NDRPDR sel BAD + - 6 # NDRPDR sel BAD + - 8 # NDRPDR sel BAD + - 9 # NDRPDR sel BAD - 14 # NDRPDR sel - 15 # NDRPDR sel @@ -1942,6 +2015,11 @@ - 83 # SOAK - 86 # SOAK csit-vpp-perf-verify-2001-2n-skx: + - 6 # NDRPDR sel BAD + - 7 # NDRPDR sel BAD + - 10 # NDRPDR sel BAD + - 11 # NDRPDR sel BAD + - 14 # NDRPDR sel BAD - 21 # NDRPDR sel - 22 # NDRPDR sel - 23 # NDRPDR sel @@ -1950,6 +2028,11 @@ - 31 # NFV csit-vpp-perf-verify-2001-2n-clx: + - 6 # NDRPDR sel BAD + - 7 # NDRPDR sel BAD + - 8 # NDRPDR sel BAD + - 10 # NDRPDR sel BAD + - 11 # NDRPDR sel BAD - 22 # NDRPDR sel - 23 # NDRPDR sel - 24 # NDRPDR sel @@ -2475,6 +2558,237 @@ - "tags" ################################################################################ +# The Bad and the good + +- type: "table" + title: "VPP Performance Comparison 3n-skx 2t1c pdr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-2t1c-pdr" + reference: + title: "The Bad" + data: "vpp-performance-3n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-3n-skx-good" + data: "vpp-performance-3n-skx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 3n-skx 4t2c pdr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-4t2c-pdr" + reference: + title: "The Bad" + data: "vpp-performance-3n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-3n-skx-good" + data: "vpp-performance-3n-skx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 3n-skx 2t1c ndr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-2t1c-ndr" + reference: + title: "The Bad" + data: "vpp-performance-3n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-3n-skx-good" + data: "vpp-performance-3n-skx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 3n-skx 4t2c ndr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-3n-skx-4t2c-ndr" + reference: + title: "The Bad" + data: "vpp-performance-3n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-3n-skx-good" + data: "vpp-performance-3n-skx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-skx 2t1c pdr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-2t1c-pdr" + reference: + title: "The Bad" + data: "vpp-performance-2n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-skx-good" + data: "vpp-performance-2n-skx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-skx 4t2c pdr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-4t2c-pdr" + reference: + title: "The Bad" + data: "vpp-performance-2n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-skx-good" + data: "vpp-performance-2n-skx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-skx 2t1c ndr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-2t1c-ndr" + reference: + title: "The Bad" + data: "vpp-performance-2n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-skx-good" + data: "vpp-performance-2n-skx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-skx 4t2c ndr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-skx-4t2c-ndr" + reference: + title: "The Bad" + data: "vpp-performance-2n-skx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-skx-good" + data: "vpp-performance-2n-skx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-clx 2t1c pdr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-2t1c-pdr" + reference: + title: "The Bad" + data: "vpp-performance-2n-clx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-clx-good" + data: "vpp-performance-2n-clx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-clx 4t2c pdr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-4t2c-pdr" + reference: + title: "The Bad" + data: "vpp-performance-2n-clx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-clx-good" + data: "vpp-performance-2n-clx" + include-tests: "PDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-clx 2t1c ndr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-2t1c-ndr" + reference: + title: "The Bad" + data: "vpp-performance-2n-clx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-clx-good" + data: "vpp-performance-2n-clx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'2T1C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +- type: "table" + title: "VPP Performance Comparison 2n-clx 4t2c ndr" + algorithm: "table_perf_comparison" + output-file: "{DIR[STATIC,VPP]}/performance-bad-good-2n-clx-4t2c-ndr" + reference: + title: "The Bad" + data: "vpp-performance-2n-clx-bad" + compare: + title: "The Good" + data: "vpp-performance-2n-clx-good" + data: "vpp-performance-2n-clx" + include-tests: "NDR" # "PDR" | "NDR" | "MRR" + filter: "'4T2C' and 'NDRPDR' and not 'NF_DENSITY'" + parameters: + - "name" + - "parent" + - "throughput" + - "tags" + +################################################################################ # VPP # VPP Performance Changes 3n-hsw 1t1c pdr |