CSIT-1395: Create comparison tables for NICs

Change-Id: Ib8583041a0d16a6242160b4f51eae973ef33504f
Signed-off-by: Tibor Frank <tifrank@cisco.com>

6 files changed, 761 insertions, 12 deletions

diff --git a/docs/report/vpp_performance_tests/comparisons/current_vs_previous_release.rst b/docs/report/vpp_performance_tests/comparisons/current_vs_previous_release.rst
index 6545b5ec1f..f15f6932aa 100644
--- a/docs/report/vpp_performance_tests/comparisons/current_vs_previous_release.rst
+++ b/docs/report/vpp_performance_tests/comparisons/current_vs_previous_release.rst
@@ -22,9 +22,12 @@ results repeatability vary between the test cases.
 .. note::
 
     Test results have been generated by
-    `FD.io test executor vpp performance job 3n-hsw`_,
-    with RF result
-    files csit-vpp-perf-|srelease|-\*.zip
+
+    - `FD.io test executor vpp performance job 3n-hsw`_,
+    - `FD.io test executor vpp performance job 3n-skx`_,
+    - `FD.io test executor vpp performance job 2n-skx`_
+
+    with RF result files csit-vpp-perf-|srelease|-\*.zip
     `archived here <../../_static/archive/>`_.
 
 3n-hsw
diff --git a/docs/report/vpp_performance_tests/comparisons/index.rst b/docs/report/vpp_performance_tests/comparisons/index.rst
index d706d33581..8c502c49d7 100644
--- a/docs/report/vpp_performance_tests/comparisons/index.rst
+++ b/docs/report/vpp_performance_tests/comparisons/index.rst
@@ -13,4 +13,5 @@ Comparisons
     current_vs_previous_release
     3n-skx_vs_3n-hsw_testbeds
     3n-skx_vs_2n-skx_testbeds
+    nic_comparison
 
diff --git a/docs/report/vpp_performance_tests/comparisons/nic_comparison.rst b/docs/report/vpp_performance_tests/comparisons/nic_comparison.rst
new file mode 100644
index 0000000000..6db2756c38
--- /dev/null
+++ b/docs/report/vpp_performance_tests/comparisons/nic_comparison.rst
@@ -0,0 +1,135 @@
+
+.. _vpp_compare_nics_release:
+
+NICs Comparison
+---------------
+
+Relative comparison of VPP packet throughput (NDR, PDR and MRR) between
+NICs (measured for |csit-release) is calculated from results of tests
+running on 3-Node Intel Xeon Haswell testbeds (3n-hsw) in 1-core, 2-core
+and 4-core (MRR only) configurations.
+
+Listed mean and standard deviation values are computed based on a series
+of the same tests executed against respective VPP releases to verify
+test results repeatability, with percentage change calculated for mean
+values. Note that the standard deviation is quite high for a small
+number of packet throughput tests, what indicates poor test results
+repeatability and makes the relative change of mean throughput value not
+fully representative for these tests. The root causes behind poor
+results repeatability vary between the test cases.
+
+.. note::
+
+    Test results have been generated by
+
+    - `FD.io test executor vpp performance job 3n-hsw`_,
+    - `FD.io test executor vpp performance job 3n-skx`_,
+    - `FD.io test executor vpp performance job 2n-skx`_
+
+    with RF result files csit-vpp-perf-|srelease|-\*.zip
+    `archived here <../../_static/archive/>`_.
+
+3n-hsw
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 1t1c NDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-1t1c-nics-ndr.txt>`_
+  - `ASCII 2t2c NDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-2t2c-nics-ndr.txt>`_
+  - `CSV 1t1c NDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-1t1c-nics-ndr.csv>`_
+  - `CSV 2t2c NDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-2t2c-nics-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 1t1c PDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-1t1c-nics-pdr.txt>`_
+  - `ASCII 2t2c PDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-2t2c-nics-pdr.txt>`_
+  - `CSV 1t1c PDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-1t1c-nics-pdr.csv>`_
+  - `CSV 2t2c PDR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-2t2c-nics-pdr.csv>`_
+
+MRR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 1t1c MRR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-1t1c-nics-mrr.txt>`_
+  - `ASCII 2t2c MRR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-2t2c-nics-mrr.txt>`_
+  - `ASCII 4t4c MRR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-4t4c-nics-mrr.txt>`_
+  - `CSV 1t1c MRR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-1t1c-nics-mrr.csv>`_
+  - `CSV 2t2c MRR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-2t2c-nics-mrr.csv>`_
+  - `CSV 4t4c MRR Intel-x520 and Intel-x710 comparison <../../_static/vpp/performance-changes-3n-hsw-4t4c-nics-mrr.csv>`_
+
+3n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 2t1c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-2t1c-nics-ndr.txt>`_
+  - `ASCII 4t2c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-4t2c-nics-ndr.txt>`_
+  - `CSV 2t1c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-2t1c-nics-ndr.csv>`_
+  - `CSV 4t2c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-4t2c-nics-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 2t1c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-2t1c-nics-pdr.txt>`_
+  - `ASCII 4t2c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-4t2c-nics-pdr.txt>`_
+  - `CSV 2t1c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-2t1c-nics-pdr.csv>`_
+  - `CSV 4t2c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-4t2c-nics-pdr.csv>`_
+
+MRR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 2t1c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-2t1c-nics-mrr.txt>`_
+  - `ASCII 4t2c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-4t2c-nics-mrr.txt>`_
+  - `ASCII 8t4c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-8t4c-nics-mrr.txt>`_
+  - `CSV 2t1c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-2t1c-nics-mrr.csv>`_
+  - `CSV 4t2c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-4t2c-nics-mrr.csv>`_
+  - `CSV 8t4c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-3n-skx-8t4c-nics-mrr.csv>`_
+
+2n-skx
+~~~~~~
+
+NDR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 2t1c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-2t1c-nics-ndr.txt>`_
+  - `ASCII 4t2c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-4t2c-nics-ndr.txt>`_
+  - `CSV 2t1c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-2t1c-nics-ndr.csv>`_
+  - `CSV 4t2c NDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-4t2c-nics-ndr.csv>`_
+
+PDR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 2t1c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-2t1c-nics-pdr.txt>`_
+  - `ASCII 4t2c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-4t2c-nics-pdr.txt>`_
+  - `CSV 2t1c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-2t1c-nics-pdr.csv>`_
+  - `CSV 4t2c PDR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-4t2c-nics-pdr.csv>`_
+
+MRR Comparison
+``````````````
+
+Comparison tables in ASCII and CSV formats:
+
+  - `ASCII 2t1c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-2t1c-nics-mrr.txt>`_
+  - `ASCII 4t2c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-4t2c-nics-mrr.txt>`_
+  - `ASCII 8t4c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-8t4c-nics-mrr.txt>`_
+  - `CSV 2t1c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-2t1c-nics-mrr.csv>`_
+  - `CSV 4t2c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-4t2c-nics-mrr.csv>`_
+  - `CSV 8t4c MRR Intel-x710 and Intel-xxv710 comparison <../../_static/vpp/performance-changes-2n-skx-8t4c-nics-mrr.csv>`_
diff --git a/resources/tools/presentation/conf.py b/resources/tools/presentation/conf.py
index c513e94b84..a728afc44b 100644
--- a/resources/tools/presentation/conf.py
+++ b/resources/tools/presentation/conf.py
@@ -90,9 +90,9 @@ rst_epilog = """
 .. _FD.io test executor dpdk performance job 3n-hsw: https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-verify-{srelease}-3n-hsw
 .. _FD.io test executor dpdk performance job 3n-skx: https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-verify-{srelease}-3n-skx
 .. _FD.io test executor dpdk performance job 2n-skx: https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-verify-{srelease}-2n-skx
-.. _FD.io test executor vpp performance job 3n-hsw: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-verify-1807-3n-hsw
-.. _FD.io test executor vpp performance job 3n-skx: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-verify-1807-3n-skx
-.. _FD.io test executor vpp performance job 2n-skx: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-verify-1807-2n-skx
+.. _FD.io test executor vpp performance job 3n-hsw: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-verify-{srelease}-3n-hsw
+.. _FD.io test executor vpp performance job 3n-skx: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-verify-{srelease}-3n-skx
+.. _FD.io test executor vpp performance job 2n-skx: https://jenkins.fd.io/view/csit/job/csit-vpp-perf-verify-{srelease}-2n-skx
 .. _FD.io test executor ligato performance jobs: https://jenkins.fd.io/job/csit-ligato-perf-{srelease}-all
 .. _FD.io test executor vpp functional jobs using Ubuntu: https://jenkins.fd.io/view/csit/job/csit-vpp-functional-{srelease}-ubuntu1604-virl
 .. _FD.io test executor vpp functional jobs using CentOs: https://jenkins.fd.io/view/csit/job/csit-vpp-functional-{srelease}-centos7-virl
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index f6537af93e..41dfb9429d 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -245,9 +245,10 @@ def table_performance_comparison(table, input_data):
                 if "across topologies" in table["title"].lower():
                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
                 if tbl_dict.get(tst_name_mod, None) is None:
-                    name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
-                                            "-".join(tst_data["name"].
-                                                     split("-")[:-1]))
+                    groups = re.search(REGEX_NIC, tst_data["parent"])
+                    nic = groups.group(0) if groups else ""
+                    name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
+                                                          split("-")[:-1]))
                     if "across testbeds" in table["title"].lower() or \
                             "across topologies" in table["title"].lower():
                         name = name.\
@@ -412,6 +413,116 @@ def table_performance_comparison(table, input_data):
     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 
 
+def table_nics_comparison(table, input_data):
+    """Generate the table(s) with algorithm: table_nics_comparison
+    specified in the specification file.
+
+    :param table: Table to generate.
+    :param input_data: Data to process.
+    :type table: pandas.Series
+    :type input_data: InputData
+    """
+
+    logging.info("  Generating the table {0} ...".
+                 format(table.get("title", "")))
+
+    # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
+    data = input_data.filter_data(table, continue_on_error=True)
+
+    # Prepare the header of the tables
+    try:
+        header = ["Test case", ]
+
+        if table["include-tests"] == "MRR":
+            hdr_param = "Receive Rate"
+        else:
+            hdr_param = "Throughput"
+
+        header.extend(
+            ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
+             "{0} Stdev [Mpps]".format(table["reference"]["title"]),
+             "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
+             "{0} Stdev [Mpps]".format(table["compare"]["title"]),
+             "Delta [%]"])
+        header_str = ",".join(header) + "\n"
+    except (AttributeError, KeyError) as err:
+        logging.error("The model is invalid, missing parameter: {0}".
+                      format(err))
+        return
+
+    # Prepare data to the table:
+    tbl_dict = dict()
+    for job, builds in table["data"].items():
+        for build in builds:
+            for tst_name, tst_data in data[job][str(build)].iteritems():
+                tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
+                    replace("-ndrpdr", "").replace("-pdrdisc", "").\
+                    replace("-ndrdisc", "").replace("-pdr", "").\
+                    replace("-ndr", "").\
+                    replace("1t1c", "1c").replace("2t1c", "1c").\
+                    replace("2t2c", "2c").replace("4t2c", "2c").\
+                    replace("4t4c", "4c").replace("8t4c", "4c")
+                tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+                if tbl_dict.get(tst_name_mod, None) is None:
+                    name = "-".join(tst_data["name"].split("-")[:-1])
+                    tbl_dict[tst_name_mod] = {"name": name,
+                                              "ref-data": list(),
+                                              "cmp-data": list()}
+                try:
+                    if table["include-tests"] == "MRR":
+                        result = tst_data["result"]["receive-rate"].avg
+                    elif table["include-tests"] == "PDR":
+                        result = tst_data["throughput"]["PDR"]["LOWER"]
+                    elif table["include-tests"] == "NDR":
+                        result = tst_data["throughput"]["NDR"]["LOWER"]
+                    else:
+                        result = None
+
+                    if result:
+                        if table["reference"]["nic"] in tst_data["tags"]:
+                            tbl_dict[tst_name_mod]["ref-data"].append(result)
+                        elif table["compare"]["nic"] in tst_data["tags"]:
+                            tbl_dict[tst_name_mod]["cmp-data"].append(result)
+                except (TypeError, KeyError) as err:
+                    logging.warning("No data for {0}".format(tst_name))
+                    logging.warning(repr(err))
+                    # No data in output.xml for this test
+
+    tbl_lst = list()
+    for tst_name in tbl_dict.keys():
+        item = [tbl_dict[tst_name]["name"], ]
+        data_t = tbl_dict[tst_name]["ref-data"]
+        if data_t:
+            item.append(round(mean(data_t) / 1000000, 2))
+            item.append(round(stdev(data_t) / 1000000, 2))
+        else:
+            item.extend([None, None])
+        data_t = tbl_dict[tst_name]["cmp-data"]
+        if data_t:
+            item.append(round(mean(data_t) / 1000000, 2))
+            item.append(round(stdev(data_t) / 1000000, 2))
+        else:
+            item.extend([None, None])
+        if item[-4] is not None and item[-2] is not None and item[-4] != 0:
+            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
+        if len(item) == len(header):
+            tbl_lst.append(item)
+
+    # Sort the table according to the relative change
+    tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+
+    # Generate csv tables:
+    csv_file = "{0}.csv".format(table["output-file"])
+    with open(csv_file, "w") as file_handler:
+        file_handler.write(header_str)
+        for test in tbl_lst:
+            file_handler.write(",".join([str(item) for item in test]) + "\n")
+
+    convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+
+
 def table_performance_trending_dashboard(table, input_data):
     """Generate the table(s) with algorithm:
     table_performance_trending_dashboard
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 119704a5f8..bd16e165c5 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -170,6 +170,19 @@
       - 41  # NDRPDR sel
       - 42  # NDRPDR sel
 
+    vpp-nic-comparison-3n-hsw:
+      csit-vpp-perf-verify-1810-3n-hsw:
+      - 8   # NDRPDR sel
+      - 12  # NDRPDR sel
+      - 14  # NDRPDR sel
+      - 18  # NDRPDR sel
+      - 23  # NDRPDR sel
+      - 25  # NDRPDR sel
+      - 38  # NDRPDR sel
+      - 39  # NDRPDR sel
+      - 41  # NDRPDR sel
+      - 42  # NDRPDR sel
+
     vpp-performance-changes-3n-skx:
       csit-vpp-perf-verify-1807-3n-skx:
       - 33  # NDRPDR sel
@@ -194,6 +207,19 @@
       - 22  # NDRPDR sel
       - 25  # NDRPDR sel
 
+    vpp-nic-comparison-3n-skx:
+      csit-vpp-perf-verify-1810-3n-skx:
+      - 10  # NDRPDR sel
+      - 14  # NDRPDR sel
+      - 15  # NDRPDR sel
+      - 16  # NDRPDR sel
+      - 17  # NDRPDR sel
+      - 19  # NDRPDR sel
+      - 20  # NDRPDR sel
+      - 21  # NDRPDR sel
+      - 22  # NDRPDR sel
+      - 25  # NDRPDR sel
+
     vpp-performance-changes-2n-skx:
       csit-vpp-perf-verify-1807-2n-skx:
       - 26  # NDRPDR sel
@@ -218,6 +244,19 @@
       - 24  # NDRPDR sel
       - 27  # NDRPDR sel
 
+    vpp-nic-comparison-2n-skx:
+      csit-vpp-perf-verify-1810-2n-skx:
+      - 9   # NDRPDR sel
+      - 13  # NDRPDR sel
+      - 14  # NDRPDR sel
+      - 19  # NDRPDR sel
+      - 20  # NDRPDR sel
+      - 21  # NDRPDR sel
+      - 22  # NDRPDR sel
+      - 23  # NDRPDR sel
+      - 24  # NDRPDR sel
+      - 27  # NDRPDR sel
+
     vpp-compare-testbeds:
       csit-vpp-perf-verify-1810-3n-hsw:
       - 8   # NDRPDR sel
@@ -313,6 +352,10 @@
       csit-vpp-perf-verify-1810-3n-hsw:
       - 9   # MRR sel
 
+    vpp-nic-comparison-3n-hsw-mrr:
+      csit-vpp-perf-verify-1810-3n-hsw:
+      - 9   # MRR sel
+
     vpp-performance-changes-3n-skx-mrr:
       csit-vpp-perf-verify-1807-3n-skx:
       - 28  # MRR sel
@@ -331,6 +374,13 @@
       - 12  # MRR sel
       - 13  # MRR sel
 
+    vpp-nic-comparison-3n-skx-mrr:
+      csit-vpp-perf-verify-1810-3n-skx:
+      - 9   # MRR sel
+      - 11  # MRR sel
+      - 12  # MRR sel
+      - 13  # MRR sel
+
     vpp-performance-changes-2n-skx-mrr:
       csit-vpp-perf-verify-1807-2n-skx:
       - 31  # MRR sel
@@ -349,6 +399,13 @@
       - 11  # MRR sel
       - 12  # MRR sel
 
+    vpp-nic-comparison-2n-skx-mrr:
+      csit-vpp-perf-verify-1810-2n-skx:
+      - 8   # MRR sel
+      - 10  # MRR sel
+      - 11  # MRR sel
+      - 12  # MRR sel
+
     vpp-perf-results-3n-hsw:
       csit-vpp-perf-verify-1810-3n-hsw:
       - 20  # NDRPDR full
@@ -1287,10 +1344,452 @@
 ###                               T A B L E S                                ###
 ################################################################################
 
+
+# VPP Compare NICs 3n-hsw 1t1c ndr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 1t1c ndr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-nics-ndr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'1T1C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-hsw 2t2c ndr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 2t2c ndr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-nics-ndr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T2C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-hsw 1t1c pdr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 1t1c pdr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-nics-pdr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'1T1C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-hsw 2t2c pdr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 2t2c pdr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-nics-pdr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T2C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-hsw 1t1c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 1t1c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-1t1c-nics-mrr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'1T1C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 3n-hsw 2t2c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 2t2c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-2t2c-nics-mrr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T2C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 3n-hsw 4t4c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-hsw 4t4c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-hsw-4t4c-nics-mrr"
+  reference:
+    title: "Intel-x520"
+    nic: "NIC_Intel-X520-DA2"
+  compare:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  data: "vpp-nic-comparison-3n-hsw-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T4C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 3n-skx 2t1c ndr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 2t1c ndr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-nics-ndr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-skx 4t2c ndr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 4t2c ndr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-nics-ndr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-skx 2t1c pdr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 2t1c pdr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-nics-pdr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+ # VPP Compare NICs 3n-skx 4t2c pdr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 4t2c pdr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-nics-pdr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 3n-skx 2t1c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 2t1c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-nics-mrr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 3n-skx 4t2c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 4t2c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-nics-mrr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+ # VPP Compare NICs 3n-skx 8t4c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 3n-skx 8t4c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-8t4c-nics-mrr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-3n-skx-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'8T4C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 2n-skx 2t1c ndr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 2t1c ndr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-nics-ndr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 2n-skx 4t2c ndr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 4t2c ndr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-nics-ndr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+ # VPP Compare NICs 2n-skx 2t1c pdr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 2t1c pdr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-nics-pdr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 2n-skx 4t2c pdr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 4t2c pdr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-nics-pdr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'NDRPDR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "throughput"
+  - "tags"
+
+# VPP Compare NICs 2n-skx 2t1c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 2t1c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-nics-mrr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'2T1C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 2n-skx 4t2c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 4t2c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-nics-mrr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'4T2C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
+# VPP Compare NICs 2n-skx 8t4c mrr
+-
+  type: "table"
+  title: "VPP Compare NICs 2n-skx 8t4c mrr"
+  algorithm: "table_nics_comparison"
+  output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-8t4c-nics-mrr"
+  reference:
+    title: "Intel-x710"
+    nic: "NIC_Intel-X710"
+  compare:
+    title: "Intel-xxv710"
+    nic: "NIC_Intel-XXV710"
+  data: "vpp-nic-comparison-2n-skx-mrr"
+  include-tests: "MRR" # "PDR" | "NDR" | "MRR"
+  filter: "'8T4C' and 'MRR'" # "('NDRPDR' or 'NDRPDRDISC')" | "'MRR'"
+  parameters:
+  - "name"
+  - "parent"
+  - "result"
+  - "tags"
+
 ################################################################################
 # VPP
 
-# VPP Performance Changes 3n-hsw 1t1c pdr 
+# VPP Performance Changes 3n-hsw 1t1c pdr
 -
   type: "table"
   title: "VPP Performance Changes 3n-hsw 1t1c pdr"
@@ -1530,7 +2029,7 @@
   - "parent"
   - "throughput"
 
-# VPP Performance Changes 3n-skx 2t1c pdr 
+# VPP Performance Changes 3n-skx 2t1c pdr
 -
   type: "table"
   title: "VPP Performance Changes 3n-skx 2t1c pdr"
@@ -1698,7 +2197,7 @@
   - "parent"
   - "throughput"
 
-# VPP Performance Changes 2n-skx 2t1c pdr 
+# VPP Performance Changes 2n-skx 2t1c pdr
 -
   type: "table"
   title: "VPP Performance Changes 2n-skx 2t1c pdr"