Report: Tables

Change-Id: I5084163758c5f15f482c6c8e08ee70c4a4f50431
Signed-off-by: Tibor Frank <tifrank@cisco.com>
(cherry picked from commit af4d3ba04cddbfb198af47b2d1d74c0655e35a08)

2 files changed, 304 insertions, 37 deletions

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 5c4452d5a5..f04495c4a1 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -415,6 +415,241 @@ def table_performance_comparison(table, input_data):
     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
 
 
+def table_performance_comparison_nic(table, input_data):
+    """Generate the table(s) with algorithm: table_performance_comparison
+    specified in the specification file.
+
+    :param table: Table to generate.
+    :param input_data: Data to process.
+    :type table: pandas.Series
+    :type input_data: InputData
+    """
+
+    logging.info("  Generating the table {0} ...".
+                 format(table.get("title", "")))
+
+    # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
+    data = input_data.filter_data(table, continue_on_error=True)
+
+    # Prepare the header of the tables
+    try:
+        header = ["Test case", ]
+
+        if table["include-tests"] == "MRR":
+            hdr_param = "Receive Rate"
+        else:
+            hdr_param = "Throughput"
+
+        history = table.get("history", None)
+        if history:
+            for item in history:
+                header.extend(
+                    ["{0} {1} [Mpps]".format(item["title"], hdr_param),
+                     "{0} Stdev [Mpps]".format(item["title"])])
+        header.extend(
+            ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
+             "{0} Stdev [Mpps]".format(table["reference"]["title"]),
+             "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
+             "{0} Stdev [Mpps]".format(table["compare"]["title"]),
+             "Delta [%]"])
+        header_str = ",".join(header) + "\n"
+    except (AttributeError, KeyError) as err:
+        logging.error("The model is invalid, missing parameter: {0}".
+                      format(err))
+        return
+
+    # Prepare data to the table:
+    tbl_dict = dict()
+    for job, builds in table["reference"]["data"].items():
+        for build in builds:
+            for tst_name, tst_data in data[job][str(build)].iteritems():
+                if table["reference"]["nic"] not in tst_data["tags"]:
+                    continue
+                tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
+                    replace("-ndrpdr", "").replace("-pdrdisc", "").\
+                    replace("-ndrdisc", "").replace("-pdr", "").\
+                    replace("-ndr", "").\
+                    replace("1t1c", "1c").replace("2t1c", "1c").\
+                    replace("2t2c", "2c").replace("4t2c", "2c").\
+                    replace("4t4c", "4c").replace("8t4c", "4c")
+                tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+                if "across topologies" in table["title"].lower():
+                    tst_name_mod = tst_name_mod.replace("2n1l-", "")
+                if tbl_dict.get(tst_name_mod, None) is None:
+                    name = "{0}".format("-".join(tst_data["name"].
+                                                 split("-")[:-1]))
+                    if "across testbeds" in table["title"].lower() or \
+                            "across topologies" in table["title"].lower():
+                        name = name.\
+                            replace("1t1c", "1c").replace("2t1c", "1c").\
+                            replace("2t2c", "2c").replace("4t2c", "2c").\
+                            replace("4t4c", "4c").replace("8t4c", "4c")
+                    tbl_dict[tst_name_mod] = {"name": name,
+                                              "ref-data": list(),
+                                              "cmp-data": list()}
+                try:
+                    # TODO: Re-work when NDRPDRDISC tests are not used
+                    if table["include-tests"] == "MRR":
+                        tbl_dict[tst_name_mod]["ref-data"]. \
+                            append(tst_data["result"]["receive-rate"].avg)
+                    elif table["include-tests"] == "PDR":
+                        if tst_data["type"] == "PDR":
+                            tbl_dict[tst_name_mod]["ref-data"]. \
+                                append(tst_data["throughput"]["value"])
+                        elif tst_data["type"] == "NDRPDR":
+                            tbl_dict[tst_name_mod]["ref-data"].append(
+                                tst_data["throughput"]["PDR"]["LOWER"])
+                    elif table["include-tests"] == "NDR":
+                        if tst_data["type"] == "NDR":
+                            tbl_dict[tst_name_mod]["ref-data"]. \
+                                append(tst_data["throughput"]["value"])
+                        elif tst_data["type"] == "NDRPDR":
+                            tbl_dict[tst_name_mod]["ref-data"].append(
+                                tst_data["throughput"]["NDR"]["LOWER"])
+                    else:
+                        continue
+                except TypeError:
+                    pass  # No data in output.xml for this test
+
+    for job, builds in table["compare"]["data"].items():
+        for build in builds:
+            for tst_name, tst_data in data[job][str(build)].iteritems():
+                if table["compare"]["nic"] not in tst_data["tags"]:
+                    continue
+                tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
+                    replace("-ndrpdr", "").replace("-pdrdisc", ""). \
+                    replace("-ndrdisc", "").replace("-pdr", ""). \
+                    replace("-ndr", "").\
+                    replace("1t1c", "1c").replace("2t1c", "1c").\
+                    replace("2t2c", "2c").replace("4t2c", "2c").\
+                    replace("4t4c", "4c").replace("8t4c", "4c")
+                tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+                if "across topologies" in table["title"].lower():
+                    tst_name_mod = tst_name_mod.replace("2n1l-", "")
+                try:
+                    # TODO: Re-work when NDRPDRDISC tests are not used
+                    if table["include-tests"] == "MRR":
+                        tbl_dict[tst_name_mod]["cmp-data"]. \
+                            append(tst_data["result"]["receive-rate"].avg)
+                    elif table["include-tests"] == "PDR":
+                        if tst_data["type"] == "PDR":
+                            tbl_dict[tst_name_mod]["cmp-data"]. \
+                                append(tst_data["throughput"]["value"])
+                        elif tst_data["type"] == "NDRPDR":
+                            tbl_dict[tst_name_mod]["cmp-data"].append(
+                                tst_data["throughput"]["PDR"]["LOWER"])
+                    elif table["include-tests"] == "NDR":
+                        if tst_data["type"] == "NDR":
+                            tbl_dict[tst_name_mod]["cmp-data"]. \
+                                append(tst_data["throughput"]["value"])
+                        elif tst_data["type"] == "NDRPDR":
+                            tbl_dict[tst_name_mod]["cmp-data"].append(
+                                tst_data["throughput"]["NDR"]["LOWER"])
+                    else:
+                        continue
+                except KeyError:
+                    pass
+                except TypeError:
+                    tbl_dict.pop(tst_name_mod, None)
+    if history:
+        for item in history:
+            for job, builds in item["data"].items():
+                for build in builds:
+                    for tst_name, tst_data in data[job][str(build)].iteritems():
+                        if item["nic"] not in tst_data["tags"]:
+                            continue
+                        tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
+                            replace("-ndrpdr", "").replace("-pdrdisc", ""). \
+                            replace("-ndrdisc", "").replace("-pdr", ""). \
+                            replace("-ndr", "").\
+                            replace("1t1c", "1c").replace("2t1c", "1c").\
+                            replace("2t2c", "2c").replace("4t2c", "2c").\
+                            replace("4t4c", "4c").replace("8t4c", "4c")
+                        tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+                        if "across topologies" in table["title"].lower():
+                            tst_name_mod = tst_name_mod.replace("2n1l-", "")
+                        if tbl_dict.get(tst_name_mod, None) is None:
+                            continue
+                        if tbl_dict[tst_name_mod].get("history", None) is None:
+                            tbl_dict[tst_name_mod]["history"] = OrderedDict()
+                        if tbl_dict[tst_name_mod]["history"].get(item["title"],
+                                                             None) is None:
+                            tbl_dict[tst_name_mod]["history"][item["title"]] = \
+                                list()
+                        try:
+                            # TODO: Re-work when NDRPDRDISC tests are not used
+                            if table["include-tests"] == "MRR":
+                                tbl_dict[tst_name_mod]["history"][item["title"
+                                ]].append(tst_data["result"]["receive-rate"].
+                                          avg)
+                            elif table["include-tests"] == "PDR":
+                                if tst_data["type"] == "PDR":
+                                    tbl_dict[tst_name_mod]["history"][
+                                        item["title"]].\
+                                        append(tst_data["throughput"]["value"])
+                                elif tst_data["type"] == "NDRPDR":
+                                    tbl_dict[tst_name_mod]["history"][item[
+                                        "title"]].append(tst_data["throughput"][
+                                        "PDR"]["LOWER"])
+                            elif table["include-tests"] == "NDR":
+                                if tst_data["type"] == "NDR":
+                                    tbl_dict[tst_name_mod]["history"][
+                                        item["title"]].\
+                                        append(tst_data["throughput"]["value"])
+                                elif tst_data["type"] == "NDRPDR":
+                                    tbl_dict[tst_name_mod]["history"][item[
+                                        "title"]].append(tst_data["throughput"][
+                                        "NDR"]["LOWER"])
+                            else:
+                                continue
+                        except (TypeError, KeyError):
+                            pass
+
+    tbl_lst = list()
+    for tst_name in tbl_dict.keys():
+        item = [tbl_dict[tst_name]["name"], ]
+        if history:
+            if tbl_dict[tst_name].get("history", None) is not None:
+                for hist_data in tbl_dict[tst_name]["history"].values():
+                    if hist_data:
+                        item.append(round(mean(hist_data) / 1000000, 2))
+                        item.append(round(stdev(hist_data) / 1000000, 2))
+                    else:
+                        item.extend([None, None])
+            else:
+                item.extend([None, None])
+        data_t = tbl_dict[tst_name]["ref-data"]
+        if data_t:
+            item.append(round(mean(data_t) / 1000000, 2))
+            item.append(round(stdev(data_t) / 1000000, 2))
+        else:
+            item.extend([None, None])
+        data_t = tbl_dict[tst_name]["cmp-data"]
+        if data_t:
+            item.append(round(mean(data_t) / 1000000, 2))
+            item.append(round(stdev(data_t) / 1000000, 2))
+        else:
+            item.extend([None, None])
+        if item[-4] is not None and item[-2] is not None and item[-4] != 0:
+            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
+        if len(item) == len(header):
+            tbl_lst.append(item)
+
+    # Sort the table according to the relative change
+    tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+
+    # Generate csv tables:
+    csv_file = "{0}.csv".format(table["output-file"])
+    with open(csv_file, "w") as file_handler:
+        file_handler.write(header_str)
+        for test in tbl_lst:
+            file_handler.write(",".join([str(item) for item in test]) + "\n")
+
+    convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+
+
 def table_nics_comparison(table, input_data):
     """Generate the table(s) with algorithm: table_nics_comparison
     specified in the specification file.
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 4fc722eb66..1e3d60e77f 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -1520,8 +1520,8 @@
   format:
     html:
     - full
-    pdf:
-    - minimal
+#    pdf:
+#    - minimal
 
 ################################################################################
 ###                               T A B L E S                                ###
@@ -2076,17 +2076,20 @@
 # VPP Performance Changes 3n-skx 2t1c pdr
 - type: "table"
   title: "VPP Performance Changes 3n-skx 2t1c pdr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-pdr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-3n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-3n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-3n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-3n-skx"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2094,21 +2097,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 3n-skx 4t2c pdr
 - type: "table"
   title: "VPP Performance Changes 3n-skx 4t2c pdr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-pdr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-3n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-3n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-3n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-3n-skx"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2116,21 +2123,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 3n-skx 2t1c ndr
 - type: "table"
   title: "VPP Performance Changes 3n-skx 2t1c ndr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-2t1c-ndr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-3n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-3n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-3n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-3n-skx"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2138,21 +2149,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 3n-skx 4t2c ndr
 - type: "table"
   title: "VPP Performance Changes 3n-skx 4t2c ndr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-3n-skx-4t2c-ndr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-3n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-3n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-3n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-3n-skx"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2160,21 +2175,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 2n-skx 2t1c pdr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 2t1c pdr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr"
   history:
-    - title: "rls1901"
-      data: "vpp-performance-changes-2n-skx-h1"
+    - title: "rls1901 Intel-X710"
+      data: "vpp-performance-changes-3n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
-    data: "vpp-performance-changes-2n-skx-ref"
+    title: "rls1904 Intel-X710"
+    data: "vpp-performance-changes-3n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
-    data: "vpp-performance-changes-2n-skx-cmp"
+    title: "rls1908 Intel-XXV710"
+    data: "vpp-performance-changes-3n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-2n-skx"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2182,21 +2201,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 2n-skx 4t2c pdr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 4t2c pdr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-pdr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-2n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-2n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-2n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-2n-skx"
   include-tests: "PDR" # "PDR" | "NDR" | "MRR"
   filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2204,21 +2227,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 2n-skx 2t1c ndr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 2t1c ndr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-ndr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-2n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-2n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-2n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-2n-skx"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'2T1C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2226,21 +2253,25 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Performance Changes 2n-skx 4t2c ndr
 - type: "table"
   title: "VPP Performance Changes 2n-skx 4t2c ndr"
-  algorithm: "table_performance_comparison"
+  algorithm: "table_performance_comparison_nic"
   output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-4t2c-ndr"
   history:
-    - title: "rls1901"
+    - title: "rls1901 Intel-X710"
       data: "vpp-performance-changes-2n-skx-h1"
+      nic: "NIC_Intel-X710"
   reference:
-    title: "rls1904"
+    title: "rls1904 Intel-X710"
     data: "vpp-performance-changes-2n-skx-ref"
+    nic: "NIC_Intel-X710"
   compare:
-    title: "rls1908"
+    title: "rls1908 Intel-XXV710"
     data: "vpp-performance-changes-2n-skx-cmp"
+    nic: "NIC_Intel-XXV710"
   data: "vpp-performance-changes-2n-skx"
   include-tests: "NDR" # "PDR" | "NDR" | "MRR"
   filter: "'4T2C' and ('NDRPDR' or 'NDRPDRDISC')"
@@ -2248,6 +2279,7 @@
   - "name"
   - "parent"
   - "throughput"
+  - "tags"
 
 # VPP Comparison Across Testbeds 3n-hsw to 3n-skx 1c ndr
 - type: "table"