Trending: NDRPDR weekly comparison

Change-Id: I6c5166e1fabfe7471af92a1423c0c89ca6b040a4
Signed-off-by: Tibor Frank <tifrank@cisco.com>

4 files changed, 364 insertions, 25 deletions

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 14130e5059..167d58d0fd 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -33,7 +33,7 @@ from numpy import nan, isnan
 from yaml import load, FullLoader, YAMLError
 
 from pal_utils import mean, stdev, classify_anomalies, \
-    convert_csv_to_pretty_txt, relative_change_stdev
+    convert_csv_to_pretty_txt, relative_change_stdev, relative_change
 
 
 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
@@ -60,12 +60,15 @@ def generate_tables(spec, data):
         u"table_failed_tests": table_failed_tests,
         u"table_failed_tests_html": table_failed_tests_html,
         u"table_oper_data_html": table_oper_data_html,
-        u"table_comparison": table_comparison
+        u"table_comparison": table_comparison,
+        u"table_weekly_comparison": table_weekly_comparison
     }
 
     logging.info(u"Generating the tables ...")
     for table in spec.tables:
         try:
+            if table[u"algorithm"] == u"table_weekly_comparison":
+                table[u"testbeds"] = spec.environment.get(u"testbeds", None)
             generator[table[u"algorithm"]](table, data)
         except NameError as err:
             logging.error(
@@ -490,7 +493,8 @@ def _tpc_sort_table(table):
 
 
 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
-                             footnote=u"", sort_data=True, title=u""):
+                             footnote=u"", sort_data=True, title=u"",
+                             generate_rst=True):
     """Generate html table from input data with simple sorting possibility.
 
     :param header: Table header.
@@ -504,6 +508,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
     :param footnote: The footnote to display below the table (and legend).
     :param sort_data: If True the data sorting is enabled.
     :param title: The table (and file) title.
+    :param generate_rst: If True, wrapping rst file is generated.
     :type header: list
     :type data: list of lists
     :type out_file_name: str
@@ -511,6 +516,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
     :type footnote: str
     :type sort_data: bool
     :type title: str
+    :type generate_rst: bool
     """
 
     try:
@@ -528,7 +534,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
             [u"left", u"left", u"right"],
             [u"left", u"left", u"left", u"right"]
         ),
-        u"width": ([28, 9], [4, 24, 10], [4, 4, 32, 10])
+        u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
     }
 
     df_data = pd.DataFrame(data, columns=header)
@@ -599,8 +605,8 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
                     direction=u"down",
                     x=0.0,
                     xanchor=u"left",
-                    y=1.045,
-                    yanchor=u"top",
+                    y=1.002,
+                    yanchor=u"bottom",
                     active=len(menu_items) - 1,
                     buttons=list(buttons)
                 )
@@ -630,6 +636,9 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
         filename=f"{out_file_name}_in.html"
     )
 
+    if not generate_rst:
+        return
+
     file_name = out_file_name.split(u"/")[-1]
     if u"vpp" in out_file_name:
         path = u"_tmp/src/vpp_performance_tests/comparisons/"
@@ -2698,3 +2707,170 @@ def table_comparison(table, input_data):
         sort_data=False,
         title=table.get(u"title", u"")
     )
+
+
+def table_weekly_comparison(table, in_data):
+    """Generate the table(s) with algorithm: table_weekly_comparison
+    specified in the specification file.
+
+    :param table: Table to generate.
+    :param in_data: Data to process.
+    :type table: pandas.Series
+    :type in_data: InputData
+    """
+    logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
+
+    # Transform the data
+    logging.info(
+        f"    Creating the data set for the {table.get(u'type', u'')} "
+        f"{table.get(u'title', u'')}."
+    )
+
+    incl_tests = table.get(u"include-tests", None)
+    if incl_tests not in (u"NDR", u"PDR"):
+        logging.error(f"Wrong tests to include specified ({incl_tests}).")
+        return
+
+    nr_cols = table.get(u"nr-of-data-columns", None)
+    if not nr_cols or nr_cols < 2:
+        logging.error(
+            f"No columns specified for {table.get(u'title', u'')}. Skipping."
+        )
+        return
+
+    data = in_data.filter_data(
+        table,
+        params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+        continue_on_error=True
+    )
+
+    header = [
+        [u"Version"],
+        [u"Date", ],
+        [u"Build", ],
+        [u"Testbed", ]
+    ]
+    tbl_dict = dict()
+    idx = 0
+    tb_tbl = table.get(u"testbeds", None)
+    for job_name, job_data in data.items():
+        for build_nr, build in job_data.items():
+            if idx >= nr_cols:
+                break
+            if build.empty:
+                continue
+
+            tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
+            if tb_ip and tb_tbl:
+                testbed = tb_tbl.get(tb_ip, u"")
+            else:
+                testbed = u""
+            header[2].insert(1, build_nr)
+            header[3].insert(1, testbed)
+            header[1].insert(
+                1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
+            )
+            header[0].insert(
+                1, in_data.metadata(job_name, build_nr).get(u"version", u"")
+            )
+
+            for tst_name, tst_data in build.items():
+                tst_name_mod = \
+                    _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
+                if not tbl_dict.get(tst_name_mod, None):
+                    tbl_dict[tst_name_mod] = dict(
+                        name=tst_data[u'name'].rsplit(u'-', 1)[0],
+                    )
+                try:
+                    tbl_dict[tst_name_mod][-idx - 1] = \
+                        tst_data[u"throughput"][incl_tests][u"LOWER"]
+                except (TypeError, IndexError, KeyError, ValueError):
+                    pass
+            idx += 1
+
+    if idx < nr_cols:
+        logging.error(u"Not enough data to build the table! Skipping")
+        return
+
+    cmp_dict = dict()
+    for idx, cmp in enumerate(table.get(u"comparisons", list())):
+        idx_ref = cmp.get(u"reference", None)
+        idx_cmp = cmp.get(u"compare", None)
+        if idx_ref is None or idx_cmp is None:
+            continue
+        header[0].append(f"Diff{idx + 1}")
+        header[1].append(header[0][idx_ref - idx - 1])
+        header[2].append(u"vs")
+        header[3].append(header[0][idx_cmp - idx - 1])
+        for tst_name, tst_data in tbl_dict.items():
+            if not cmp_dict.get(tst_name, None):
+                cmp_dict[tst_name] = list()
+            ref_data = tst_data.get(idx_ref, None)
+            cmp_data = tst_data.get(idx_cmp, None)
+            if ref_data is None or cmp_data is None:
+                cmp_dict[tst_name].append(float('nan'))
+            else:
+                cmp_dict[tst_name].append(
+                    relative_change(ref_data, cmp_data)
+                )
+
+    tbl_lst = list()
+    for tst_name, tst_data in tbl_dict.items():
+        itm_lst = [tst_data[u"name"], ]
+        for idx in range(nr_cols):
+            item = tst_data.get(-idx - 1, None)
+            if item is None:
+                itm_lst.insert(1, None)
+            else:
+                itm_lst.insert(1, round(item / 1e6, 1))
+        itm_lst.extend(
+            [
+                None if itm is None else round(itm, 1)
+                for itm in cmp_dict[tst_name]
+            ]
+        )
+        tbl_lst.append(itm_lst)
+
+    tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
+    tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+
+    # Generate csv table:
+    csv_file = f"{table[u'output-file']}.csv"
+    with open(csv_file, u"wt", encoding='utf-8') as file_handler:
+        for hdr in header:
+            file_handler.write(u",".join(hdr) + u"\n")
+        for test in tbl_lst:
+            file_handler.write(u",".join(
+                [
+                    str(item).replace(u"None", u"-").replace(u"nan", u"-").
+                    replace(u"null", u"-") for item in test
+                ]
+            ) + u"\n")
+
+    txt_file = f"{table[u'output-file']}.txt"
+    convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
+
+    # Reorganize header in txt table
+    txt_table = list()
+    with open(txt_file, u"rt", encoding='utf-8') as file_handler:
+        for line in file_handler:
+            txt_table.append(line)
+    try:
+        txt_table.insert(5, txt_table.pop(2))
+        with open(txt_file, u"wt", encoding='utf-8') as file_handler:
+            file_handler.writelines(txt_table)
+    except IndexError:
+        pass
+
+    # Generate html table:
+    hdr_html = [
+        u"<br>".join(row) for row in zip(*header)
+    ]
+    _tpc_generate_html_table(
+        hdr_html,
+        tbl_lst,
+        table[u'output-file'],
+        sort_data=True,
+        title=table.get(u"title", u""),
+        generate_rst=False
+    )
diff --git a/resources/tools/presentation/pal_utils.py b/resources/tools/presentation/pal_utils.py
index 20bf9a65c4..1b9131a30e 100644
--- a/resources/tools/presentation/pal_utils.py
+++ b/resources/tools/presentation/pal_utils.py
@@ -320,6 +320,8 @@ def convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u","):
                 )
     txt_table.align = u"r"
     txt_table.align[u"Test Case"] = u"l"
+    txt_table.align[u"Build"] = u"l"
+    txt_table.align[u"Version"] = u"l"
 
     if not txt_table:
         return
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 1e199511bd..22b31e2556 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -198,112 +198,124 @@
       csit-vpp-perf-mrr-daily-master:
         start: 1086
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
       csit-dpdk-perf-mrr-weekly-master:
         start: 107
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
         start: 1086
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
         start: 107
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
         start: 834
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         start: 72
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
         start: 834
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         start: 72
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
         start: 858
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         start: 78
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
         start: 858
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         start: 78
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-nfv-2n-skx:
       csit-vpp-perf-mrr-weekly-master-2n-skx:
         start: 50
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-2n-clx:
       csit-vpp-perf-mrr-daily-master-2n-clx:
         start: 236
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-dpdk-2n-clx:
       csit-dpdk-perf-mrr-weekly-master-2n-clx:
         start: 23
         end: "lastCompletedBuild"
-        max-builds: 15  # Max nr of builds to use
 
     # 3n-tsh
     plot-performance-trending-all-3n-tsh:
       csit-vpp-perf-mrr-daily-master-3n-tsh:
         start: 144
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-vpp-3n-tsh:
       csit-vpp-perf-mrr-daily-master-3n-tsh:
         start: 144
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-vpp-3n-dnv:
       csit-vpp-perf-mrr-daily-master-3n-dnv:
         start: 329
         end: "lastCompletedBuild"
-        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-vpp-2n-dnv:
       csit-vpp-perf-mrr-daily-master-2n-dnv:
         start: 335
         end: "lastCompletedBuild"
 
+    table-ndrpdr-comparison-2n-skx:
+      csit-vpp-perf-ndrpdr-weekly-master-2n-skx:
+        start: 84
+        end: "lastCompletedBuild"
+        max-builds: 4
+        reverse: True  # Must be reversed
+
+    table-ndrpdr-comparison-3n-skx:
+      csit-vpp-perf-ndrpdr-weekly-master-3n-skx:
+        start: 84
+        end: "lastCompletedBuild"
+        max-builds: 4
+        reverse: True  # Must be reversed
+        skip:
+          - 87
+
+    table-ndrpdr-comparison-2n-clx:
+      csit-vpp-perf-ndrpdr-weekly-master-2n-clx:
+        start: 30
+        end: "lastCompletedBuild"
+        max-builds: 4
+        reverse: True  # Must be reversed
+
+    table-ndrpdr-comparison-3n-hsw:
+      csit-vpp-perf-ndrpdr-weekly-master-3n-hsw:
+        start: 84
+        end: "lastCompletedBuild"
+        max-builds: 4
+        reverse: True  # Must be reversed
+
   plot-layouts:
 
     plot-cpta:
@@ -398,6 +410,32 @@
 
   builds:
 
+    # 2n-skx ndrpdr
+    csit-vpp-perf-ndrpdr-weekly-master-2n-skx:
+      start: 84
+      end: "lastCompletedBuild"
+      max-builds: 4
+
+    # 3n-skx ndrpdr
+    csit-vpp-perf-ndrpdr-weekly-master-3n-skx:
+      start: 84
+      end: "lastCompletedBuild"
+      max-builds: 4
+      skip:
+        - 87
+
+    # 2n-clx ndrpdr
+    csit-vpp-perf-ndrpdr-weekly-master-2n-clx:
+      start: 30
+      end: "lastCompletedBuild"
+      max-builds: 4
+
+    # 3n-hsw ndrpdr
+    csit-vpp-perf-ndrpdr-weekly-master-3n-hsw:
+      start: 84
+      end: "lastCompletedBuild"
+      max-builds: 4
+
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
       start: 1086
@@ -548,6 +586,126 @@
 
 ################################################################################
 
+# VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c ndr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c ndr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-skx-2t1c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-2n-skx"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c pdr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 2n-skx-xxv710 2t1c pdr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-skx-2t1c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-2n-skx"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c ndr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c ndr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-skx-2t1c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-3n-skx"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c pdr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 3n-skx-xxv710 2t1c pdr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-skx-2t1c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-3n-skx"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c ndr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c ndr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-clx-2t1c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-2n-clx"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c pdr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 2n-clx-xxv710 2t1c pdr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-2n-clx-2t1c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-2n-clx"
+  filter: "'2T1C' and 'NDRPDR' and 'NIC_Intel-XXV710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 3n-hsw-xl710 2t1c ndr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 3n-hsw-xl710 1t1c ndr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-hsw-1t1c-ndr"
+  include-tests: "NDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-3n-hsw"
+  filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
+# VPP NDRPDR weekly comparison 3n-hsw-xl710 2t1c pdr
+- type: "table"
+  title: "VPP NDRPDR weekly comparison 3n-hsw-xl710 1t1c pdr"
+  algorithm: "table_weekly_comparison"
+  output-file: "{DIR[STATIC,VPP]}/ndrpdr-weekly-comp-3n-hsw-1t1c-pdr"
+  include-tests: "PDR" # "PDR" | "NDR" | "MRR"
+  data: "table-ndrpdr-comparison-3n-hsw"
+  filter: "'1T1C' and 'NDRPDR' and 'NIC_Intel-XL710' and not 'NF_DENSITY'"
+  nr-of-data-columns: 3
+  comparisons:
+    - reference: -3
+      compare: -1
+    - reference: -2
+      compare: -1
+
 # Compressed failed tests (last build)
 -
   type: "table"
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index 548bbff532..302ce037ab 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -529,6 +529,7 @@ class Specification:
                 if isinstance(builds, dict):
                     build_end = builds.get(u"end", None)
                     max_builds = builds.get(u"max-builds", None)
+                    reverse = builds.get(u"reverse", False)
                     try:
                         build_end = int(build_end)
                     except ValueError:
@@ -536,7 +537,9 @@ class Specification:
                         build_end = self._get_build_number(job, build_end)
                     builds = [x for x in range(builds[u"start"], build_end + 1)]
                     if max_builds and max_builds < len(builds):
-                        builds = builds[:max_builds]
+                        builds = builds[-max_builds:]
+                    if reverse:
+                        builds.reverse()
                     self.configuration[u"data-sets"][set_name][job] = builds
                 elif isinstance(builds, list):
                     for idx, item in enumerate(builds):