CSIT-1412: Create trending graphs for svc density

 - static content
 - plots

Change-Id: If3d77f5415acfa2efe2bc7ca13e426747b027f8e
Signed-off-by: Tibor Frank <tifrank@cisco.com>

3 files changed, 371 insertions, 55 deletions

diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index b42b639fe4..1e7719153f 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -25,6 +25,7 @@ import plotly.exceptions as plerr
 
 from collections import OrderedDict
 from datetime import datetime
+from copy import deepcopy
 
 from utils import archive_input_data, execute_command, \
     classify_anomalies, Worker
@@ -347,6 +348,7 @@ def _generate_all_charts(spec, input_data):
             return
 
         chart_data = dict()
+        chart_tags = dict()
         for job, job_data in data.iteritems():
             if job != job_name:
                 continue
@@ -357,6 +359,7 @@ def _generate_all_charts(spec, input_data):
                     try:
                         chart_data[test_name][int(index)] = \
                             test["result"]["receive-rate"]
+                        chart_tags[test_name] = test.get("tags", None)
                     except (KeyError, TypeError):
                         pass
 
@@ -369,45 +372,126 @@ def _generate_all_charts(spec, input_data):
                     itm = itm.avg
                 tst_lst.append(str(itm))
             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
+
         # Generate traces:
         traces = list()
         index = 0
-        for test_name, test_data in chart_data.items():
-            if not test_data:
-                logs.append(("WARNING", "No data for the test '{0}'".
-                             format(test_name)))
-                continue
-            message = "index: {index}, test: {test}".format(
-                index=index, test=test_name)
-            test_name = test_name.split('.')[-1]
-            try:
-                trace, rslt = _generate_trending_traces(
-                    test_data,
-                    job_name=job_name,
-                    build_info=build_info,
-                    name='-'.join(test_name.split('-')[2:-1]),
-                    color=COLORS[index])
-            except IndexError:
-                message = "Out of colors: {}".format(message)
-                logs.append(("ERROR", message))
-                logging.error(message)
+        groups = graph.get("groups", None)
+        visibility = list()
+
+        if groups:
+            for group in groups:
+                visible = list()
+                for tag in group:
+                    for test_name, test_data in chart_data.items():
+                        if not test_data:
+                            logs.append(("WARNING",
+                                         "No data for the test '{0}'".
+                                         format(test_name)))
+                            continue
+                        if tag in chart_tags[test_name]:
+                            message = "index: {index}, test: {test}".format(
+                                index=index, test=test_name)
+                            test_name = test_name.split('.')[-1]
+                            try:
+                                trace, rslt = _generate_trending_traces(
+                                    test_data,
+                                    job_name=job_name,
+                                    build_info=build_info,
+                                    name='-'.join(test_name.split('-')[2:-1]),
+                                    color=COLORS[index])
+                            except IndexError:
+                                message = "Out of colors: {}".format(message)
+                                logs.append(("ERROR", message))
+                                logging.error(message)
+                                index += 1
+                                continue
+                            traces.extend(trace)
+                            visible.extend([True for _ in range(len(trace))])
+                            res.append(rslt)
+                            index += 1
+                            break
+                visibility.append(visible)
+        else:
+            for test_name, test_data in chart_data.items():
+                if not test_data:
+                    logs.append(("WARNING", "No data for the test '{0}'".
+                                 format(test_name)))
+                    continue
+                message = "index: {index}, test: {test}".format(
+                    index=index, test=test_name)
+                test_name = test_name.split('.')[-1]
+                try:
+                    trace, rslt = _generate_trending_traces(
+                        test_data,
+                        job_name=job_name,
+                        build_info=build_info,
+                        name='-'.join(test_name.split('-')[2:-1]),
+                        color=COLORS[index])
+                except IndexError:
+                    message = "Out of colors: {}".format(message)
+                    logs.append(("ERROR", message))
+                    logging.error(message)
+                    index += 1
+                    continue
+                traces.extend(trace)
+                res.append(rslt)
                 index += 1
-                continue
-            traces.extend(trace)
-            res.append(rslt)
-            index += 1
 
         if traces:
             # Generate the chart:
-            graph["layout"]["title"] = \
-                "<b>{title}</b>".format(title=graph.get("title", ""))
+            try:
+                layout = deepcopy(graph["layout"])
+            except KeyError as err:
+                logging.error("Finished with error: No layout defined")
+                logging.error(repr(err))
+                return
+            if groups:
+                show = list()
+                for i in range(len(visibility)):
+                    visible = list()
+                    for r in range(len(visibility)):
+                        for _ in range(len(visibility[r])):
+                            visible.append(i == r)
+                    show.append(visible)
+
+                buttons = list()
+                buttons.append(dict(
+                    label="All",
+                    method="update",
+                    args=[{"visible": [True for _ in range(len(show[0]))]}, ]
+                ))
+                for i in range(len(groups)):
+                    try:
+                        label = graph["group-names"][i]
+                    except (IndexError, KeyError):
+                        label = "Group {num}".format(num=i + 1)
+                    buttons.append(dict(
+                        label=label,
+                        method="update",
+                        args=[{"visible": show[i]}, ]
+                    ))
+
+                layout['updatemenus'] = list([
+                    dict(
+                        active=0,
+                        type="dropdown",
+                        direction="down",
+                        xanchor="left",
+                        yanchor="bottom",
+                        x=-0.12,
+                        y=1.0,
+                        buttons=buttons
+                    )
+                ])
+
             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
                                             graph["output-file-name"],
                                             spec.cpta["output-file-type"])
 
             logs.append(("INFO", "    Writing the file '{0}' ...".
                          format(name_file)))
-            plpl = plgo.Figure(data=traces, layout=graph["layout"])
+            plpl = plgo.Figure(data=traces, layout=layout)
             try:
                 ploff.plot(plpl, show_link=False, auto_open=False,
                            filename=name_file)
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 29cbc5d8df..ed6f6cb9cb 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -151,60 +151,72 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 470
+        start: 482
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
-        start: 54
+        start: 55
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 470
+        start: 482
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
-        start: 54
+        start: 55
         end: "lastCompletedBuild"
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 220
+        start: 232
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 19
-        end: 30 # "lastCompletedBuild"
+        start: 20
+        end: "lastCompletedBuild"
+        skip:
+          - 31
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 220
+        start: 232
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 19
-        end: 30 # "lastCompletedBuild"
+        start: 20
+        end: "lastCompletedBuild"
+        skip:
+          - 31
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 220
+        start: 232
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 19
+        start: 20
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 220
+        start: 232
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 19
+        start: 20
         end: "lastCompletedBuild"
 
+    plot-performance-trending-vpp-nfv-2n-skx:
+      csit-vpp-perf-mrr-weekly-master-2n-skx:
+        start: 1
+        end: "lastCompletedBuild"
+        skip:
+          - 3
+          - 5
+
   plot-layouts:
 
     plot-cpta:
@@ -295,26 +307,34 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 470
+      start: 482
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
-      start: 54
+      start: 55
       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 220
+      start: 232
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-3n-skx:
-      start: 19
-      end: 30 # "lastCompletedBuild"
+      start: 20
+      end: "lastCompletedBuild"
+      skip:
+        - 31
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 220
+      start: 232
       end: "lastCompletedBuild"
+    csit-vpp-perf-mrr-weekly-master-2n-skx:
+      start: 1
+      end: "lastCompletedBuild"
+      skip:
+        - 3
+        - 5
     csit-dpdk-perf-mrr-weekly-master-2n-skx:
-      start: 19
+      start: 20
       end: "lastCompletedBuild"
 
 -
@@ -672,6 +692,216 @@
   plots:
 
 ################################################################################
+# NFV
+
+    - title: "VPP 2T1C NF Service Density VNFc 64B MRR Trending 2n-skx"
+      output-file-name: "vnfc-eth-l2bd-64b-2t1c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'VM' and 'CHAIN' and 'MRR' and '64B' and '2T1C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Chain"
+      - "2 Chains"
+      - "4 Chains"
+      - "6 Chains"
+      - "8 Chains"
+      - "10 Chains"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 4T2C NF Service Density VNFc 64B MRR Trending 2n-skx"
+      output-file-name: "vnfc-eth-l2bd-64b-4t2c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'VM' and 'CHAIN' and 'MRR' and '64B' and '4T2C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Chain"
+      - "2 Chains"
+      - "4 Chains"
+      - "6 Chains"
+      - "8 Chains"
+      - "10 Chains"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 8T4C NF Service Density VNFc 64B MRR Trending 2n-skx"
+      output-file-name: "vnfc-eth-l2bd-64b-8t4c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'VM' and 'CHAIN' and 'MRR' and '64B' and '8T4C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Chain"
+      - "2 Chains"
+      - "4 Chains"
+      - "6 Chains"
+      - "8 Chains"
+      - "10 Chains"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 2T1C NF Service Density CNFc 64B MRR Trending 2n-skx"
+      output-file-name: "cnfc-eth-l2bd-64b-2t1c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'DOCKER' and 'CHAIN' and 'MRR' and '64B' and '2T1C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Chain"
+      - "2 Chains"
+      - "4 Chains"
+      - "6 Chains"
+      - "8 Chains"
+      - "10 Chains"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 4T2C NF Service Density CNFc 64B MRR Trending 2n-skx"
+      output-file-name: "cnfc-eth-l2bd-64b-4t2c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'DOCKER' and 'CHAIN' and 'MRR' and '64B' and '4T2C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Chain"
+      - "2 Chains"
+      - "4 Chains"
+      - "6 Chains"
+      - "8 Chains"
+      - "10 Chains"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 8T4C NF Service Density CNFc 64B MRR Trending 2n-skx"
+      output-file-name: "cnfc-eth-l2bd-64b-8t4c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'DOCKER' and 'CHAIN' and 'MRR' and '64B' and '8T4C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Chain"
+      - "2 Chains"
+      - "4 Chains"
+      - "6 Chains"
+      - "8 Chains"
+      - "10 Chains"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 2T1C NF Service Density CNFp 64B MRR Trending 2n-skx"
+      output-file-name: "cnfp-eth-l2bd-64b-2t1c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'DOCKER' and 'PIPELINE' and 'MRR' and '64B' and '2T1C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Pipe"
+      - "2 Pipes"
+      - "4 Pipes"
+      - "6 Pipes"
+      - "8 Pipes"
+      - "10 Pipes"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 4T2C NF Service Density CNFp 64B MRR Trending 2n-skx"
+      output-file-name: "cnfp-eth-l2bd-64b-4t2c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'DOCKER' and 'PIPELINE' and 'MRR' and '64B' and '4T2C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Pipe"
+      - "2 Pipes"
+      - "4 Pipes"
+      - "6 Pipes"
+      - "8 Pipes"
+      - "10 Pipes"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+    - title: "VPP 8T4C NF Service Density CNFp 64B MRR Trending 2n-skx"
+      output-file-name: "cnfp-eth-l2bd-64b-8t4c-x710-2n-skx"
+      data: "plot-performance-trending-vpp-nfv-2n-skx"
+      filter: "'NIC_Intel-X710' and 'NF_DENSITY' and 'DOCKER' and 'PIPELINE' and 'MRR' and '64B' and '8T4C' and 'L2BDMACLRN'"
+      groups:
+      - ["1R1C", "1R2C", "1R4C", "1R6C", "1R8C", "1R10C"]
+      - ["2R1C", "2R2C", "2R4C", "2R6C", "2R8C"]
+      - ["4R1C", "4R2C", "4R4C"]
+      - ["6R1C", "6R2C"]
+      - ["8R1C", "8R2C"]
+      - ["10R1C"]
+      group-names:
+      - "1 Pipe"
+      - "2 Pipes"
+      - "4 Pipes"
+      - "6 Pipes"
+      - "8 Pipes"
+      - "10 Pipes"
+      parameters:
+      - "result"
+      - "tags"
+      layout: "plot-cpta"
+
+################################################################################
 # 3n-hsw
 
 # L2 - VIC-1227
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index b459bd3117..ae566c67fa 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -425,13 +425,14 @@ class Specification(object):
             for job, builds in data_set.items():
                 if builds:
                     if isinstance(builds, dict):
-                        build_nr = builds.get("end", None)
+                        build_end = builds.get("end", None)
                         try:
-                            build_nr = int(build_nr)
+                            build_end = int(build_end)
                         except ValueError:
                             # defined as a range <start, build_type>
-                            build_nr = self._get_build_number(job, build_nr)
-                        builds = [x for x in range(builds["start"], build_nr+1)]
+                            build_end = self._get_build_number(job, build_end)
+                        builds = [x for x in range(builds["start"], build_end+1)
+                                  if x not in builds.get("skip", list())]
                         self.configuration["data-sets"][set_name][job] = builds
 
         # Data sets: add sub-sets to sets (only one level):
@@ -515,13 +516,14 @@ class Specification(object):
             for job, builds in self._cfg_yaml[idx]["builds"].items():
                 if builds:
                     if isinstance(builds, dict):
-                        build_nr = builds.get("end", None)
+                        build_end = builds.get("end", None)
                         try:
-                            build_nr = int(build_nr)
+                            build_end = int(build_end)
                         except ValueError:
                             # defined as a range <start, build_type>
-                            build_nr = self._get_build_number(job, build_nr)
-                        builds = [x for x in range(builds["start"], build_nr+1)]
+                            build_end = self._get_build_number(job, build_end)
+                        builds = [x for x in range(builds["start"], build_end+1)
+                                  if x not in builds.get("skip", list())]
                     self._specification["input"]["builds"][job] = list()
                     for build in builds:
                         self._specification["input"]["builds"][job]. \