Bar graph for Soak tests results

CSIT-1401: Create bar graph for Soak tests results
CSIT-1405: Add data pre-processing for "soak tests" graphs

Change-Id: I158c54e713cb904eb1780190153413929c7bab6d
Signed-off-by: Tibor Frank <tifrank@cisco.com>

3 files changed, 383 insertions, 1 deletions

diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 32f146bca8..21dd1a0555 100644
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -195,6 +195,248 @@ def plot_performance_box(plot, input_data):
         return
 
 
+def plot_soak_bars(plot, input_data):
+    """Generate the plot(s) with algorithm: plot_soak_bars
+    specified in the specification file.
+
+    :param plot: Plot to generate.
+    :param input_data: Data to process.
+    :type plot: pandas.Series
+    :type input_data: InputData
+    """
+
+    # Transform the data
+    plot_title = plot.get("title", "")
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(plot.get("type", ""), plot_title))
+    data = input_data.filter_data(plot)
+    if data is None:
+        logging.error("No data.")
+        return
+
+    # Prepare the data for the plot
+    y_vals = dict()
+    y_tags = dict()
+    for job in data:
+        for build in job:
+            for test in build:
+                if y_vals.get(test["parent"], None) is None:
+                    y_tags[test["parent"]] = test.get("tags", None)
+                try:
+                    if test["type"] in ("SOAK", ):
+                        y_vals[test["parent"]] = test["throughput"]
+                    else:
+                        continue
+                except (KeyError, TypeError):
+                    y_vals[test["parent"]] = dict()
+
+    # Sort the tests
+    order = plot.get("sort", None)
+    if order and y_tags:
+        y_sorted = OrderedDict()
+        y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
+        for tag in order:
+            logging.debug(tag)
+            for suite, tags in y_tags_l.items():
+                if "not " in tag:
+                    tag = tag.split(" ")[-1]
+                    if tag.lower() in tags:
+                        continue
+                else:
+                    if tag.lower() not in tags:
+                        continue
+                try:
+                    y_sorted[suite] = y_vals.pop(suite)
+                    y_tags_l.pop(suite)
+                    logging.debug(suite)
+                except KeyError as err:
+                    logging.error("Not found: {0}".format(repr(err)))
+                finally:
+                    break
+    else:
+        y_sorted = y_vals
+
+    idx = 0
+    y_max = 0
+    traces = list()
+    for test_name, test_data in y_sorted.items():
+        idx += 1
+        name = "{nr}. {name}".\
+            format(nr=idx, name=test_name.lower().replace('-soak', ''))
+        if len(name) > 50:
+            name_lst = name.split('-')
+            name = ""
+            split_name = True
+            for segment in name_lst:
+                if (len(name) + len(segment) + 1) > 50 and split_name:
+                    name += "<br>    "
+                    split_name = False
+                name += segment + '-'
+            name = name[:-1]
+
+        y_val = test_data.get("LOWER", None)
+        if y_val:
+            y_val /= 1000000
+            if y_val > y_max:
+                y_max = y_val
+
+        time = "No Information"
+        result = "No Information"
+        hovertext = ("{name}<br>"
+                     "Packet Throughput: {val:.2f}Mpps<br>"
+                     "Final Duration: {time}<br>"
+                     "Result: {result}".format(name=name,
+                                               val=y_val,
+                                               time=time,
+                                               result=result))
+        traces.append(plgo.Bar(x=[str(idx) + '.', ],
+                               y=[y_val, ],
+                               name=name,
+                               text=hovertext,
+                               hoverinfo="text"))
+    try:
+        # Create plot
+        layout = deepcopy(plot["layout"])
+        if layout.get("title", None):
+            layout["title"] = "<b>Packet Throughput:</b> {0}". \
+                format(layout["title"])
+        if y_max:
+            layout["yaxis"]["range"] = [0, y_max + 1]
+        plpl = plgo.Figure(data=traces, layout=layout)
+        # Export Plot
+        logging.info("    Writing file '{0}{1}'.".
+                     format(plot["output-file"], plot["output-file-type"]))
+        ploff.plot(plpl, show_link=False, auto_open=False,
+                   filename='{0}{1}'.format(plot["output-file"],
+                                            plot["output-file-type"]))
+    except PlotlyError as err:
+        logging.error("   Finished with error: {}".
+                      format(repr(err).replace("\n", " ")))
+        return
+
+
+def plot_soak_boxes(plot, input_data):
+    """Generate the plot(s) with algorithm: plot_soak_boxes
+    specified in the specification file.
+
+    :param plot: Plot to generate.
+    :param input_data: Data to process.
+    :type plot: pandas.Series
+    :type input_data: InputData
+    """
+
+    # Transform the data
+    plot_title = plot.get("title", "")
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(plot.get("type", ""), plot_title))
+    data = input_data.filter_data(plot)
+    if data is None:
+        logging.error("No data.")
+        return
+
+    # Prepare the data for the plot
+    y_vals = dict()
+    y_tags = dict()
+    for job in data:
+        for build in job:
+            for test in build:
+                if y_vals.get(test["parent"], None) is None:
+                    y_tags[test["parent"]] = test.get("tags", None)
+                try:
+                    if test["type"] in ("SOAK", ):
+                        y_vals[test["parent"]] = test["throughput"]
+                    else:
+                        continue
+                except (KeyError, TypeError):
+                    y_vals[test["parent"]] = dict()
+
+    # Sort the tests
+    order = plot.get("sort", None)
+    if order and y_tags:
+        y_sorted = OrderedDict()
+        y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()}
+        for tag in order:
+            logging.debug(tag)
+            for suite, tags in y_tags_l.items():
+                if "not " in tag:
+                    tag = tag.split(" ")[-1]
+                    if tag.lower() in tags:
+                        continue
+                else:
+                    if tag.lower() not in tags:
+                        continue
+                try:
+                    y_sorted[suite] = y_vals.pop(suite)
+                    y_tags_l.pop(suite)
+                    logging.debug(suite)
+                except KeyError as err:
+                    logging.error("Not found: {0}".format(repr(err)))
+                finally:
+                    break
+    else:
+        y_sorted = y_vals
+
+    idx = 0
+    y_max = 0
+    traces = list()
+    for test_name, test_data in y_sorted.items():
+        idx += 1
+        name = "{nr}. {name}".\
+            format(nr=idx, name=test_name.lower().replace('-soak', ''))
+        if len(name) > 50:
+            name_lst = name.split('-')
+            name = ""
+            split_name = True
+            for segment in name_lst:
+                if (len(name) + len(segment) + 1) > 50 and split_name:
+                    name += "<br>    "
+                    split_name = False
+                name += segment + '-'
+            name = name[:-1]
+
+        y_val = test_data.get("UPPER", None)
+        if y_val:
+            y_val /= 1000000
+            if y_val > y_max:
+                y_max = y_val
+
+        y_base = test_data.get("LOWER", None)
+        if y_base:
+            y_base /= 1000000
+
+        hovertext = ("{name}<br>"
+                     "Upper bound: {upper:.2f}Mpps<br>"
+                     "Lower bound: {lower:.2f}Mpps".format(name=name,
+                                                           upper=y_val,
+                                                           lower=y_base))
+        traces.append(plgo.Bar(x=[str(idx) + '.', ],
+                               # +0.05 to see the value in case lower == upper
+                               y=[y_val - y_base + 0.05, ],
+                               base=y_base,
+                               name=name,
+                               text=hovertext,
+                               hoverinfo="text"))
+    try:
+        # Create plot
+        layout = deepcopy(plot["layout"])
+        if layout.get("title", None):
+            layout["title"] = "<b>Soak Tests:</b> {0}". \
+                format(layout["title"])
+        if y_max:
+            layout["yaxis"]["range"] = [0, y_max + 1]
+        plpl = plgo.Figure(data=traces, layout=layout)
+        # Export Plot
+        logging.info("    Writing file '{0}{1}'.".
+                     format(plot["output-file"], plot["output-file-type"]))
+        ploff.plot(plpl, show_link=False, auto_open=False,
+                   filename='{0}{1}'.format(plot["output-file"],
+                                            plot["output-file-type"]))
+    except PlotlyError as err:
+        logging.error("   Finished with error: {}".
+                      format(repr(err).replace("\n", " ")))
+        return
+
+
 def plot_latency_error_bars(plot, input_data):
     """Generate the plot(s) with algorithm: plot_latency_error_bars
     specified in the specification file.
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 5e4ca42de4..7b36352ed9 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -252,6 +252,9 @@ class ExecutionChecker(ResultVisitor):
     # TODO: Remove when definitely no NDRPDRDISC tests are used:
     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 
+    REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n'
+                                r'PLRsearch upper bound::\s(\d+.\d+)')
+
     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
@@ -567,6 +570,33 @@ class ExecutionChecker(ResultVisitor):
 
         return throughput, status
 
+    def _get_plr_throughput(self, msg):
+        """Get PLRsearch lower bound and PLRsearch upper bound from the test
+        message.
+
+        :param msg: The test message to be parsed.
+        :type msg: str
+        :returns: Parsed data as a dict and the status (PASS/FAIL).
+        :rtype: tuple(dict, str)
+        """
+
+        throughput = {
+            "LOWER": -1.0,
+            "UPPER": -1.0
+        }
+        status = "FAIL"
+        groups = re.search(self.REGEX_PLR_RATE, msg)
+
+        if groups is not None:
+            try:
+                throughput["LOWER"] = float(groups.group(1))
+                throughput["UPPER"] = float(groups.group(2))
+                status = "PASS"
+            except (IndexError, ValueError):
+                pass
+
+        return throughput, status
+
     def _get_ndrpdr_latency(self, msg):
         """Get LATENCY from the test message.
 
@@ -739,6 +769,7 @@ class ExecutionChecker(ResultVisitor):
 
         if test.status == "PASS" and ("NDRPDRDISC" in tags or
                                       "NDRPDR" in tags or
+                                      "SOAK" in tags or
                                       "TCP" in tags or
                                       "MRR" in tags or
                                       "BMRR" in tags):
@@ -750,6 +781,8 @@ class ExecutionChecker(ResultVisitor):
                 test_result["type"] = "PDR"
             elif "NDRPDR" in tags:
                 test_result["type"] = "NDRPDR"
+            elif "SOAK" in tags:
+                test_result["type"] = "SOAK"
             elif "TCP" in tags:
                 test_result["type"] = "TCP"
             elif "MRR" in tags:
@@ -790,6 +823,10 @@ class ExecutionChecker(ResultVisitor):
                 test_result["latency"], test_result["status"] = \
                     self._get_ndrpdr_latency(test.message)
 
+            elif test_result["type"] in ("SOAK", ):
+                test_result["throughput"], test_result["status"] = \
+                    self._get_plr_throughput(test.message)
+
             elif test_result["type"] in ("TCP", ):
                 groups = re.search(self.REGEX_TCP, test.message)
                 test_result["result"] = int(groups.group(2))
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index bd16e165c5..832f54d574 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -488,6 +488,10 @@
       - 41  # NDRPDR sel
       - 42  # NDRPDR sel
 
+    plot-vpp-soak-2n-skx:
+      csit-vpp-perf-verify-master-2n-skx:
+      - 207  # SOAK sel
+
     plot-vpp-throughput-latency-3n-skx:
       csit-vpp-perf-verify-1810-3n-skx:
       - 24  # NDRPDR full
@@ -750,6 +754,57 @@
 
   plot-layouts:
 
+    plot-soak-throughput:
+      titlefont:
+        size: 18
+      xaxis:
+        title: "<b>Test Cases [Index]</b>"
+        titlefont:
+          size: 16
+        autorange: True
+        fixedrange: False
+        gridcolor: "rgb(230, 230, 230)"
+        linecolor: "rgb(220, 220, 220)"
+        linewidth: 1
+        showgrid: True
+        showline: True
+        showticklabels: True
+        tickcolor: "rgb(220, 220, 220)"
+        tickmode: "linear"
+        tickfont:
+          size: 16
+        zeroline: False
+      yaxis:
+        title: "<b>Packet Throughput [Mpps]</b>"
+        titlefont:
+          size: 16
+        gridcolor: "rgb(230, 230, 230)"
+        hoverformat: ".4s"
+        tickformat: ".3s"
+        linecolor: "rgb(220, 220, 220)"
+        linewidth: 1
+        showgrid: True
+        showline: True
+        showticklabels: True
+        tickcolor: "rgb(220, 220, 220)"
+        tickfont:
+          size: 16
+        zeroline: False
+        range: [0,50]
+      autosize: False
+      margin:
+        t: 50
+        b: 0
+        l: 80
+        r: 20
+      showlegend: True
+      legend:
+        orientation: "h"
+        font:
+          size: 16
+      width: 700
+      height: 900
+
     plot-cps:
       titlefont:
         size: 18
@@ -1227,6 +1282,8 @@
     - 23  # NDRPDR sel
     - 24  # NDRPDR sel
     - 27  # NDRPDR sel
+    csit-vpp-perf-verify-master-2n-skx:
+    - 207  # SOAK sel
     csit-vpp-perf-check-1804:
     - 5   # mrr - full
     - 6   # mrr - sel
@@ -5199,6 +5256,52 @@
 ###                                P L O T S                                 ###
 ################################################################################
 
+# Soak test - example
+-
+  type: "plot"
+  title: "VPP Throughput Soak: Example"
+  algorithm: "plot_soak_boxes"
+  output-file-type: ".html"
+  output-file: "{DIR[STATIC,VPP]}/example-soak-boxes"
+  data: "plot-vpp-soak-2n-skx"
+  filter: "'SOAK'"
+  parameters:
+  - "throughput"
+  - "parent"
+  - "tags"
+  sort:
+  - "L2PATCH"
+  - "L2XCBASE"
+  - "L2BDMACLRN"
+  - "IP4FWD"
+  - "IP6FWD"
+  layout:
+    title: "example-30min-boxes"
+    layout: "plot-soak-throughput"
+
+# Soak test - example
+-
+  type: "plot"
+  title: "VPP Throughput Soak: Example"
+  algorithm: "plot_soak_bars"
+  output-file-type: ".html"
+  output-file: "{DIR[STATIC,VPP]}/example-soak-bars"
+  data: "plot-vpp-soak-2n-skx"
+  filter: "'SOAK'"
+  parameters:
+  - "throughput"
+  - "parent"
+  - "tags"
+  sort:
+  - "L2PATCH"
+  - "L2XCBASE"
+  - "L2BDMACLRN"
+  - "IP4FWD"
+  - "IP6FWD"
+  layout:
+    title: "example-30min-bars"
+    layout: "plot-soak-throughput"
+
 ################################################################################
 # Plots VPP HTTP Server Performance
 -