CSIT-1041: Trending dashboard

Change-Id: I983c5cccd165fb32742d395cf7e8aa02c7f9394a
Signed-off-by: Tibor Frank <tifrank@cisco.com>

5 files changed, 118 insertions, 73 deletions

diff --git a/docs/cpta/index.rst b/docs/cpta/index.rst
index 6ae8a521dc..f9fddfc58f 100644
--- a/docs/cpta/index.rst
+++ b/docs/cpta/index.rst
@@ -20,6 +20,9 @@ VPP Performance Trending Dashboard
 .. include:: ../../_build/_static/vpp/performance-trending-dashboard-4t4c.rst
 
 
+Contents
+--------
+
 .. toctree::
     :maxdepth: 6
     :numbered:
diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 3a8ea93e0a..066bfbddc8 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -164,19 +164,21 @@ def _evaluate_results(in_data, trimmed_data, window=10):
 
     if len(in_data) > 2:
         win_size = in_data.size if in_data.size < window else window
-        results = [0.0, ] * win_size
+        results = [0.0, ]
         median = in_data.rolling(window=win_size).median()
         stdev_t = trimmed_data.rolling(window=win_size, min_periods=2).std()
         m_vals = median.values
         s_vals = stdev_t.values
         d_vals = in_data.values
-        for day in range(win_size, in_data.size):
-            if np.isnan(m_vals[day - 1]) or np.isnan(s_vals[day - 1]):
+        for day in range(1, in_data.size):
+            if np.isnan(m_vals[day]) \
+                    or np.isnan(s_vals[day]) \
+                    or np.isnan(d_vals[day]):
                 results.append(0.0)
-            elif d_vals[day] < (m_vals[day - 1] - 3 * s_vals[day - 1]):
+            elif d_vals[day] < (m_vals[day] - 3 * s_vals[day]):
                 results.append(0.33)
-            elif (m_vals[day - 1] - 3 * s_vals[day - 1]) <= d_vals[day] <= \
-                    (m_vals[day - 1] + 3 * s_vals[day - 1]):
+            elif (m_vals[day] - 3 * s_vals[day]) <= d_vals[day] <= \
+                    (m_vals[day] + 3 * s_vals[day]):
                 results.append(0.66)
             else:
                 results.append(1.0)
@@ -244,7 +246,7 @@ def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
     data_y = [val for val in in_data.values()]
     data_pd = pd.Series(data_y, index=data_x)
 
-    t_data, outliers = find_outliers(data_pd)
+    t_data, outliers = find_outliers(data_pd, outlier_const=1.5)
 
     results = _evaluate_results(data_pd, t_data, window=moving_win_size)
 
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 13c8efffdb..12cbee2dae 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -574,38 +574,75 @@ def table_performance_trending_dashboard(table, input_data):
     tbl_lst = list()
     for tst_name in tbl_dict.keys():
         if len(tbl_dict[tst_name]["data"]) > 2:
-            pd_data = pd.Series(tbl_dict[tst_name]["data"])
+            sample_lst = tbl_dict[tst_name]["data"]
+            pd_data = pd.Series(sample_lst)
             win_size = pd_data.size \
                 if pd_data.size < table["window"] else table["window"]
             # Test name:
             name = tbl_dict[tst_name]["name"]
-            # Throughput trend:
-            trend = list(pd_data.rolling(window=win_size, min_periods=2).
-                         median())[-2]
-            # Anomaly:
+
+            # Trend list:
+            trend_lst = list(pd_data.rolling(window=win_size, min_periods=2).
+                             median())
+            # Stdevs list:
             t_data, _ = find_outliers(pd_data)
-            last = list(t_data)[-1]
-            t_stdev = list(t_data.rolling(window=win_size, min_periods=2).
-                         std())[-2]
-            if isnan(last):
-                classification = "outlier"
-                last = list(pd_data)[-1]
-            elif last < (trend - 3 * t_stdev):
+            t_data_lst = list(t_data)
+            stdev_lst = list(t_data.rolling(window=win_size, min_periods=2).
+                             std())
+
+            rel_change_lst = [None, ]
+            classification_lst = [None, ]
+            for idx in range(1, len(trend_lst)):
+                # Relative changes list:
+                if not isnan(sample_lst[idx]) \
+                        and not isnan(trend_lst[idx])\
+                        and trend_lst[idx] != 0:
+                    rel_change_lst.append(
+                        int(relative_change(float(trend_lst[idx]),
+                                            float(sample_lst[idx]))))
+                else:
+                    rel_change_lst.append(None)
+                # Classification list:
+                if isnan(t_data_lst[idx]) or isnan(stdev_lst[idx]):
+                    classification_lst.append("outlier")
+                elif sample_lst[idx] < (trend_lst[idx] - 3*stdev_lst[idx]):
+                    classification_lst.append("regression")
+                elif sample_lst[idx] > (trend_lst[idx] + 3*stdev_lst[idx]):
+                    classification_lst.append("progression")
+                else:
+                    classification_lst.append("normal")
+
+            last_idx = len(sample_lst) - 1
+            first_idx = last_idx - int(table["evaluated-window"])
+            if first_idx < 0:
+                first_idx = 0
+
+            if "regression" in classification_lst[first_idx:]:
                 classification = "regression"
-            elif last > (trend + 3 * t_stdev):
+            elif "outlier" in classification_lst[first_idx:]:
+                classification = "outlier"
+            elif "progression" in classification_lst[first_idx:]:
                 classification = "progression"
             else:
                 classification = "normal"
 
-            if not isnan(last) and not isnan(trend) and trend != 0:
-                # Relative change:
-                rel_change = int(relative_change(float(trend), float(last)))
-
-                tbl_lst.append([name,
-                                round(float(trend) / 1000000, 2),
-                                round(float(last) / 1000000, 2),
-                                rel_change,
-                                classification])
+            idx = len(classification_lst) - 1
+            while idx:
+                if classification_lst[idx] == classification:
+                    break
+                idx -= 1
+
+            trend = round(float(trend_lst[-2]) / 1000000, 2) \
+                if not isnan(trend_lst[-2]) else ''
+            sample = round(float(sample_lst[idx]) / 1000000, 2) \
+                if not isnan(sample_lst[idx]) else ''
+            rel_change = rel_change_lst[idx] \
+                if rel_change_lst[idx] is not None else ''
+            tbl_lst.append([name,
+                            trend,
+                            sample,
+                            rel_change,
+                            classification])
 
     # Sort the table according to the classification
     tbl_sorted = list()
@@ -684,11 +721,11 @@ def table_performance_trending_dashboard_html(table, input_data):
             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
             if c_idx == 4:
                 if item == "regression":
-                    td.set("bgcolor", "#FF0000")
+                    td.set("bgcolor", "#eca1a6")
                 elif item == "outlier":
-                    td.set("bgcolor", "#818181")
+                    td.set("bgcolor", "#d6cbd3")
                 elif item == "progression":
-                    td.set("bgcolor", "#008000")
+                    td.set("bgcolor", "#bdcebe")
             td.text = item
 
     try:
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 98642c898c..aaeacaac15 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -87,48 +87,48 @@ def main():
         return 1
 
     ret_code = 0
-    try:
-        env = Environment(spec.environment, args.force)
-        env.set_environment()
-
-        if spec.is_debug:
-            if spec.debug["input-format"] == "zip":
-                unzip_files(spec)
-        else:
-            download_data_files(spec)
-
-        prepare_static_content(spec)
-
-        data = InputData(spec)
-        data.read_data()
-
-        generate_tables(spec, data)
-        generate_plots(spec, data)
-        generate_files(spec, data)
-
-        if spec.output["output"] == "report":
-            generate_report(args.release, spec)
-            logging.info("Successfully finished.")
-        elif spec.output["output"] == "CPTA":
-            sys.stdout.write(generate_cpta(spec, data))
-            logging.info("Successfully finished.")
-        else:
-            logging.critical("The output '{0}' is not supported.".
-                             format(spec.output["output"]))
-            ret_code = 1
-
-    except (KeyError, ValueError, PresentationError) as err:
-        logging.info("Finished with an error.")
-        logging.critical(str(err))
-        ret_code = 1
-    except Exception as err:
-        logging.info("Finished with an unexpected error.")
-        logging.critical(str(err))
+    # try:
+    env = Environment(spec.environment, args.force)
+    env.set_environment()
+
+    if spec.is_debug:
+        if spec.debug["input-format"] == "zip":
+            unzip_files(spec)
+    else:
+        download_data_files(spec)
+
+    prepare_static_content(spec)
+
+    data = InputData(spec)
+    data.read_data()
+
+    generate_tables(spec, data)
+    generate_plots(spec, data)
+    generate_files(spec, data)
+
+    if spec.output["output"] == "report":
+        generate_report(args.release, spec)
+        logging.info("Successfully finished.")
+    elif spec.output["output"] == "CPTA":
+        sys.stdout.write(generate_cpta(spec, data))
+        logging.info("Successfully finished.")
+    else:
+        logging.critical("The output '{0}' is not supported.".
+                         format(spec.output["output"]))
         ret_code = 1
-    finally:
-        if spec is not None and not spec.is_debug:
-            clean_environment(spec.environment)
-        return ret_code
+
+    # except (KeyError, ValueError, PresentationError) as err:
+    #     logging.info("Finished with an error.")
+    #     logging.critical(str(err))
+    #     ret_code = 1
+    # except Exception as err:
+    #     logging.info("Finished with an unexpected error.")
+    #     logging.critical(str(err))
+    #     ret_code = 1
+    # finally:
+    #     if spec is not None and not spec.is_debug:
+    #         clean_environment(spec.environment)
+    #     return ret_code
 
 
 if __name__ == '__main__':
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 0544acd040..f6d07e8063 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -227,6 +227,7 @@
   nr-of-tests-shown: 20
   outlier-const: 1.5
   window: 10
+  evaluated-window: 14
 
 -
   type: "table"
@@ -245,6 +246,7 @@
   nr-of-tests-shown: 20
   outlier-const: 1.5
   window: 10
+  evaluated-window: 14
 
 -
   type: "table"
@@ -263,6 +265,7 @@
   nr-of-tests-shown: 20
   outlier-const: 1.5
   window: 10
+  evaluated-window: 14
 
 -
   type: "table"