Trending: Align the content of old and new dirs

Change-Id: I39d1f71adac0bad3dbe18a987909904ea5c1e6cb
Signed-off-by: Tibor Frank <tifrank@cisco.com>

9 files changed, 132 insertions, 41 deletions

diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index cbdb1fd07b..83dfe2eb17 100644
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -70,7 +70,13 @@ class Alerting(object):
         # Implemented alerts:
         self._ALERTS = ("failed-tests", )
 
-        self._spec = spec.alerting
+        try:
+            self._spec = spec.alerting
+        except KeyError as err:
+            raise  AlertingError("Alerting is not configured, skipped.",
+                                 repr(err),
+                                 "WARNING")
+
         self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 5e419210aa..39cba09d96 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -112,8 +112,11 @@ def main():
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))
-            alert = Alerting(spec)
-            alert.generate_alerts()
+            try:
+                alert = Alerting(spec)
+                alert.generate_alerts()
+            except AlertingError as err:
+                logging.warning(repr(err))
             logging.info("Successfully finished.")
         ret_code = 0
 
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 846fa712d3..45ea85c12c 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -140,20 +140,20 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 290
+        start: 300
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
-        start: 38
+        start: 50
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 290
+        start: 300
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
-        start: 38
+        start: 50
         end: "lastCompletedBuild"
 
     # 3n-skx
@@ -282,10 +282,10 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 290
+      start: 300
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
-      start: 38
+      start: 50
       end: "lastCompletedBuild"
 
     # 3n-skx
diff --git a/resources/tools/presentation_new/generator_CPTA.py b/resources/tools/presentation_new/generator_CPTA.py
index f230a64c2f..24dc28c2f7 100644
--- a/resources/tools/presentation_new/generator_CPTA.py
+++ b/resources/tools/presentation_new/generator_CPTA.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -427,7 +427,8 @@ def _generate_all_charts(spec, input_data):
             builds_dict[job] = list()
         for build in spec.input["builds"][job]:
             status = build["status"]
-            if status != "failed" and status != "not found":
+            if status != "failed" and status != "not found" and \
+                status != "removed":
                 builds_dict[job].append(str(build["build"]))
 
     # Create "build ID": "date" dict:
diff --git a/resources/tools/presentation_new/generator_alerts.py b/resources/tools/presentation_new/generator_alerts.py
index cbdb1fd07b..83dfe2eb17 100644
--- a/resources/tools/presentation_new/generator_alerts.py
+++ b/resources/tools/presentation_new/generator_alerts.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -70,7 +70,13 @@ class Alerting(object):
         # Implemented alerts:
         self._ALERTS = ("failed-tests", )
 
-        self._spec = spec.alerting
+        try:
+            self._spec = spec.alerting
+        except KeyError as err:
+            raise  AlertingError("Alerting is not configured, skipped.",
+                                 repr(err),
+                                 "WARNING")
+
         self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:
diff --git a/resources/tools/presentation_new/input_data_parser.py b/resources/tools/presentation_new/input_data_parser.py
index 0712156f77..f7a2094e58 100644
--- a/resources/tools/presentation_new/input_data_parser.py
+++ b/resources/tools/presentation_new/input_data_parser.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -30,12 +30,19 @@ from robot import errors
 from collections import OrderedDict
 from string import replace
 from os import remove
+from os.path import join
+from datetime import datetime as dt
+from datetime import timedelta
 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
 
 from input_data_files import download_and_unzip_data_file
 from utils import Worker
 
 
+# Separator used in file names
+SEPARATOR = "__"
+
+
 class ExecutionChecker(ResultVisitor):
     """Class to traverse through the test suite structure.
 
@@ -696,7 +703,7 @@ class ExecutionChecker(ResultVisitor):
                 else:
                     test_result["status"] = "FAIL"
                     self._data["tests"][self._test_ID] = test_result
-                    logging.error("The test '{0}' has no or more than one "
+                    logging.debug("The test '{0}' has no or more than one "
                                   "multi-threading tags.".format(self._test_ID))
                     logging.debug("Tags: {0}".format(test_result["tags"]))
                     return
@@ -1139,7 +1146,45 @@ class InputData(object):
                 remove(build["file-name"])
             except OSError as err:
                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
-                             format(build["file-name"], err)))
+                             format(build["file-name"], repr(err))))
+
+        # If the time-period is defined in the specification file, remove all
+        # files which are outside the time period.
+        timeperiod = self._cfg.input.get("time-period", None)
+        if timeperiod and data:
+            now = dt.utcnow()
+            timeperiod = timedelta(int(timeperiod))
+            metadata = data.get("metadata", None)
+            if metadata:
+                generated = metadata.get("generated", None)
+                if generated:
+                    generated = dt.strptime(generated, "%Y%m%d %H:%M")
+                    if (now - generated) > timeperiod:
+                        # Remove the data and the file:
+                        state = "removed"
+                        data = None
+                        logs.append(
+                            ("INFO",
+                             "    The build {job}/{build} is outdated, will be "
+                             "removed".format(job=job, build=build["build"])))
+                        file_name = self._cfg.input["file-name"]
+                        full_name = join(
+                            self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
+                            "{job}{sep}{build}{sep}{name}".
+                                format(job=job,
+                                       sep=SEPARATOR,
+                                       build=build["build"],
+                                       name=file_name))
+                        try:
+                            remove(full_name)
+                            logs.append(("INFO",
+                                         "    The file {name} has been removed".
+                                         format(name=full_name)))
+                        except OSError as err:
+                            logs.append(("ERROR",
+                                        "Cannot remove the file '{0}': {1}".
+                                        format(full_name, repr(err))))
+
         logs.append(("INFO", "  Done."))
 
         result = {
diff --git a/resources/tools/presentation_new/pal.py b/resources/tools/presentation_new/pal.py
index 5e419210aa..39cba09d96 100644
--- a/resources/tools/presentation_new/pal.py
+++ b/resources/tools/presentation_new/pal.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -112,8 +112,11 @@ def main():
             logging.info("Successfully finished.")
         elif spec.output["output"] == "CPTA":
             sys.stdout.write(generate_cpta(spec, data))
-            alert = Alerting(spec)
-            alert.generate_alerts()
+            try:
+                alert = Alerting(spec)
+                alert.generate_alerts()
+            except AlertingError as err:
+                logging.warning(repr(err))
             logging.info("Successfully finished.")
         ret_code = 0
 
diff --git a/resources/tools/presentation_new/specification_CPTA.yaml b/resources/tools/presentation_new/specification_CPTA.yaml
index 88a68d1117..730f2fea2e 100644
--- a/resources/tools/presentation_new/specification_CPTA.yaml
+++ b/resources/tools/presentation_new/specification_CPTA.yaml
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -73,58 +73,58 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 260
+        start: 300
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
-        start: 3
+        start: 50
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 260
+        start: 300
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
-        start: 3
+        start: 50
         end: "lastCompletedBuild"
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 20
+        start: 50
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 1
+        start: 3
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 20
+        start: 50
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 1
+        start: 3
         end: "lastCompletedBuild"
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 20
+        start: 50
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 1
+        start: 3
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 20
+        start: 50
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 1
+        start: 3
         end: "lastCompletedBuild"
 
   plot-layouts:
@@ -207,31 +207,34 @@
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"
+    # Number of days from now to the past. Only files generated in this
+    # time period are used.
+    time-period: 90
 
   builds:
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 260
+      start: 300
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
-      start: 3
+      start: 50
       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 20
+      start: 50
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-3n-skx:
-      start: 1
+      start: 3
       end: "lastCompletedBuild"
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 20
+      start: 50
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-2n-skx:
-      start: 1
+      start: 3
       end: "lastCompletedBuild"
 
 -
diff --git a/resources/tools/presentation_new/utils.py b/resources/tools/presentation_new/utils.py
index 51bb1d0305..3fdec85774 100644
--- a/resources/tools/presentation_new/utils.py
+++ b/resources/tools/presentation_new/utils.py
@@ -25,6 +25,7 @@ from os import walk, makedirs, environ
 from os.path import join, isdir
 from shutil import move, Error
 from math import sqrt
+from datetime import datetime
 
 from errors import PresentationError
 from jumpavg.BitCountingClassifier import BitCountingClassifier
@@ -178,6 +179,29 @@ def get_last_completed_build_number(jenkins_url, job_name):
     return execute_command(cmd)
 
 
+def get_build_timestamp(jenkins_url, job_name, build_nr):
+    """Get the timestamp of the build of the given job.
+
+    :param jenkins_url: Jenkins URL.
+    :param job_name: Job name.
+    :param build_nr: Build number.
+    :type jenkins_url: str
+    :type job_name: str
+    :type build_nr: int
+    :returns: The timestamp.
+    :rtype: datetime.datetime
+    """
+
+    url = "{jenkins_url}/{job_name}/{build_nr}".format(jenkins_url=jenkins_url,
+                                                       job_name=job_name,
+                                                       build_nr=build_nr)
+    cmd = "wget -qO- {url}".format(url=url)
+
+    timestamp = execute_command(cmd)
+
+    return datetime.fromtimestamp(timestamp/1000)
+
+
 def archive_input_data(spec):
     """Archive the report.