PAL: Reverse download order for trending

Change-Id: Idd6bf8b89ce0fb5ebf9734b39a8f6058861b8ec5
Signed-off-by: Tibor Frank <tifrank@cisco.com>
(cherry picked from commit 1945ebc5fe96f2289c6ffc4cc162daa1d3b8658f)

5 files changed, 143 insertions, 167 deletions

diff --git a/resources/tools/presentation/generator_cpta.py b/resources/tools/presentation/generator_cpta.py
index 8d10e15a49..ac0a5c60e1 100644
--- a/resources/tools/presentation/generator_cpta.py
+++ b/resources/tools/presentation/generator_cpta.py
@@ -338,11 +338,7 @@ def _generate_all_charts(spec, input_data):
         :rtype: dict
         """
 
-        logs = list()
-
-        logs.append(
-            (u"INFO", f"  Generating the chart {graph.get(u'title', u'')} ...")
-        )
+        logging.info(f"  Generating the chart {graph.get(u'title', u'')} ...")
 
         job_name = list(graph[u"data"].keys())[0]
 
@@ -350,11 +346,9 @@ def _generate_all_charts(spec, input_data):
         res = dict()
 
         # Transform the data
-        logs.append(
-            (u"INFO",
+        logging.info(
              f"    Creating the data set for the {graph.get(u'type', u'')} "
              f"{graph.get(u'title', u'')}."
-            )
         )
 
         if graph.get(u"include", None):
@@ -410,13 +404,10 @@ def _generate_all_charts(spec, input_data):
                 for tag in group:
                     for tst_name, test_data in chart_data.items():
                         if not test_data:
-                            logs.append(
-                                (u"WARNING", f"No data for the test {tst_name}")
-                            )
+                            logging.warning(f"No data for the test {tst_name}")
                             continue
                         if tag not in chart_tags[tst_name]:
                             continue
-                        message = f"index: {index}, test: {tst_name}"
                         try:
                             trace, rslt = _generate_trending_traces(
                                 test_data,
@@ -426,10 +417,8 @@ def _generate_all_charts(spec, input_data):
                                                split(u'-')[2:-1]),
                                 color=COLORS[index])
                         except IndexError:
-                            logs.append(
-                                (u"ERROR", f"Out of colors: {message}")
-                            )
-                            logging.error(f"Out of colors: {message}")
+                            logging.error(f"Out of colors: index: "
+                                          f"{index}, test: {tst_name}")
                             index += 1
                             continue
                         traces.extend(trace)
@@ -441,11 +430,8 @@ def _generate_all_charts(spec, input_data):
         else:
             for tst_name, test_data in chart_data.items():
                 if not test_data:
-                    logs.append(
-                        (u"WARNING", f"No data for the test {tst_name}")
-                    )
+                    logging.warning(f"No data for the test {tst_name}")
                     continue
-                message = f"index: {index}, test: {tst_name}"
                 try:
                     trace, rslt = _generate_trending_traces(
                         test_data,
@@ -455,8 +441,9 @@ def _generate_all_charts(spec, input_data):
                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
                         color=COLORS[index])
                 except IndexError:
-                    logs.append((u"ERROR", f"Out of colors: {message}"))
-                    logging.error(f"Out of colors: {message}")
+                    logging.error(
+                        f"Out of colors: index: {index}, test: {tst_name}"
+                    )
                     index += 1
                     continue
                 traces.extend(trace)
@@ -514,25 +501,13 @@ def _generate_all_charts(spec, input_data):
                 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
                 f"{spec.cpta[u'output-file-type']}")
 
-            logs.append((u"INFO", f"    Writing the file {name_file} ..."))
+            logging.info(f"    Writing the file {name_file} ...")
             plpl = plgo.Figure(data=traces, layout=layout)
             try:
                 ploff.plot(plpl, show_link=False, auto_open=False,
                            filename=name_file)
             except plerr.PlotlyEmptyDataError:
-                logs.append((u"WARNING", u"No data for the plot. Skipped."))
-
-        for level, line in logs:
-            if level == u"INFO":
-                logging.info(line)
-            elif level == u"ERROR":
-                logging.error(line)
-            elif level == u"DEBUG":
-                logging.debug(line)
-            elif level == u"CRITICAL":
-                logging.critical(line)
-            elif level == u"WARNING":
-                logging.warning(line)
+                logging.warning(u"No data for the plot. Skipped.")
 
         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 
@@ -542,7 +517,7 @@ def _generate_all_charts(spec, input_data):
             builds_dict[job] = list()
         for build in spec.input[u"builds"][job]:
             status = build[u"status"]
-            if status not in (u"failed", u"not found", u"removed"):
+            if status not in (u"failed", u"not found", u"removed", None):
                 builds_dict[job].append(str(build[u"build"]))
 
     # Create "build ID": "date" dict:
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index e1fa5b2284..8b941f2f94 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -42,16 +42,14 @@ SEPARATOR = u"__"
 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
 
 
-def _download_file(url, file_name, log, arch=False):
+def _download_file(url, file_name, arch=False):
     """Download a file with input data.
 
     :param url: URL to the file to download.
     :param file_name: Name of file to download.
-    :param log: List of log messages.
     :param arch: If True, also .gz file is downloaded
     :type url: str
     :type file_name: str
-    :type log: list of tuples (severity, msg)
     :type arch: bool
     :returns: True if the download was successful, otherwise False.
     :rtype: bool
@@ -90,29 +88,27 @@ def _download_file(url, file_name, log, arch=False):
     success = False
     session = None
     try:
-        log.append((u"INFO", f"    Connecting to {url} ..."))
+        logging.info(f"    Connecting to {url} ...")
         session = requests_retry_session()
         response = session.get(url, stream=True)
         code = response.status_code
-        log.append((u"INFO", f"    {code}: {responses[code]}"))
+        logging.info(f"    {code}: {responses[code]}")
 
         if code != codes[u"OK"]:
             if session:
                 session.close()
             url = url.replace(u"_info", u"")
-            log.append((u"INFO", f"    Connecting to {url} ..."))
+            logging.info(f"    Connecting to {url} ...")
             session = requests_retry_session()
             response = session.get(url, stream=True)
             code = response.status_code
-            log.append((u"INFO", f"    {code}: {responses[code]}"))
+            logging.info(f"    {code}: {responses[code]}")
             if code != codes[u"OK"]:
                 return False, file_name
             file_name = file_name.replace(u"_info", u"")
 
         dst_file_name = file_name.replace(u".gz", u"")
-        log.append(
-            (u"INFO", f"    Downloading the file {url} to {dst_file_name} ...")
-        )
+        logging.info(f"    Downloading the file {url} to {dst_file_name} ...")
         with open(dst_file_name, u"wb") as file_handle:
             for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
                 if chunk:
@@ -121,44 +117,37 @@ def _download_file(url, file_name, log, arch=False):
         if arch and u".gz" in file_name:
             if session:
                 session.close()
-            log.append(
-                (u"INFO", f"    Downloading the file {url} to {file_name} ...")
-            )
+            logging.info(f"    Downloading the file {url} to {file_name} ...")
             session = requests_retry_session()
             response = session.get(url, stream=True)
             if response.status_code == codes[u"OK"]:
                 with open(file_name, u"wb") as file_handle:
                     file_handle.write(response.raw.read())
             else:
-                log.append(
-                    (u"ERROR", f"Not possible to download the file {url} to "
-                               f"{file_name} ...")
+                logging.error(
+                    f"Not possible to download the file {url} to {file_name}"
                 )
 
         success = True
     except RequestException as err:
-        log.append(
-            (u"ERROR", f"HTTP Request exception:\n{repr(err)}")
-        )
+        logging.error(f"HTTP Request exception:\n{repr(err)}")
     except (IOError, ValueError, KeyError) as err:
-        log.append((u"ERROR", f"Download failed.\n{repr(err)}"))
+        logging.error(f"Download failed.\n{repr(err)}")
     finally:
         if session:
             session.close()
 
-    log.append((u"INFO", u"    Download finished."))
+    logging.info(u"    Download finished.")
     return success, file_name
 
 
-def _unzip_file(spec, build, pid, log):
+def _unzip_file(spec, build, pid):
     """Unzip downloaded source file.
 
     :param spec: Specification read form the specification file.
     :param build: Information about the build.
-    :param log: List of log messages.
     :type spec: Specification
     :type build: dict
-    :type log: list of tuples (severity, msg)
     :returns: True if the download was successful, otherwise False.
     :rtype: bool
     """
@@ -178,42 +167,35 @@ def _unzip_file(spec, build, pid, log):
     new_name = \
         f"{file_name.rsplit(u'.')[-2]}{SEPARATOR}{data_file.split(u'/')[-1]}"
 
-    log.append((u"INFO", f"    Unzipping: {data_file} from {file_name}."))
+    logging.info(f"    Unzipping: {data_file} from {file_name}.")
     try:
         with ZipFile(file_name, u'r') as zip_file:
             zip_file.extract(data_file, tmp_dir)
-        log.append(
-            (u"INFO", f"    Renaming the file {join(tmp_dir, data_file)} to "
-                      f"{new_name}")
+        logging.info(
+            f"    Renaming the file {join(tmp_dir, data_file)} to {new_name}"
         )
         rename(join(tmp_dir, data_file), new_name)
         build[u"file-name"] = new_name
         return True
     except (BadZipfile, RuntimeError) as err:
-        log.append(
-            (u"ERROR", f"Failed to unzip the file {file_name}: {repr(err)}.")
-        )
+        logging.error(f"Failed to unzip the file {file_name}: {repr(err)}.")
         return False
     except OSError as err:
-        log.append(
-            (u"ERROR", f"Failed to rename the file {data_file}: {repr(err)}.")
-        )
+        logging.error(f"Failed to rename the file {data_file}: {repr(err)}.")
         return False
 
 
-def download_and_unzip_data_file(spec, job, build, pid, log):
+def download_and_unzip_data_file(spec, job, build, pid):
     """Download and unzip a source file.
 
     :param spec: Specification read form the specification file.
     :param job: Name of the Jenkins job.
     :param build: Information about the build.
     :param pid: PID of the process executing this method.
-    :param log: List of log messages.
     :type spec: Specification
     :type job: str
     :type build: dict
     :type pid: int
-    :type log: list of tuples (severity, msg)
     :returns: True if the download was successful, otherwise False.
     :rtype: bool
     """
@@ -235,7 +217,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
     logging.info(f"Trying to download {url}")
 
     arch = bool(spec.configuration.get(u"archive-inputs", True))
-    success, downloaded_name = _download_file(url, new_name, log, arch=arch)
+    success, downloaded_name = _download_file(url, new_name, arch=arch)
 
     if not success:
 
@@ -261,9 +243,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
                 spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
                 f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name[idx]}"
             )
-            success, downloaded_name = _download_file(
-                url, new_name, log, arch=arch
-            )
+            success, downloaded_name = _download_file(url, new_name, arch=arch)
             if success:
                 file_name = file_name[idx]
                 if file_name.endswith(u".gz"):
@@ -296,11 +276,11 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
 
         logging.info(f"Downloading {url}")
 
-        success, downloaded_name = _download_file(url, new_name, log)
+        success, downloaded_name = _download_file(url, new_name)
 
     if success and downloaded_name.endswith(u".zip"):
         if not is_zipfile(downloaded_name):
-            log.append((u"ERROR", f"Zip file {new_name} is corrupted."))
+            logging.error(f"Zip file {new_name} is corrupted.")
             success = False
 
     if success:
@@ -310,6 +290,6 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
             build[u"file-name"] = downloaded_name[:-3]
 
         if downloaded_name.endswith(u".zip"):
-            success = _unzip_file(spec, build, pid, log)
+            success = _unzip_file(spec, build, pid)
 
     return success
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index cd9c1a248d..27db6a84d8 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -1394,16 +1394,14 @@ class InputData:
         """
         return self.data[job][build][u"tests"]
 
-    def _parse_tests(self, job, build, log):
+    def _parse_tests(self, job, build):
         """Process data from robot output.xml file and return JSON structured
         data.
 
         :param job: The name of job which build output data will be processed.
         :param build: The build which output data will be processed.
-        :param log: List of log messages.
         :type job: str
         :type build: dict
-        :type log: list of tuples (severity, msg)
         :returns: JSON data structure.
         :rtype: dict
         """
@@ -1417,9 +1415,8 @@ class InputData:
             try:
                 result = ExecutionResult(data_file)
             except errors.DataError as err:
-                log.append(
-                    (u"ERROR", f"Error occurred while parsing output.xml: "
-                               f"{repr(err)}")
+                logging.error(
+                    f"Error occurred while parsing output.xml: {repr(err)}"
                 )
                 return None
         checker = ExecutionChecker(metadata, self._cfg.mapping,
@@ -1444,40 +1441,30 @@ class InputData:
         :type repeat: int
         """
 
-        logs = list()
-
-        logs.append(
-            (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
-        )
+        logging.info(f"  Processing the job/build: {job}: {build[u'build']}")
 
         state = u"failed"
         success = False
         data = None
         do_repeat = repeat
         while do_repeat:
-            success = download_and_unzip_data_file(self._cfg, job, build, pid,
-                                                   logs)
+            success = download_and_unzip_data_file(self._cfg, job, build, pid)
             if success:
                 break
             do_repeat -= 1
         if not success:
-            logs.append(
-                (u"ERROR",
+            logging.error(
                  f"It is not possible to download the input data file from the "
                  f"job {job}, build {build[u'build']}, or it is damaged. "
-                 f"Skipped.")
+                 f"Skipped."
             )
         if success:
-            logs.append(
-                (u"INFO",
-                 f"    Processing data from the build {build[u'build']} ...")
-            )
-            data = self._parse_tests(job, build, logs)
+            logging.info(f"    Processing data from build {build[u'build']}")
+            data = self._parse_tests(job, build)
             if data is None:
-                logs.append(
-                    (u"ERROR",
+                logging.error(
                      f"Input data file from the job {job}, build "
-                     f"{build[u'build']} is damaged. Skipped.")
+                     f"{build[u'build']} is damaged. Skipped."
                 )
             else:
                 state = u"processed"
@@ -1485,13 +1472,13 @@ class InputData:
             try:
                 remove(build[u"file-name"])
             except OSError as err:
-                logs.append(
-                    ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
-                              f"{repr(err)}")
+                logging.error(
+                    f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
                 )
 
         # If the time-period is defined in the specification file, remove all
         # files which are outside the time period.
+        is_last = False
         timeperiod = self._cfg.input.get(u"time-period", None)
         if timeperiod and data:
             now = dt.utcnow()
@@ -1505,26 +1492,20 @@ class InputData:
                         # Remove the data and the file:
                         state = u"removed"
                         data = None
-                        logs.append(
-                            (u"INFO",
-                             f"    The build {job}/{build[u'build']} is "
-                             f"outdated, will be removed.")
+                        is_last = True
+                        logging.info(
+                            f"    The build {job}/{build[u'build']} is "
+                            f"outdated, will be removed."
                         )
-        logs.append((u"INFO", u"  Done."))
-
-        for level, line in logs:
-            if level == u"INFO":
-                logging.info(line)
-            elif level == u"ERROR":
-                logging.error(line)
-            elif level == u"DEBUG":
-                logging.debug(line)
-            elif level == u"CRITICAL":
-                logging.critical(line)
-            elif level == u"WARNING":
-                logging.warning(line)
-
-        return {u"data": data, u"state": state, u"job": job, u"build": build}
+        logging.info(u"  Done.")
+
+        return {
+            u"data": data,
+            u"state": state,
+            u"job": job,
+            u"build": build,
+            u"last": is_last
+        }
 
     def download_and_parse_data(self, repeat=1):
         """Download the input data files, parse input data from input files and
@@ -1541,6 +1522,8 @@ class InputData:
             for build in builds:
 
                 result = self._download_and_parse_build(job, build, repeat)
+                if result[u"last"]:
+                    break
                 build_nr = result[u"build"][u"build"]
 
                 if result[u"data"]:
@@ -1908,9 +1891,10 @@ class InputData:
                                                 data[job][str(build)][
                                                     test_id][param] = u"No Data"
                         except KeyError as err:
-                            logging.error(repr(err))
                             if continue_on_error:
+                                logging.debug(repr(err))
                                 continue
+                            logging.error(repr(err))
                             return None
             return data
 
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 4d162a06e7..20efefeec3 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -214,6 +214,7 @@
         start: 1086
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -249,14 +250,13 @@
 >>>>>>> CHANGE (51210b Trending: Update graphs - dnv)
           - 1102
 >>>>>>> CHANGE (b51708 Trending: Plots)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
       csit-dpdk-perf-mrr-weekly-master:
         start: 107
         end: "lastCompletedBuild"
-        skip:
-          - 110
-          - 111
-          - 112
-          - 114
+        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
@@ -278,6 +278,7 @@
         start: 1086
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -313,16 +314,15 @@
 >>>>>>> CHANGE (51210b Trending: Update graphs - dnv)
           - 1102
 >>>>>>> CHANGE (b51708 Trending: Plots)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
         start: 107
         end: "lastCompletedBuild"
-        skip:
-          - 110
-          - 111
-          - 112
-          - 114
+        max-builds: 15  # Max nr of builds to use
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
@@ -345,6 +345,7 @@
         start: 834
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -399,13 +400,13 @@
           - 865
           - 866
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         start: 72
         end: "lastCompletedBuild"
-        skip:
-          - 76
-          - 77
-          - 79
+        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
@@ -427,6 +428,7 @@
         start: 834
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -481,15 +483,15 @@
           - 865
           - 866
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
         start: 72
         end: "lastCompletedBuild"
-        skip:
-          - 76
-          - 77
-          - 79
+        max-builds: 15  # Max nr of builds to use
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
@@ -512,6 +514,7 @@
         start: 858
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -549,11 +552,13 @@
           - 870
           - 874
 >>>>>>> CHANGE (01597c Trending: Update graphs)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         start: 78
         end: "lastCompletedBuild"
-        skip:
-          - 79
+        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
@@ -575,6 +580,7 @@
         start: 858
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -619,22 +625,21 @@
           - 870
           - 874
 >>>>>>> CHANGE (01597c Trending: Update graphs)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
         start: 78
         end: "lastCompletedBuild"
-        skip:
-          - 79
+        max-builds: 15  # Max nr of builds to use
 
     plot-performance-trending-vpp-nfv-2n-skx:
       csit-vpp-perf-mrr-weekly-master-2n-skx:
         start: 50
         end: "lastCompletedBuild"
-        skip:
-          - 52
-          - 54
-          - 55
+        max-builds: 15  # Max nr of builds to use
 
 <<<<<<< HEAD   (d3ea22 Report: Add data)
 =======
@@ -642,19 +647,13 @@
       csit-vpp-perf-mrr-daily-master-2n-clx:
         start: 236
         end: "lastCompletedBuild"
-        skip:
-          - 239
-          - 248
-          - 251
-          - 252
-          - 253
+        max-builds: 180  # Max nr of builds to use
 
     plot-performance-trending-dpdk-2n-clx:
       csit-dpdk-perf-mrr-weekly-master-2n-clx:
         start: 23
         end: "lastCompletedBuild"
-        skip:
-          - 24
+        max-builds: 15  # Max nr of builds to use
 
 >>>>>>> CHANGE (b51708 Trending: Plots)
     # 3n-tsh
@@ -662,6 +661,7 @@
       csit-vpp-perf-mrr-daily-master-3n-tsh:
         start: 144
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -687,11 +687,15 @@
           - 156
           - 157
 >>>>>>> CHANGE (01597c Trending: Update graphs)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
     plot-performance-trending-vpp-3n-tsh:
       csit-vpp-perf-mrr-daily-master-3n-tsh:
         start: 144
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -717,6 +721,9 @@
           - 156
           - 157
 >>>>>>> CHANGE (01597c Trending: Update graphs)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
     plot-performance-trending-vpp-3n-dnv:
       csit-vpp-perf-mrr-daily-master-3n-dnv:
@@ -734,6 +741,7 @@
         start: 329
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -790,6 +798,9 @@
           - 391
           - 392
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
+=======
+        max-builds: 180  # Max nr of builds to use
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
     plot-performance-trending-vpp-2n-dnv:
       csit-vpp-perf-mrr-daily-master-2n-dnv:
@@ -807,6 +818,7 @@
         start: 335
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
         end: "lastCompletedBuild"
+<<<<<<< HEAD   (89bdc8 Trending: Fixes in plots, data)
         skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
           - 145
@@ -854,6 +866,8 @@
           - 396
           - 397
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
+=======
+>>>>>>> CHANGE (1945eb PAL: Reverse download order for trending)
 
   plot-layouts:
 
@@ -969,6 +983,7 @@
       start: 1086
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -1007,6 +1022,7 @@
     csit-dpdk-perf-mrr-weekly-master:
       start: 113
       end: "lastCompletedBuild"
+      max-builds: 15  # Max nr of builds to download
       skip:
         - 114
 
@@ -1030,6 +1046,7 @@
       start: 834
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -1087,6 +1104,7 @@
     csit-dpdk-perf-mrr-weekly-master-3n-skx:
       start: 78
       end: "lastCompletedBuild"
+      max-builds: 15  # Max nr of builds to download
       skip:
         - 79
 
@@ -1110,6 +1128,7 @@
       start: 858
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -1150,6 +1169,7 @@
     csit-vpp-perf-mrr-weekly-master-2n-skx:
       start: 50
       end: "lastCompletedBuild"
+      max-builds: 15  # Max nr of builds to download
       skip:
         - 52
         - 54
@@ -1157,12 +1177,14 @@
     csit-dpdk-perf-mrr-weekly-master-2n-skx:
       start: 78
       end: "lastCompletedBuild"
+      max-builds: 15  # Max nr of builds to download
       skip:
         - 79
 
     csit-vpp-perf-mrr-daily-master-2n-clx:
       start: 236
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (914f8e Report: Add 2n-skx Soak and Reconf tests)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -1197,6 +1219,7 @@
     csit-dpdk-perf-mrr-weekly-master-2n-clx:
       start: 23
       end: "lastCompletedBuild"
+      max-builds: 15  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
         - 18
@@ -1214,6 +1237,7 @@
     csit-vpp-perf-mrr-daily-master-3n-tsh:
       start: 144
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -1256,6 +1280,7 @@
       start: 329
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
 <<<<<<< HEAD   (b54dc7 PAL: Local mode uses xml file name as a build number)
@@ -1329,6 +1354,7 @@
       start: 335
 >>>>>>> CHANGE (12d5c9 Trending: Fixes in plots, data)
       end: "lastCompletedBuild"
+      max-builds: 180  # Max nr of builds to download
       skip:
 <<<<<<< HEAD   (f7dd81 Trending: Update graphs - dnv)
         - 145
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index 61ef42ef8d..548bbff532 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -528,14 +528,15 @@ class Specification:
                     continue
                 if isinstance(builds, dict):
                     build_end = builds.get(u"end", None)
+                    max_builds = builds.get(u"max-builds", None)
                     try:
                         build_end = int(build_end)
                     except ValueError:
                         # defined as a range <start, build_type>
                         build_end = self._get_build_number(job, build_end)
-                    builds = [x for x in range(builds[u"start"],
-                                               build_end + 1)
-                              if x not in builds.get(u"skip", list())]
+                    builds = [x for x in range(builds[u"start"], build_end + 1)]
+                    if max_builds and max_builds < len(builds):
+                        builds = builds[:max_builds]
                     self.configuration[u"data-sets"][set_name][job] = builds
                 elif isinstance(builds, list):
                     for idx, item in enumerate(builds):
@@ -590,14 +591,23 @@ class Specification:
                 if builds:
                     if isinstance(builds, dict):
                         build_end = builds.get(u"end", None)
+                        max_builds = builds.get(u"max-builds", None)
+                        reverse = bool(builds.get(u"reverse", False))
                         try:
                             build_end = int(build_end)
                         except ValueError:
                             # defined as a range <start, build_type>
+                            if build_end in (u"lastCompletedBuild",
+                                             u"lastSuccessfulBuild"):
+                                reverse = True
                             build_end = self._get_build_number(job, build_end)
                         builds = [x for x in range(builds[u"start"],
                                                    build_end + 1)
                                   if x not in builds.get(u"skip", list())]
+                        if reverse:
+                            builds.reverse()
+                        if max_builds and max_builds < len(builds):
+                            builds = builds[:max_builds]
                     self._specification[u"input"][u"builds"][job] = list()
                     for build in builds:
                         self._specification[u"input"][u"builds"][job]. \
@@ -608,6 +618,7 @@ class Specification:
                         f"No build is defined for the job {job}. Trying to "
                         f"continue without it."
                     )
+
         except KeyError:
             raise PresentationError(u"No data to process.")