CSIT-1483: Add processing of output_info.xml as the first choice

Change-Id: I27687ed6dda3a8717538653e75ad0d61c6516c7e
Signed-off-by: Tibor Frank <tifrank@cisco.com>

5 files changed, 99 insertions, 79 deletions

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 262c6a0a63..119a3102e0 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -52,46 +52,50 @@ def _download_file(url, file_name, log):
     success = False
     try:
         log.append(("INFO", "    Connecting to '{0}' ...".format(url)))
-
         response = get(url, stream=True)
         code = response.status_code
-
         log.append(("INFO", "    {0}: {1}".format(code, responses[code])))
 
         if code != codes["OK"]:
-            return False
+            url = url.replace("_info", "")
+            log.append(("INFO", "    Connecting to '{0}' ...".format(url)))
+            response = get(url, stream=True)
+            code = response.status_code
+            log.append(("INFO", "    {0}: {1}".format(code, responses[code])))
+            if code != codes["OK"]:
+                return False, file_name
+            file_name = file_name.replace("_info", "")
 
         log.append(("INFO", "    Downloading the file '{0}' to '{1}' ...".
                     format(url, file_name)))
 
-        file_handle = open(file_name, "wb")
-        for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
-            if chunk:
-                file_handle.write(chunk)
-        file_handle.close()
+        with open(file_name, "wb") as file_handle:
+            for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+                if chunk:
+                    file_handle.write(chunk)
         success = True
     except ConnectionError as err:
         log.append(("ERROR", "Not possible to connect to '{0}'.".format(url)))
-        log.append(("DEBUG", str(err)))
+        log.append(("DEBUG", repr(err)))
     except HTTPError as err:
         log.append(("ERROR", "Invalid HTTP response from '{0}'.".format(url)))
-        log.append(("DEBUG", str(err)))
+        log.append(("DEBUG", repr(err)))
     except TooManyRedirects as err:
         log.append(("ERROR", "Request exceeded the configured number "
                              "of maximum re-directions."))
-        log.append(("DEBUG", str(err)))
+        log.append(("DEBUG", repr(err)))
     except Timeout as err:
         log.append(("ERROR", "Request timed out."))
-        log.append(("DEBUG", str(err)))
+        log.append(("DEBUG", repr(err)))
     except RequestException as err:
         log.append(("ERROR", "Unexpected HTTP request exception."))
-        log.append(("DEBUG", str(err)))
+        log.append(("DEBUG", repr(err)))
     except (IOError, ValueError, KeyError) as err:
         log.append(("ERROR", "Download failed."))
-        log.append(("DEBUG", str(err)))
+        log.append(("DEBUG", repr(err)))
 
     log.append(("INFO", "    Download finished."))
-    return success
+    return success, file_name
 
 
 def _unzip_file(spec, build, pid, log):
@@ -169,8 +173,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
     elif job.startswith("intel-dnv-"):
         url = spec.environment["urls"]["URL[VIRL,DNV]"].format(release=job[-4:])
     else:
-        raise PresentationError("No url defined for the job '{}'.".
-                                format(job))
+        raise PresentationError("No url defined for the job '{}'.".format(job))
     file_name = spec.input["file-name"]
     full_name = spec.input["download-path"]. \
         format(job=job, build=build["build"], filename=file_name)
@@ -182,7 +185,9 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
                            name=file_name))
 
     # Download the file from the defined source (Jenkins, logs.fd.io):
-    success = _download_file(url, new_name, log)
+    success, downloaded_name = _download_file(url, new_name, log)
+    if success:
+        new_name = downloaded_name
 
     if success and new_name.endswith(".zip"):
         if not is_zipfile(new_name):
@@ -205,7 +210,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
                        release=rls,
                        dir=spec.environment["urls"]["DIR[NEXUS]"],
                        file=nexus_file_name)
-            success = _download_file(url, new_name, log)
+            success, new_name = _download_file(url, new_name, log)
             if success:
                 break
 
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 31d6d52fd1..84245c429c 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -500,7 +500,7 @@ class ExecutionChecker(ResultVisitor):
         :type msg: Message
         :returns: Nothing.
         """
-        if msg.message.count("return STDOUT Thread "):
+        if msg.message.count("Thread 0 vpp_main"):
             self._show_run_lookup_nr += 1
             if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
                 self._data["tests"][self._test_ID]["show-run"] = str()
@@ -987,8 +987,7 @@ class ExecutionChecker(ResultVisitor):
         if setup_kw.name.count("Show Vpp Version On All Duts") \
                 and not self._version:
             self._msg_type = "vpp-version"
-
-        elif setup_kw.name.count("Setup performance global Variables") \
+        elif setup_kw.name.count("Set Global Variable") \
                 and not self._timestamp:
             self._msg_type = "timestamp"
         elif setup_kw.name.count("Setup Framework") and not self._testbed:
@@ -1210,9 +1209,6 @@ class InputData(object):
 
         logs = list()
 
-        logging.info("  Processing the job/build: {0}: {1}".
-                     format(job, build["build"]))
-
         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
                      format(job, build["build"])))
 
@@ -1232,7 +1228,7 @@ class InputData(object):
                                   "'{build}', or it is damaged. Skipped.".
                          format(job=job, build=build["build"])))
         if success:
-            logs.append(("INFO", "  Processing data from the build '{0}' ...".
+            logs.append(("INFO", "    Processing data from the build '{0}' ...".
                          format(build["build"])))
             data = self._parse_tests(job, build, logs)
             if data is None:
@@ -1284,15 +1280,25 @@ class InputData(object):
                             logs.append(("ERROR",
                                         "Cannot remove the file '{0}': {1}".
                                         format(full_name, repr(err))))
-
         logs.append(("INFO", "  Done."))
 
+        for level, line in logs:
+            if level == "INFO":
+                logging.info(line)
+            elif level == "ERROR":
+                logging.error(line)
+            elif level == "DEBUG":
+                logging.debug(line)
+            elif level == "CRITICAL":
+                logging.critical(line)
+            elif level == "WARNING":
+                logging.warning(line)
+
         result = {
             "data": data,
             "state": state,
             "job": job,
-            "build": build,
-            "logs": logs
+            "build": build
         }
         data_queue.put(result)
 
@@ -1356,18 +1362,6 @@ class InputData(object):
 
             self._cfg.set_input_state(job, build_nr, result["state"])
 
-            for item in result["logs"]:
-                if item[0] == "INFO":
-                    logging.info(item[1])
-                elif item[0] == "ERROR":
-                    logging.error(item[1])
-                elif item[0] == "DEBUG":
-                    logging.debug(item[1])
-                elif item[0] == "CRITICAL":
-                    logging.critical(item[1])
-                elif item[0] == "WARNING":
-                    logging.warning(item[1])
-
         del data_queue
 
         # Terminate all workers
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 39cba09d96..dfed6b3e62 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -101,7 +101,7 @@ def main():
         prepare_static_content(spec)
 
         data = InputData(spec)
-        data.download_and_parse_data(repeat=2)
+        data.download_and_parse_data(repeat=1)
 
         generate_tables(spec, data)
         generate_plots(spec, data)
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index ada861d790..1d0778d227 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -169,7 +169,7 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 510
+        start: 515
         end: "lastCompletedBuild"
         skip:
           - 593
@@ -180,12 +180,12 @@
           - 616
           - 617
       csit-dpdk-perf-mrr-weekly-master:
-        start: 55
+        start: 57
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 510
+        start: 515
         end: "lastCompletedBuild"
         skip:
           - 593
@@ -198,13 +198,13 @@
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
-        start: 55
+        start: 57
         end: "lastCompletedBuild"
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 260
+        start: 265
         end: "lastCompletedBuild"
         skip:
           - 356
@@ -230,14 +230,14 @@
           - 377
           - 378
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 20
+        start: 22
         end: "lastCompletedBuild"
         skip:
           - 31
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 260
+        start: 265
         end: "lastCompletedBuild"
         skip:
           - 356
@@ -265,7 +265,7 @@
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 20
+        start: 22
         end: "lastCompletedBuild"
         skip:
           - 31
@@ -273,9 +273,12 @@
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 260
+        start: 265
         end: "lastCompletedBuild"
         skip:
+          - 288
+          - 289
+          - 290
           - 347
           - 358
           - 359
@@ -292,15 +295,22 @@
           - 380
           - 381
           - 396
+          - 398
+          - 403
+          - 404
+          - 405
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 20
+        start: 22
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 260
+        start: 265
         end: "lastCompletedBuild"
         skip:
+          - 288
+          - 289
+          - 290
           - 347
           - 358
           - 359
@@ -317,10 +327,14 @@
           - 380
           - 381
           - 396
+          - 398
+          - 403
+          - 404
+          - 405
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 20
+        start: 22
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-nfv-2n-skx:
@@ -409,7 +423,7 @@
 -
   type: "input"
   general:
-    file-name: "output.xml.gz"
+    file-name: "output_info.xml.gz"
     file-format: ".gz"
     download-path: "{job}/{build}/archives/{filename}"
     extract: "output.xml"
@@ -421,7 +435,7 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 510
+      start: 515
       end: "lastCompletedBuild"
       skip:
         - 593
@@ -432,12 +446,12 @@
         - 616
         - 617
     csit-dpdk-perf-mrr-weekly-master:
-      start: 55
+      start: 57
       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 260
+      start: 265
       end: "lastCompletedBuild"
       skip:
         - 356
@@ -463,16 +477,19 @@
         - 377
         - 378
     csit-dpdk-perf-mrr-weekly-master-3n-skx:
-      start: 20
+      start: 22
       end: "lastCompletedBuild"
       skip:
         - 31
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 260
+      start: 265
       end: "lastCompletedBuild"
       skip:
+        - 288
+        - 289
+        - 290
         - 347
         - 358
         - 359
@@ -489,6 +506,10 @@
         - 380
         - 381
         - 396
+        - 398
+        - 403
+        - 404
+        - 405
     csit-vpp-perf-mrr-weekly-master-2n-skx:
       start: 1
       end: "lastCompletedBuild"
@@ -496,7 +517,7 @@
         - 3
         - 5
     csit-dpdk-perf-mrr-weekly-master-2n-skx:
-      start: 20
+      start: 22
       end: "lastCompletedBuild"
 
 -
diff --git a/resources/tools/presentation_new/specification_CPTA.yaml b/resources/tools/presentation_new/specification_CPTA.yaml
index 4412a7f786..7bcbe7ef80 100644
--- a/resources/tools/presentation_new/specification_CPTA.yaml
+++ b/resources/tools/presentation_new/specification_CPTA.yaml
@@ -84,58 +84,58 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 510
+        start: 530
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master:
-        start: 54
+        start: 57
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 510
+        start: 530
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-hsw:
       csit-dpdk-perf-mrr-weekly-master:
-        start: 54
+        start: 57
         end: "lastCompletedBuild"
 
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 260
+        start: 280
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 19
+        start: 22
         end: 30 # "lastCompletedBuild"
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 260
+        start: 280
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-3n-skx:
       csit-dpdk-perf-mrr-weekly-master-3n-skx:
-        start: 19
+        start: 22
         end: 30 # "lastCompletedBuild"
 
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 260
+        start: 280
         end: "lastCompletedBuild"
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 19
+        start: 22
         end: "lastCompletedBuild"
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 260
+        start: 280
         end: "lastCompletedBuild"
 
     plot-performance-trending-dpdk-2n-skx:
       csit-dpdk-perf-mrr-weekly-master-2n-skx:
-        start: 19
+        start: 22
         end: "lastCompletedBuild"
 
   plot-layouts:
@@ -228,26 +228,26 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 510
+      start: 530
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master:
-      start: 54
+      start: 57
       end: "lastCompletedBuild"
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 260
+      start: 280
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-3n-skx:
-      start: 19
+      start: 22
       end: 30 # "lastCompletedBuild"
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 260
+      start: 280
       end: "lastCompletedBuild"
     csit-dpdk-perf-mrr-weekly-master-2n-skx:
-      start: 19
+      start: 22
       end: "lastCompletedBuild"
 
 -