diff options
Diffstat (limited to 'resources/tools/presentation')
| -rw-r--r-- | resources/tools/presentation/input_data_files.py | 43 | ||||
| -rw-r--r-- | resources/tools/presentation/input_data_parser.py | 38 | ||||
| -rw-r--r-- | resources/tools/presentation/pal.py | 2 | ||||
| -rw-r--r-- | resources/tools/presentation/specification_CPTA.yaml | 59 |
4 files changed, 81 insertions, 61 deletions
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index 262c6a0a63..119a3102e0 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -52,46 +52,50 @@ def _download_file(url, file_name, log): success = False try: log.append(("INFO", " Connecting to '{0}' ...".format(url))) - response = get(url, stream=True) code = response.status_code - log.append(("INFO", " {0}: {1}".format(code, responses[code]))) if code != codes["OK"]: - return False + url = url.replace("_info", "") + log.append(("INFO", " Connecting to '{0}' ...".format(url))) + response = get(url, stream=True) + code = response.status_code + log.append(("INFO", " {0}: {1}".format(code, responses[code]))) + if code != codes["OK"]: + return False, file_name + file_name = file_name.replace("_info", "") log.append(("INFO", " Downloading the file '{0}' to '{1}' ...". format(url, file_name))) - file_handle = open(file_name, "wb") - for chunk in response.iter_content(chunk_size=CHUNK_SIZE): - if chunk: - file_handle.write(chunk) - file_handle.close() + with open(file_name, "wb") as file_handle: + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + file_handle.write(chunk) success = True except ConnectionError as err: log.append(("ERROR", "Not possible to connect to '{0}'.".format(url))) - log.append(("DEBUG", str(err))) + log.append(("DEBUG", repr(err))) except HTTPError as err: log.append(("ERROR", "Invalid HTTP response from '{0}'.".format(url))) - log.append(("DEBUG", str(err))) + log.append(("DEBUG", repr(err))) except TooManyRedirects as err: log.append(("ERROR", "Request exceeded the configured number " "of maximum re-directions.")) - log.append(("DEBUG", str(err))) + log.append(("DEBUG", repr(err))) except Timeout as err: log.append(("ERROR", "Request timed out.")) - log.append(("DEBUG", str(err))) + log.append(("DEBUG", repr(err))) except RequestException as err: log.append(("ERROR", "Unexpected HTTP request exception.")) - log.append(("DEBUG", str(err))) + log.append(("DEBUG", repr(err))) except (IOError, ValueError, KeyError) as err: log.append(("ERROR", "Download failed.")) - log.append(("DEBUG", str(err))) + log.append(("DEBUG", repr(err))) log.append(("INFO", " Download finished.")) - return success + return success, file_name def _unzip_file(spec, build, pid, log): @@ -169,8 +173,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log): elif job.startswith("intel-dnv-"): url = spec.environment["urls"]["URL[VIRL,DNV]"].format(release=job[-4:]) else: - raise PresentationError("No url defined for the job '{}'.". - format(job)) + raise PresentationError("No url defined for the job '{}'.".format(job)) file_name = spec.input["file-name"] full_name = spec.input["download-path"]. \ format(job=job, build=build["build"], filename=file_name) @@ -182,7 +185,9 @@ def download_and_unzip_data_file(spec, job, build, pid, log): name=file_name)) # Download the file from the defined source (Jenkins, logs.fd.io): - success = _download_file(url, new_name, log) + success, downloaded_name = _download_file(url, new_name, log) + if success: + new_name = downloaded_name if success and new_name.endswith(".zip"): if not is_zipfile(new_name): @@ -205,7 +210,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log): release=rls, dir=spec.environment["urls"]["DIR[NEXUS]"], file=nexus_file_name) - success = _download_file(url, new_name, log) + success, new_name = _download_file(url, new_name, log) if success: break diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 31d6d52fd1..84245c429c 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -500,7 +500,7 @@ class ExecutionChecker(ResultVisitor): :type msg: Message :returns: Nothing. """ - if msg.message.count("return STDOUT Thread "): + if msg.message.count("Thread 0 vpp_main"): self._show_run_lookup_nr += 1 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1: self._data["tests"][self._test_ID]["show-run"] = str() @@ -987,8 +987,7 @@ class ExecutionChecker(ResultVisitor): if setup_kw.name.count("Show Vpp Version On All Duts") \ and not self._version: self._msg_type = "vpp-version" - - elif setup_kw.name.count("Setup performance global Variables") \ + elif setup_kw.name.count("Set Global Variable") \ and not self._timestamp: self._msg_type = "timestamp" elif setup_kw.name.count("Setup Framework") and not self._testbed: @@ -1210,9 +1209,6 @@ class InputData(object): logs = list() - logging.info(" Processing the job/build: {0}: {1}". - format(job, build["build"])) - logs.append(("INFO", " Processing the job/build: {0}: {1}". format(job, build["build"]))) @@ -1232,7 +1228,7 @@ class InputData(object): "'{build}', or it is damaged. Skipped.". format(job=job, build=build["build"]))) if success: - logs.append(("INFO", " Processing data from the build '{0}' ...". + logs.append(("INFO", " Processing data from the build '{0}' ...". format(build["build"]))) data = self._parse_tests(job, build, logs) if data is None: @@ -1284,15 +1280,25 @@ class InputData(object): logs.append(("ERROR", "Cannot remove the file '{0}': {1}". format(full_name, repr(err)))) - logs.append(("INFO", " Done.")) + for level, line in logs: + if level == "INFO": + logging.info(line) + elif level == "ERROR": + logging.error(line) + elif level == "DEBUG": + logging.debug(line) + elif level == "CRITICAL": + logging.critical(line) + elif level == "WARNING": + logging.warning(line) + result = { "data": data, "state": state, "job": job, - "build": build, - "logs": logs + "build": build } data_queue.put(result) @@ -1356,18 +1362,6 @@ class InputData(object): self._cfg.set_input_state(job, build_nr, result["state"]) - for item in result["logs"]: - if item[0] == "INFO": - logging.info(item[1]) - elif item[0] == "ERROR": - logging.error(item[1]) - elif item[0] == "DEBUG": - logging.debug(item[1]) - elif item[0] == "CRITICAL": - logging.critical(item[1]) - elif item[0] == "WARNING": - logging.warning(item[1]) - del data_queue # Terminate all workers diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py index 39cba09d96..dfed6b3e62 100644 --- a/resources/tools/presentation/pal.py +++ b/resources/tools/presentation/pal.py @@ -101,7 +101,7 @@ def main(): prepare_static_content(spec) data = InputData(spec) - data.download_and_parse_data(repeat=2) + data.download_and_parse_data(repeat=1) generate_tables(spec, data) generate_plots(spec, data) diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index ada861d790..1d0778d227 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -169,7 +169,7 @@ # 3n-hsw plot-performance-trending-all-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 510 + start: 515 end: "lastCompletedBuild" skip: - 593 @@ -180,12 +180,12 @@ - 616 - 617 csit-dpdk-perf-mrr-weekly-master: - start: 55 + start: 57 end: "lastCompletedBuild" plot-performance-trending-vpp-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 510 + start: 515 end: "lastCompletedBuild" skip: - 593 @@ -198,13 +198,13 @@ plot-performance-trending-dpdk-3n-hsw: csit-dpdk-perf-mrr-weekly-master: - start: 55 + start: 57 end: "lastCompletedBuild" # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 260 + start: 265 end: "lastCompletedBuild" skip: - 356 @@ -230,14 +230,14 @@ - 377 - 378 csit-dpdk-perf-mrr-weekly-master-3n-skx: - start: 20 + start: 22 end: "lastCompletedBuild" skip: - 31 plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 260 + start: 265 end: "lastCompletedBuild" skip: - 356 @@ -265,7 +265,7 @@ plot-performance-trending-dpdk-3n-skx: csit-dpdk-perf-mrr-weekly-master-3n-skx: - start: 20 + start: 22 end: "lastCompletedBuild" skip: - 31 @@ -273,9 +273,12 @@ # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 260 + start: 265 end: "lastCompletedBuild" skip: + - 288 + - 289 + - 290 - 347 - 358 - 359 @@ -292,15 +295,22 @@ - 380 - 381 - 396 + - 398 + - 403 + - 404 + - 405 csit-dpdk-perf-mrr-weekly-master-2n-skx: - start: 20 + start: 22 end: "lastCompletedBuild" plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 260 + start: 265 end: "lastCompletedBuild" skip: + - 288 + - 289 + - 290 - 347 - 358 - 359 @@ -317,10 +327,14 @@ - 380 - 381 - 396 + - 398 + - 403 + - 404 + - 405 plot-performance-trending-dpdk-2n-skx: csit-dpdk-perf-mrr-weekly-master-2n-skx: - start: 20 + start: 22 end: "lastCompletedBuild" plot-performance-trending-vpp-nfv-2n-skx: @@ -409,7 +423,7 @@ - type: "input" general: - file-name: "output.xml.gz" + file-name: "output_info.xml.gz" file-format: ".gz" download-path: "{job}/{build}/archives/{filename}" extract: "output.xml" @@ -421,7 +435,7 @@ # 3n-hsw csit-vpp-perf-mrr-daily-master: - start: 510 + start: 515 end: "lastCompletedBuild" skip: - 593 @@ -432,12 +446,12 @@ - 616 - 617 csit-dpdk-perf-mrr-weekly-master: - start: 55 + start: 57 end: "lastCompletedBuild" # 3n-skx csit-vpp-perf-mrr-daily-master-3n-skx: - start: 260 + start: 265 end: "lastCompletedBuild" skip: - 356 @@ -463,16 +477,19 @@ - 377 - 378 csit-dpdk-perf-mrr-weekly-master-3n-skx: - start: 20 + start: 22 end: "lastCompletedBuild" skip: - 31 # 2n-skx csit-vpp-perf-mrr-daily-master-2n-skx: - start: 260 + start: 265 end: "lastCompletedBuild" skip: + - 288 + - 289 + - 290 - 347 - 358 - 359 @@ -489,6 +506,10 @@ - 380 - 381 - 396 + - 398 + - 403 + - 404 + - 405 csit-vpp-perf-mrr-weekly-master-2n-skx: start: 1 end: "lastCompletedBuild" @@ -496,7 +517,7 @@ - 3 - 5 csit-dpdk-perf-mrr-weekly-master-2n-skx: - start: 20 + start: 22 end: "lastCompletedBuild" - |