aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--resources/tools/presentation/input_data_files.py43
-rw-r--r--resources/tools/presentation/input_data_parser.py38
-rw-r--r--resources/tools/presentation/pal.py2
-rw-r--r--resources/tools/presentation/specification_CPTA.yaml59
-rw-r--r--resources/tools/presentation_new/specification_CPTA.yaml36
5 files changed, 99 insertions, 79 deletions
diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 262c6a0a63..119a3102e0 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -52,46 +52,50 @@ def _download_file(url, file_name, log):
success = False
try:
log.append(("INFO", " Connecting to '{0}' ...".format(url)))
-
response = get(url, stream=True)
code = response.status_code
-
log.append(("INFO", " {0}: {1}".format(code, responses[code])))
if code != codes["OK"]:
- return False
+ url = url.replace("_info", "")
+ log.append(("INFO", " Connecting to '{0}' ...".format(url)))
+ response = get(url, stream=True)
+ code = response.status_code
+ log.append(("INFO", " {0}: {1}".format(code, responses[code])))
+ if code != codes["OK"]:
+ return False, file_name
+ file_name = file_name.replace("_info", "")
log.append(("INFO", " Downloading the file '{0}' to '{1}' ...".
format(url, file_name)))
- file_handle = open(file_name, "wb")
- for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
- if chunk:
- file_handle.write(chunk)
- file_handle.close()
+ with open(file_name, "wb") as file_handle:
+ for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+ if chunk:
+ file_handle.write(chunk)
success = True
except ConnectionError as err:
log.append(("ERROR", "Not possible to connect to '{0}'.".format(url)))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except HTTPError as err:
log.append(("ERROR", "Invalid HTTP response from '{0}'.".format(url)))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except TooManyRedirects as err:
log.append(("ERROR", "Request exceeded the configured number "
"of maximum re-directions."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except Timeout as err:
log.append(("ERROR", "Request timed out."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except RequestException as err:
log.append(("ERROR", "Unexpected HTTP request exception."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
except (IOError, ValueError, KeyError) as err:
log.append(("ERROR", "Download failed."))
- log.append(("DEBUG", str(err)))
+ log.append(("DEBUG", repr(err)))
log.append(("INFO", " Download finished."))
- return success
+ return success, file_name
def _unzip_file(spec, build, pid, log):
@@ -169,8 +173,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
elif job.startswith("intel-dnv-"):
url = spec.environment["urls"]["URL[VIRL,DNV]"].format(release=job[-4:])
else:
- raise PresentationError("No url defined for the job '{}'.".
- format(job))
+ raise PresentationError("No url defined for the job '{}'.".format(job))
file_name = spec.input["file-name"]
full_name = spec.input["download-path"]. \
format(job=job, build=build["build"], filename=file_name)
@@ -182,7 +185,9 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
name=file_name))
# Download the file from the defined source (Jenkins, logs.fd.io):
- success = _download_file(url, new_name, log)
+ success, downloaded_name = _download_file(url, new_name, log)
+ if success:
+ new_name = downloaded_name
if success and new_name.endswith(".zip"):
if not is_zipfile(new_name):
@@ -205,7 +210,7 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
release=rls,
dir=spec.environment["urls"]["DIR[NEXUS]"],
file=nexus_file_name)
- success = _download_file(url, new_name, log)
+ success, new_name = _download_file(url, new_name, log)
if success:
break
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index 31d6d52fd1..84245c429c 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -500,7 +500,7 @@ class ExecutionChecker(ResultVisitor):
:type msg: Message
:returns: Nothing.
"""
- if msg.message.count("return STDOUT Thread "):
+ if msg.message.count("Thread 0 vpp_main"):
self._show_run_lookup_nr += 1
if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
self._data["tests"][self._test_ID]["show-run"] = str()
@@ -987,8 +987,7 @@ class ExecutionChecker(ResultVisitor):
if setup_kw.name.count("Show Vpp Version On All Duts") \
and not self._version:
self._msg_type = "vpp-version"
-
- elif setup_kw.name.count("Setup performance global Variables") \
+ elif setup_kw.name.count("Set Global Variable") \
and not self._timestamp:
self._msg_type = "timestamp"
elif setup_kw.name.count("Setup Framework") and not self._testbed:
@@ -1210,9 +1209,6 @@ class InputData(object):
logs = list()
- logging.info(" Processing the job/build: {0}: {1}".
- format(job, build["build"]))
-
logs.append(("INFO", " Processing the job/build: {0}: {1}".
format(job, build["build"])))
@@ -1232,7 +1228,7 @@ class InputData(object):
"'{build}', or it is damaged. Skipped.".
format(job=job, build=build["build"])))
if success:
- logs.append(("INFO", " Processing data from the build '{0}' ...".
+ logs.append(("INFO", " Processing data from the build '{0}' ...".
format(build["build"])))
data = self._parse_tests(job, build, logs)
if data is None:
@@ -1284,15 +1280,25 @@ class InputData(object):
logs.append(("ERROR",
"Cannot remove the file '{0}': {1}".
format(full_name, repr(err))))
-
logs.append(("INFO", " Done."))
+ for level, line in logs:
+ if level == "INFO":
+ logging.info(line)
+ elif level == "ERROR":
+ logging.error(line)
+ elif level == "DEBUG":
+ logging.debug(line)
+ elif level == "CRITICAL":
+ logging.critical(line)
+ elif level == "WARNING":
+ logging.warning(line)
+
result = {
"data": data,
"state": state,
"job": job,
- "build": build,
- "logs": logs
+ "build": build
}
data_queue.put(result)
@@ -1356,18 +1362,6 @@ class InputData(object):
self._cfg.set_input_state(job, build_nr, result["state"])
- for item in result["logs"]:
- if item[0] == "INFO":
- logging.info(item[1])
- elif item[0] == "ERROR":
- logging.error(item[1])
- elif item[0] == "DEBUG":
- logging.debug(item[1])
- elif item[0] == "CRITICAL":
- logging.critical(item[1])
- elif item[0] == "WARNING":
- logging.warning(item[1])
-
del data_queue
# Terminate all workers
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 39cba09d96..dfed6b3e62 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -101,7 +101,7 @@ def main():
prepare_static_content(spec)
data = InputData(spec)
- data.download_and_parse_data(repeat=2)
+ data.download_and_parse_data(repeat=1)
generate_tables(spec, data)
generate_plots(spec, data)
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index ada861d790..1d0778d227 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -169,7 +169,7 @@
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 510
+ start: 515
end: "lastCompletedBuild"
skip:
- 593
@@ -180,12 +180,12 @@
- 616
- 617
csit-dpdk-perf-mrr-weekly-master:
- start: 55
+ start: 57
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 510
+ start: 515
end: "lastCompletedBuild"
skip:
- 593
@@ -198,13 +198,13 @@
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
- start: 55
+ start: 57
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 260
+ start: 265
end: "lastCompletedBuild"
skip:
- 356
@@ -230,14 +230,14 @@
- 377
- 378
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 20
+ start: 22
end: "lastCompletedBuild"
skip:
- 31
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 260
+ start: 265
end: "lastCompletedBuild"
skip:
- 356
@@ -265,7 +265,7 @@
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 20
+ start: 22
end: "lastCompletedBuild"
skip:
- 31
@@ -273,9 +273,12 @@
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 260
+ start: 265
end: "lastCompletedBuild"
skip:
+ - 288
+ - 289
+ - 290
- 347
- 358
- 359
@@ -292,15 +295,22 @@
- 380
- 381
- 396
+ - 398
+ - 403
+ - 404
+ - 405
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 20
+ start: 22
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 260
+ start: 265
end: "lastCompletedBuild"
skip:
+ - 288
+ - 289
+ - 290
- 347
- 358
- 359
@@ -317,10 +327,14 @@
- 380
- 381
- 396
+ - 398
+ - 403
+ - 404
+ - 405
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 20
+ start: 22
end: "lastCompletedBuild"
plot-performance-trending-vpp-nfv-2n-skx:
@@ -409,7 +423,7 @@
-
type: "input"
general:
- file-name: "output.xml.gz"
+ file-name: "output_info.xml.gz"
file-format: ".gz"
download-path: "{job}/{build}/archives/{filename}"
extract: "output.xml"
@@ -421,7 +435,7 @@
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 510
+ start: 515
end: "lastCompletedBuild"
skip:
- 593
@@ -432,12 +446,12 @@
- 616
- 617
csit-dpdk-perf-mrr-weekly-master:
- start: 55
+ start: 57
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 260
+ start: 265
end: "lastCompletedBuild"
skip:
- 356
@@ -463,16 +477,19 @@
- 377
- 378
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 20
+ start: 22
end: "lastCompletedBuild"
skip:
- 31
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 260
+ start: 265
end: "lastCompletedBuild"
skip:
+ - 288
+ - 289
+ - 290
- 347
- 358
- 359
@@ -489,6 +506,10 @@
- 380
- 381
- 396
+ - 398
+ - 403
+ - 404
+ - 405
csit-vpp-perf-mrr-weekly-master-2n-skx:
start: 1
end: "lastCompletedBuild"
@@ -496,7 +517,7 @@
- 3
- 5
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 20
+ start: 22
end: "lastCompletedBuild"
-
diff --git a/resources/tools/presentation_new/specification_CPTA.yaml b/resources/tools/presentation_new/specification_CPTA.yaml
index 4412a7f786..7bcbe7ef80 100644
--- a/resources/tools/presentation_new/specification_CPTA.yaml
+++ b/resources/tools/presentation_new/specification_CPTA.yaml
@@ -84,58 +84,58 @@
# 3n-hsw
plot-performance-trending-all-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 510
+ start: 530
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 54
+ start: 57
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
csit-vpp-perf-mrr-daily-master:
- start: 510
+ start: 530
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
- start: 54
+ start: 57
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 260
+ start: 280
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 19
+ start: 22
end: 30 # "lastCompletedBuild"
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 260
+ start: 280
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 19
+ start: 22
end: 30 # "lastCompletedBuild"
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 260
+ start: 280
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 19
+ start: 22
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 260
+ start: 280
end: "lastCompletedBuild"
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 19
+ start: 22
end: "lastCompletedBuild"
plot-layouts:
@@ -228,26 +228,26 @@
# 3n-hsw
csit-vpp-perf-mrr-daily-master:
- start: 510
+ start: 530
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 54
+ start: 57
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 260
+ start: 280
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 19
+ start: 22
end: 30 # "lastCompletedBuild"
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 260
+ start: 280
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 19
+ start: 22
end: "lastCompletedBuild"
-