aboutsummaryrefslogtreecommitdiffstats
path: root/resources
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2019-01-03 10:30:30 +0100
committerTibor Frank <tifrank@cisco.com>2019-01-04 15:09:41 +0000
commit6c3d971c8b529d094f7c7355e685ea5ab63904e5 (patch)
tree51f8a5f4ed3e973d8facd8b8866c03ff2b31bd0b /resources
parentcebbe0a29e2c6329c54d09a77a64e874703c7d4f (diff)
Trending: Remove unneeded tests from Dashboard
Change-Id: I0f57735bcc578b42994721ca88dc1bb60af475de Signed-off-by: Tibor Frank <tifrank@cisco.com>
Diffstat (limited to 'resources')
-rw-r--r--resources/tools/presentation/generator_CPTA.py45
-rw-r--r--resources/tools/presentation/generator_tables.py18
-rw-r--r--resources/tools/presentation/input_data_parser.py37
-rw-r--r--resources/tools/presentation/specification_CPTA.yaml50
-rw-r--r--resources/tools/presentation/specification_parser.py6
5 files changed, 104 insertions, 52 deletions
diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 7d5af3fe2d..f9d42ba069 100644
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -174,28 +174,29 @@ def _generate_trending_traces(in_data, job_name, build_info,
xaxis = list()
for idx in data_x:
date = build_info[job_name][str(idx)][0]
- hover_str = ("date: {0}<br>"
- "value: {1:,}<br>"
- "{2}-ref: {3}<br>"
- "csit-ref: mrr-{4}-build-{5}")
+ hover_str = ("date: {date}<br>"
+ "value: {value:,}<br>"
+ "{sut}-ref: {build}<br>"
+ "csit-ref: mrr-{period}-build-{build_nr}<br>"
+ "testbed: {testbed}")
if "dpdk" in job_name:
hover_text.append(hover_str.format(
- date,
- int(in_data[idx].avg),
- "dpdk",
- build_info[job_name][str(idx)][1].
- rsplit('~', 1)[0],
- "weekly",
- idx))
+ date=date,
+ value=int(in_data[idx].avg),
+ sut="dpdk",
+ build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
+ period="weekly",
+ build_nr=idx,
+ testbed=build_info[job_name][str(idx)][2]))
elif "vpp" in job_name:
hover_text.append(hover_str.format(
- date,
- int(in_data[idx].avg),
- "vpp",
- build_info[job_name][str(idx)][1].
- rsplit('~', 1)[0],
- "daily",
- idx))
+ date=date,
+ value=int(in_data[idx].avg),
+ sut="vpp",
+ build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
+ period="daily",
+ build_nr=idx,
+ testbed=build_info[job_name][str(idx)][2]))
xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
int(date[9:11]), int(date[12:])))
@@ -433,13 +434,19 @@ def _generate_all_charts(spec, input_data):
# Create "build ID": "date" dict:
build_info = dict()
+ tb_tbl = spec.environment.get("testbeds", None)
for job_name, job_data in builds_dict.items():
if build_info.get(job_name, None) is None:
build_info[job_name] = OrderedDict()
for build in job_data:
+ testbed = ""
+ tb_ip = input_data.metadata(job_name, build).get("testbed", "")
+ if tb_ip and tb_tbl:
+ testbed = tb_tbl.get(tb_ip, "")
build_info[job_name][build] = (
input_data.metadata(job_name, build).get("generated", ""),
- input_data.metadata(job_name, build).get("version", "")
+ input_data.metadata(job_name, build).get("version", ""),
+ testbed
)
work_queue = multiprocessing.JoinableQueue()
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 7590daa8fe..3ad1e3c756 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -497,12 +497,15 @@ def table_performance_trending_dashboard(table, input_data):
if classification_lst:
if isnan(rel_change_last) and isnan(rel_change_long):
continue
+ if (isnan(last_avg) or
+ isnan(rel_change_last) or
+ isnan(rel_change_long)):
+ continue
tbl_lst.append(
[tbl_dict[tst_name]["name"],
- '-' if isnan(last_avg) else
round(last_avg / 1000000, 2),
- '-' if isnan(rel_change_last) else rel_change_last,
- '-' if isnan(rel_change_long) else rel_change_long,
+ rel_change_last,
+ rel_change_long,
classification_lst[-win_size:].count("regression"),
classification_lst[-win_size:].count("progression")])
@@ -798,9 +801,11 @@ def table_failed_tests(table, input_data):
generated,
input_data.metadata(job, build).get("version", ""),
build)
- except (TypeError, KeyError):
- pass # No data in output.xml for this test
+ except (TypeError, KeyError) as err:
+ logging.warning("tst_name: {} - err: {}".
+ format(tst_name, repr(err)))
+ max_fails = 0
tbl_lst = list()
for tst_data in tbl_dict.values():
fails_nr = 0
@@ -811,6 +816,7 @@ def table_failed_tests(table, input_data):
fails_last_vpp = val[2]
fails_last_csit = val[3]
if fails_nr:
+ max_fails = fails_nr if fails_nr > max_fails else max_fails
tbl_lst.append([tst_data["name"],
fails_nr,
fails_last_date,
@@ -819,7 +825,7 @@ def table_failed_tests(table, input_data):
tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
tbl_sorted = list()
- for nrf in range(table["window"], -1, -1):
+ for nrf in range(max_fails, -1, -1):
tbl_fails = [item for item in tbl_lst if item[1] == nrf]
tbl_sorted.extend(tbl_fails)
file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py
index fe6213628b..703ea33ba1 100644
--- a/resources/tools/presentation/input_data_parser.py
+++ b/resources/tools/presentation/input_data_parser.py
@@ -33,6 +33,7 @@ from os import remove
from os.path import join
from datetime import datetime as dt
from datetime import timedelta
+from json import loads
from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
from input_data_files import download_and_unzip_data_file
@@ -275,7 +276,8 @@ class ExecutionChecker(ResultVisitor):
REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
r'[\D\d]*')
- REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
+ REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
+ r"VPP Version:\s*)(.*)")
REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
r"(RTE Version: 'DPDK )(.*)(')")
@@ -318,6 +320,9 @@ class ExecutionChecker(ResultVisitor):
# Timestamp
self._timestamp = None
+ # Testbed. The testbed is identified by TG node IP address.
+ self._testbed = None
+
# Mapping of TCs long names
self._mapping = mapping
@@ -358,7 +363,8 @@ class ExecutionChecker(ResultVisitor):
"vpp-version": self._get_vpp_version,
"dpdk-version": self._get_dpdk_version,
"teardown-vat-history": self._get_vat_history,
- "test-show-runtime": self._get_show_run
+ "test-show-runtime": self._get_show_run,
+ "testbed": self._get_testbed
}
@property
@@ -370,6 +376,28 @@ class ExecutionChecker(ResultVisitor):
"""
return self._data
+ def _get_testbed(self, msg):
+ """Called when extraction of testbed IP is required.
+ The testbed is identified by TG node IP address.
+
+ :param msg: Message to process.
+ :type msg: Message
+ :returns: Nothing.
+ """
+
+ if msg.message.count("Arguments:"):
+ message = str(msg.message).replace(' ', '').replace('\n', '').\
+ replace("'", '"').replace('b"', '"').\
+ replace("honeycom", "honeycomb")
+ message = loads(message[11:-1])
+ try:
+ self._testbed = message["TG"]["host"]
+ except (KeyError, ValueError):
+ pass
+ finally:
+ self._data["metadata"]["testbed"] = self._testbed
+ self._msg_type = None
+
def _get_vpp_version(self, msg):
"""Called when extraction of VPP version is required.
@@ -378,7 +406,8 @@ class ExecutionChecker(ResultVisitor):
:returns: Nothing.
"""
- if msg.message.count("return STDOUT Version:"):
+ if msg.message.count("return STDOUT Version:") or \
+ msg.message.count("VPP Version:"):
self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
group(2))
self._data["metadata"]["version"] = self._version
@@ -901,6 +930,8 @@ class ExecutionChecker(ResultVisitor):
elif setup_kw.name.count("Setup performance global Variables") \
and not self._timestamp:
self._msg_type = "timestamp"
+ elif setup_kw.name.count("Setup Framework") and not self._testbed:
+ self._msg_type = "testbed"
else:
return
setup_kw.messages.visit(self)
diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 45ea85c12c..1368bb01d4 100644
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -61,6 +61,17 @@
# All directories MUST be defined in "paths" section.
- "DIR[BUILD,HTML]"
+ testbeds:
+ "10.30.51.45": "LF-2n-SKX-21"
+ "10.30.51.53": "LF-2n-SKX-22"
+ "10.30.51.55": "LF-2n-SKX-23"
+ "10.30.51.57": "LF-2n-SKX-24"
+ "10.30.51.16": "LF-3n-HSW-01"
+ "10.30.51.20": "LF-3n-HSW-02"
+ "10.30.51.24": "LF-3n-HSW-03"
+ "10.30.51.48": "LF-3n-SKX-31"
+ "10.30.51.60": "LF-3n-SKX-32"
+
-
type: "configuration"
@@ -143,7 +154,7 @@
start: 300
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 50
+ start: 35
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-hsw:
@@ -153,45 +164,45 @@
plot-performance-trending-dpdk-3n-hsw:
csit-dpdk-perf-mrr-weekly-master:
- start: 50
+ start: 35
end: "lastCompletedBuild"
# 3n-skx
plot-performance-trending-all-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 3
+ start: 7
end: "lastCompletedBuild"
plot-performance-trending-vpp-3n-skx:
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
plot-performance-trending-dpdk-3n-skx:
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 3
+ start: 7
end: "lastCompletedBuild"
# 2n-skx
plot-performance-trending-all-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 3
+ start: 7
end: "lastCompletedBuild"
plot-performance-trending-vpp-2n-skx:
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
plot-performance-trending-dpdk-2n-skx:
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 3
+ start: 7
end: "lastCompletedBuild"
plot-layouts:
@@ -285,23 +296,23 @@
start: 300
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master:
- start: 50
+ start: 35
end: "lastCompletedBuild"
# 3n-skx
csit-vpp-perf-mrr-daily-master-3n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-3n-skx:
- start: 3
+ start: 7
end: "lastCompletedBuild"
# 2n-skx
csit-vpp-perf-mrr-daily-master-2n-skx:
- start: 50
+ start: 100
end: "lastCompletedBuild"
csit-dpdk-perf-mrr-weekly-master-2n-skx:
- start: 3
+ start: 7
end: "lastCompletedBuild"
-
@@ -336,7 +347,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-1t1c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -357,7 +367,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-2t2c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -378,7 +387,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-4t4c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -451,7 +459,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -472,7 +479,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -493,7 +499,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -566,7 +571,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -587,7 +591,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
@@ -608,7 +611,6 @@
- "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc"
outlier-const: 1.5
window: 14
- evaluated-window: 14
long-trend-window: 180
-
diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py
index 83838d8212..c149e3e51c 100644
--- a/resources/tools/presentation/specification_parser.py
+++ b/resources/tools/presentation/specification_parser.py
@@ -392,6 +392,12 @@ class Specification(object):
except KeyError:
self._specification["environment"]["build-dirs"] = None
+ try:
+ self._specification["environment"]["testbeds"] = \
+ self._cfg_yaml[idx]["testbeds"]
+ except KeyError:
+ self._specification["environment"]["testbeds"] = None
+
logging.info("Done.")
def _parse_configuration(self):