diff options
6 files changed, 108 insertions, 56 deletions
diff --git a/docs/report/dpdk_performance_tests/comparisons/current_vs_previous_release.rst b/docs/report/dpdk_performance_tests/comparisons/current_vs_previous_release.rst index a6e44480d7..e345a43700 100644 --- a/docs/report/dpdk_performance_tests/comparisons/current_vs_previous_release.rst +++ b/docs/report/dpdk_performance_tests/comparisons/current_vs_previous_release.rst @@ -55,9 +55,9 @@ NDR Comparison Comparison tables in ASCII and CSV formats: - `ASCII 2t1c NDR comparison <../../_static/dpdk/performance-changes-3n-skx-2t1c-ndr.txt>`_ - - `ASCII 4t1c NDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t1c-ndr.txt>`_ + - `ASCII 4t1c NDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t2c-ndr.txt>`_ - `CSV 2t1c NDR comparison <../../_static/dpdk/performance-changes-3n-skx-2t1c-ndr.csv>`_ - - `CSV 4t1c NDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t1c-ndr.csv>`_ + - `CSV 4t1c NDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t2c-ndr.csv>`_ PDR Comparison `````````````` @@ -65,6 +65,6 @@ PDR Comparison Comparison tables in ASCII and CSV formats: - `ASCII 2t1c PDR comparison <../../_static/dpdk/performance-changes-3n-skx-2t1c-pdr.txt>`_ - - `ASCII 4t1c PDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t1c-pdr.txt>`_ + - `ASCII 4t1c PDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t2c-pdr.txt>`_ - `CSV 2t1c PDR comparison <../../_static/dpdk/performance-changes-3n-skx-2t1c-pdr.csv>`_ - - `CSV 4t1c PDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t1c-pdr.csv>`_ + - `CSV 4t1c PDR comparison <../../_static/dpdk/performance-changes-3n-skx-4t2c-pdr.csv>`_ diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py index 7d5af3fe2d..f9d42ba069 100644 --- a/resources/tools/presentation/generator_CPTA.py +++ b/resources/tools/presentation/generator_CPTA.py @@ -174,28 +174,29 @@ def _generate_trending_traces(in_data, job_name, build_info, xaxis = list() for idx in data_x: date = build_info[job_name][str(idx)][0] - hover_str = ("date: {0}<br>" - "value: {1:,}<br>" - "{2}-ref: {3}<br>" - "csit-ref: mrr-{4}-build-{5}") + hover_str = ("date: {date}<br>" + "value: {value:,}<br>" + "{sut}-ref: {build}<br>" + "csit-ref: mrr-{period}-build-{build_nr}<br>" + "testbed: {testbed}") if "dpdk" in job_name: hover_text.append(hover_str.format( - date, - int(in_data[idx].avg), - "dpdk", - build_info[job_name][str(idx)][1]. - rsplit('~', 1)[0], - "weekly", - idx)) + date=date, + value=int(in_data[idx].avg), + sut="dpdk", + build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0], + period="weekly", + build_nr=idx, + testbed=build_info[job_name][str(idx)][2])) elif "vpp" in job_name: hover_text.append(hover_str.format( - date, - int(in_data[idx].avg), - "vpp", - build_info[job_name][str(idx)][1]. - rsplit('~', 1)[0], - "daily", - idx)) + date=date, + value=int(in_data[idx].avg), + sut="vpp", + build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0], + period="daily", + build_nr=idx, + testbed=build_info[job_name][str(idx)][2])) xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]), int(date[9:11]), int(date[12:]))) @@ -433,13 +434,19 @@ def _generate_all_charts(spec, input_data): # Create "build ID": "date" dict: build_info = dict() + tb_tbl = spec.environment.get("testbeds", None) for job_name, job_data in builds_dict.items(): if build_info.get(job_name, None) is None: build_info[job_name] = OrderedDict() for build in job_data: + testbed = "" + tb_ip = input_data.metadata(job_name, build).get("testbed", "") + if tb_ip and tb_tbl: + testbed = tb_tbl.get(tb_ip, "") build_info[job_name][build] = ( input_data.metadata(job_name, build).get("generated", ""), - input_data.metadata(job_name, build).get("version", "") + input_data.metadata(job_name, build).get("version", ""), + testbed ) work_queue = multiprocessing.JoinableQueue() diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 7590daa8fe..3ad1e3c756 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -497,12 +497,15 @@ def table_performance_trending_dashboard(table, input_data): if classification_lst: if isnan(rel_change_last) and isnan(rel_change_long): continue + if (isnan(last_avg) or + isnan(rel_change_last) or + isnan(rel_change_long)): + continue tbl_lst.append( [tbl_dict[tst_name]["name"], - '-' if isnan(last_avg) else round(last_avg / 1000000, 2), - '-' if isnan(rel_change_last) else rel_change_last, - '-' if isnan(rel_change_long) else rel_change_long, + rel_change_last, + rel_change_long, classification_lst[-win_size:].count("regression"), classification_lst[-win_size:].count("progression")]) @@ -798,9 +801,11 @@ def table_failed_tests(table, input_data): generated, input_data.metadata(job, build).get("version", ""), build) - except (TypeError, KeyError): - pass # No data in output.xml for this test + except (TypeError, KeyError) as err: + logging.warning("tst_name: {} - err: {}". + format(tst_name, repr(err))) + max_fails = 0 tbl_lst = list() for tst_data in tbl_dict.values(): fails_nr = 0 @@ -811,6 +816,7 @@ def table_failed_tests(table, input_data): fails_last_vpp = val[2] fails_last_csit = val[3] if fails_nr: + max_fails = fails_nr if fails_nr > max_fails else max_fails tbl_lst.append([tst_data["name"], fails_nr, fails_last_date, @@ -819,7 +825,7 @@ def table_failed_tests(table, input_data): tbl_lst.sort(key=lambda rel: rel[2], reverse=True) tbl_sorted = list() - for nrf in range(table["window"], -1, -1): + for nrf in range(max_fails, -1, -1): tbl_fails = [item for item in tbl_lst if item[1] == nrf] tbl_sorted.extend(tbl_fails) file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index fe6213628b..703ea33ba1 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -33,6 +33,7 @@ from os import remove from os.path import join from datetime import datetime as dt from datetime import timedelta +from json import loads from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory from input_data_files import download_and_unzip_data_file @@ -275,7 +276,8 @@ class ExecutionChecker(ResultVisitor): REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s' r'[\D\d]*') - REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)") + REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|" + r"VPP Version:\s*)(.*)") REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)" r"(RTE Version: 'DPDK )(.*)(')") @@ -318,6 +320,9 @@ class ExecutionChecker(ResultVisitor): # Timestamp self._timestamp = None + # Testbed. The testbed is identified by TG node IP address. + self._testbed = None + # Mapping of TCs long names self._mapping = mapping @@ -358,7 +363,8 @@ class ExecutionChecker(ResultVisitor): "vpp-version": self._get_vpp_version, "dpdk-version": self._get_dpdk_version, "teardown-vat-history": self._get_vat_history, - "test-show-runtime": self._get_show_run + "test-show-runtime": self._get_show_run, + "testbed": self._get_testbed } @property @@ -370,6 +376,28 @@ class ExecutionChecker(ResultVisitor): """ return self._data + def _get_testbed(self, msg): + """Called when extraction of testbed IP is required. + The testbed is identified by TG node IP address. + + :param msg: Message to process. + :type msg: Message + :returns: Nothing. + """ + + if msg.message.count("Arguments:"): + message = str(msg.message).replace(' ', '').replace('\n', '').\ + replace("'", '"').replace('b"', '"').\ + replace("honeycom", "honeycomb") + message = loads(message[11:-1]) + try: + self._testbed = message["TG"]["host"] + except (KeyError, ValueError): + pass + finally: + self._data["metadata"]["testbed"] = self._testbed + self._msg_type = None + def _get_vpp_version(self, msg): """Called when extraction of VPP version is required. @@ -378,7 +406,8 @@ class ExecutionChecker(ResultVisitor): :returns: Nothing. """ - if msg.message.count("return STDOUT Version:"): + if msg.message.count("return STDOUT Version:") or \ + msg.message.count("VPP Version:"): self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message). group(2)) self._data["metadata"]["version"] = self._version @@ -901,6 +930,8 @@ class ExecutionChecker(ResultVisitor): elif setup_kw.name.count("Setup performance global Variables") \ and not self._timestamp: self._msg_type = "timestamp" + elif setup_kw.name.count("Setup Framework") and not self._testbed: + self._msg_type = "testbed" else: return setup_kw.messages.visit(self) diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 45ea85c12c..1368bb01d4 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -61,6 +61,17 @@ # All directories MUST be defined in "paths" section. - "DIR[BUILD,HTML]" + testbeds: + "10.30.51.45": "LF-2n-SKX-21" + "10.30.51.53": "LF-2n-SKX-22" + "10.30.51.55": "LF-2n-SKX-23" + "10.30.51.57": "LF-2n-SKX-24" + "10.30.51.16": "LF-3n-HSW-01" + "10.30.51.20": "LF-3n-HSW-02" + "10.30.51.24": "LF-3n-HSW-03" + "10.30.51.48": "LF-3n-SKX-31" + "10.30.51.60": "LF-3n-SKX-32" + - type: "configuration" @@ -143,7 +154,7 @@ start: 300 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master: - start: 50 + start: 35 end: "lastCompletedBuild" plot-performance-trending-vpp-3n-hsw: @@ -153,45 +164,45 @@ plot-performance-trending-dpdk-3n-hsw: csit-dpdk-perf-mrr-weekly-master: - start: 50 + start: 35 end: "lastCompletedBuild" # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 50 + start: 100 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-3n-skx: - start: 3 + start: 7 end: "lastCompletedBuild" plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 50 + start: 100 end: "lastCompletedBuild" plot-performance-trending-dpdk-3n-skx: csit-dpdk-perf-mrr-weekly-master-3n-skx: - start: 3 + start: 7 end: "lastCompletedBuild" # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 50 + start: 100 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-2n-skx: - start: 3 + start: 7 end: "lastCompletedBuild" plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 50 + start: 100 end: "lastCompletedBuild" plot-performance-trending-dpdk-2n-skx: csit-dpdk-perf-mrr-weekly-master-2n-skx: - start: 3 + start: 7 end: "lastCompletedBuild" plot-layouts: @@ -285,23 +296,23 @@ start: 300 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master: - start: 50 + start: 35 end: "lastCompletedBuild" # 3n-skx csit-vpp-perf-mrr-daily-master-3n-skx: - start: 50 + start: 100 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-3n-skx: - start: 3 + start: 7 end: "lastCompletedBuild" # 2n-skx csit-vpp-perf-mrr-daily-master-2n-skx: - start: 50 + start: 100 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-2n-skx: - start: 3 + start: 7 end: "lastCompletedBuild" - @@ -336,7 +347,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-1t1c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -357,7 +367,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-2t2c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -378,7 +387,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-4t4c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -451,7 +459,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -472,7 +479,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -493,7 +499,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -566,7 +571,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc01-64b-2t1c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -587,7 +591,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc05-64b-4t2c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - @@ -608,7 +611,6 @@ - "tests.vpp.perf.l2.10ge2p1x520-eth-l2bdscale1mmaclrn-mrr.tc09-64b-8t4c-eth-l2bdscale1mmaclrn-ndrdisc" outlier-const: 1.5 window: 14 - evaluated-window: 14 long-trend-window: 180 - diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index 83838d8212..c149e3e51c 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -392,6 +392,12 @@ class Specification(object): except KeyError: self._specification["environment"]["build-dirs"] = None + try: + self._specification["environment"]["testbeds"] = \ + self._cfg_yaml[idx]["testbeds"] + except KeyError: + self._specification["environment"]["testbeds"] = None + logging.info("Done.") def _parse_configuration(self): |