diff options
Diffstat (limited to 'resources/tools')
| -rw-r--r-- | resources/tools/presentation/generator_plots.py | 244 | ||||
| -rw-r--r-- | resources/tools/presentation/input_data_parser.py | 37 | ||||
| -rw-r--r-- | resources/tools/presentation/specification.yaml | 103 |
3 files changed, 383 insertions, 1 deletions
diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py index 32f146bca8..21dd1a0555 100644 --- a/resources/tools/presentation/generator_plots.py +++ b/resources/tools/presentation/generator_plots.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2019 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -195,6 +195,248 @@ def plot_performance_box(plot, input_data): return +def plot_soak_bars(plot, input_data): + """Generate the plot(s) with algorithm: plot_soak_bars + specified in the specification file. + + :param plot: Plot to generate. + :param input_data: Data to process. + :type plot: pandas.Series + :type input_data: InputData + """ + + # Transform the data + plot_title = plot.get("title", "") + logging.info(" Creating the data set for the {0} '{1}'.". + format(plot.get("type", ""), plot_title)) + data = input_data.filter_data(plot) + if data is None: + logging.error("No data.") + return + + # Prepare the data for the plot + y_vals = dict() + y_tags = dict() + for job in data: + for build in job: + for test in build: + if y_vals.get(test["parent"], None) is None: + y_tags[test["parent"]] = test.get("tags", None) + try: + if test["type"] in ("SOAK", ): + y_vals[test["parent"]] = test["throughput"] + else: + continue + except (KeyError, TypeError): + y_vals[test["parent"]] = dict() + + # Sort the tests + order = plot.get("sort", None) + if order and y_tags: + y_sorted = OrderedDict() + y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} + for tag in order: + logging.debug(tag) + for suite, tags in y_tags_l.items(): + if "not " in tag: + tag = tag.split(" ")[-1] + if tag.lower() in tags: + continue + else: + if tag.lower() not in tags: + continue + try: + y_sorted[suite] = y_vals.pop(suite) + y_tags_l.pop(suite) + logging.debug(suite) + except KeyError as err: + logging.error("Not found: {0}".format(repr(err))) + finally: + break + else: + y_sorted = y_vals + + idx = 0 + y_max = 0 + traces = list() + for test_name, test_data in y_sorted.items(): + idx += 1 + name = "{nr}. {name}".\ + format(nr=idx, name=test_name.lower().replace('-soak', '')) + if len(name) > 50: + name_lst = name.split('-') + name = "" + split_name = True + for segment in name_lst: + if (len(name) + len(segment) + 1) > 50 and split_name: + name += "<br> " + split_name = False + name += segment + '-' + name = name[:-1] + + y_val = test_data.get("LOWER", None) + if y_val: + y_val /= 1000000 + if y_val > y_max: + y_max = y_val + + time = "No Information" + result = "No Information" + hovertext = ("{name}<br>" + "Packet Throughput: {val:.2f}Mpps<br>" + "Final Duration: {time}<br>" + "Result: {result}".format(name=name, + val=y_val, + time=time, + result=result)) + traces.append(plgo.Bar(x=[str(idx) + '.', ], + y=[y_val, ], + name=name, + text=hovertext, + hoverinfo="text")) + try: + # Create plot + layout = deepcopy(plot["layout"]) + if layout.get("title", None): + layout["title"] = "<b>Packet Throughput:</b> {0}". \ + format(layout["title"]) + if y_max: + layout["yaxis"]["range"] = [0, y_max + 1] + plpl = plgo.Figure(data=traces, layout=layout) + # Export Plot + logging.info(" Writing file '{0}{1}'.". + format(plot["output-file"], plot["output-file-type"])) + ploff.plot(plpl, show_link=False, auto_open=False, + filename='{0}{1}'.format(plot["output-file"], + plot["output-file-type"])) + except PlotlyError as err: + logging.error(" Finished with error: {}". + format(repr(err).replace("\n", " "))) + return + + +def plot_soak_boxes(plot, input_data): + """Generate the plot(s) with algorithm: plot_soak_boxes + specified in the specification file. + + :param plot: Plot to generate. + :param input_data: Data to process. + :type plot: pandas.Series + :type input_data: InputData + """ + + # Transform the data + plot_title = plot.get("title", "") + logging.info(" Creating the data set for the {0} '{1}'.". + format(plot.get("type", ""), plot_title)) + data = input_data.filter_data(plot) + if data is None: + logging.error("No data.") + return + + # Prepare the data for the plot + y_vals = dict() + y_tags = dict() + for job in data: + for build in job: + for test in build: + if y_vals.get(test["parent"], None) is None: + y_tags[test["parent"]] = test.get("tags", None) + try: + if test["type"] in ("SOAK", ): + y_vals[test["parent"]] = test["throughput"] + else: + continue + except (KeyError, TypeError): + y_vals[test["parent"]] = dict() + + # Sort the tests + order = plot.get("sort", None) + if order and y_tags: + y_sorted = OrderedDict() + y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} + for tag in order: + logging.debug(tag) + for suite, tags in y_tags_l.items(): + if "not " in tag: + tag = tag.split(" ")[-1] + if tag.lower() in tags: + continue + else: + if tag.lower() not in tags: + continue + try: + y_sorted[suite] = y_vals.pop(suite) + y_tags_l.pop(suite) + logging.debug(suite) + except KeyError as err: + logging.error("Not found: {0}".format(repr(err))) + finally: + break + else: + y_sorted = y_vals + + idx = 0 + y_max = 0 + traces = list() + for test_name, test_data in y_sorted.items(): + idx += 1 + name = "{nr}. {name}".\ + format(nr=idx, name=test_name.lower().replace('-soak', '')) + if len(name) > 50: + name_lst = name.split('-') + name = "" + split_name = True + for segment in name_lst: + if (len(name) + len(segment) + 1) > 50 and split_name: + name += "<br> " + split_name = False + name += segment + '-' + name = name[:-1] + + y_val = test_data.get("UPPER", None) + if y_val: + y_val /= 1000000 + if y_val > y_max: + y_max = y_val + + y_base = test_data.get("LOWER", None) + if y_base: + y_base /= 1000000 + + hovertext = ("{name}<br>" + "Upper bound: {upper:.2f}Mpps<br>" + "Lower bound: {lower:.2f}Mpps".format(name=name, + upper=y_val, + lower=y_base)) + traces.append(plgo.Bar(x=[str(idx) + '.', ], + # +0.05 to see the value in case lower == upper + y=[y_val - y_base + 0.05, ], + base=y_base, + name=name, + text=hovertext, + hoverinfo="text")) + try: + # Create plot + layout = deepcopy(plot["layout"]) + if layout.get("title", None): + layout["title"] = "<b>Soak Tests:</b> {0}". \ + format(layout["title"]) + if y_max: + layout["yaxis"]["range"] = [0, y_max + 1] + plpl = plgo.Figure(data=traces, layout=layout) + # Export Plot + logging.info(" Writing file '{0}{1}'.". + format(plot["output-file"], plot["output-file-type"])) + ploff.plot(plpl, show_link=False, auto_open=False, + filename='{0}{1}'.format(plot["output-file"], + plot["output-file-type"])) + except PlotlyError as err: + logging.error(" Finished with error: {}". + format(repr(err).replace("\n", " "))) + return + + def plot_latency_error_bars(plot, input_data): """Generate the plot(s) with algorithm: plot_latency_error_bars specified in the specification file. diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 5e4ca42de4..7b36352ed9 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -252,6 +252,9 @@ class ExecutionChecker(ResultVisitor): # TODO: Remove when definitely no NDRPDRDISC tests are used: REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)') + REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n' + r'PLRsearch upper bound::\s(\d+.\d+)') + REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n' r'NDR_UPPER:\s(\d+.\d+).*\n' r'PDR_LOWER:\s(\d+.\d+).*\n.*\n' @@ -567,6 +570,33 @@ class ExecutionChecker(ResultVisitor): return throughput, status + def _get_plr_throughput(self, msg): + """Get PLRsearch lower bound and PLRsearch upper bound from the test + message. + + :param msg: The test message to be parsed. + :type msg: str + :returns: Parsed data as a dict and the status (PASS/FAIL). + :rtype: tuple(dict, str) + """ + + throughput = { + "LOWER": -1.0, + "UPPER": -1.0 + } + status = "FAIL" + groups = re.search(self.REGEX_PLR_RATE, msg) + + if groups is not None: + try: + throughput["LOWER"] = float(groups.group(1)) + throughput["UPPER"] = float(groups.group(2)) + status = "PASS" + except (IndexError, ValueError): + pass + + return throughput, status + def _get_ndrpdr_latency(self, msg): """Get LATENCY from the test message. @@ -739,6 +769,7 @@ class ExecutionChecker(ResultVisitor): if test.status == "PASS" and ("NDRPDRDISC" in tags or "NDRPDR" in tags or + "SOAK" in tags or "TCP" in tags or "MRR" in tags or "BMRR" in tags): @@ -750,6 +781,8 @@ class ExecutionChecker(ResultVisitor): test_result["type"] = "PDR" elif "NDRPDR" in tags: test_result["type"] = "NDRPDR" + elif "SOAK" in tags: + test_result["type"] = "SOAK" elif "TCP" in tags: test_result["type"] = "TCP" elif "MRR" in tags: @@ -790,6 +823,10 @@ class ExecutionChecker(ResultVisitor): test_result["latency"], test_result["status"] = \ self._get_ndrpdr_latency(test.message) + elif test_result["type"] in ("SOAK", ): + test_result["throughput"], test_result["status"] = \ + self._get_plr_throughput(test.message) + elif test_result["type"] in ("TCP", ): groups = re.search(self.REGEX_TCP, test.message) test_result["result"] = int(groups.group(2)) diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index bd16e165c5..832f54d574 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -488,6 +488,10 @@ - 41 # NDRPDR sel - 42 # NDRPDR sel + plot-vpp-soak-2n-skx: + csit-vpp-perf-verify-master-2n-skx: + - 207 # SOAK sel + plot-vpp-throughput-latency-3n-skx: csit-vpp-perf-verify-1810-3n-skx: - 24 # NDRPDR full @@ -750,6 +754,57 @@ plot-layouts: + plot-soak-throughput: + titlefont: + size: 18 + xaxis: + title: "<b>Test Cases [Index]</b>" + titlefont: + size: 16 + autorange: True + fixedrange: False + gridcolor: "rgb(230, 230, 230)" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickmode: "linear" + tickfont: + size: 16 + zeroline: False + yaxis: + title: "<b>Packet Throughput [Mpps]</b>" + titlefont: + size: 16 + gridcolor: "rgb(230, 230, 230)" + hoverformat: ".4s" + tickformat: ".3s" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickfont: + size: 16 + zeroline: False + range: [0,50] + autosize: False + margin: + t: 50 + b: 0 + l: 80 + r: 20 + showlegend: True + legend: + orientation: "h" + font: + size: 16 + width: 700 + height: 900 + plot-cps: titlefont: size: 18 @@ -1227,6 +1282,8 @@ - 23 # NDRPDR sel - 24 # NDRPDR sel - 27 # NDRPDR sel + csit-vpp-perf-verify-master-2n-skx: + - 207 # SOAK sel csit-vpp-perf-check-1804: - 5 # mrr - full - 6 # mrr - sel @@ -5199,6 +5256,52 @@ ### P L O T S ### ################################################################################ +# Soak test - example +- + type: "plot" + title: "VPP Throughput Soak: Example" + algorithm: "plot_soak_boxes" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/example-soak-boxes" + data: "plot-vpp-soak-2n-skx" + filter: "'SOAK'" + parameters: + - "throughput" + - "parent" + - "tags" + sort: + - "L2PATCH" + - "L2XCBASE" + - "L2BDMACLRN" + - "IP4FWD" + - "IP6FWD" + layout: + title: "example-30min-boxes" + layout: "plot-soak-throughput" + +# Soak test - example +- + type: "plot" + title: "VPP Throughput Soak: Example" + algorithm: "plot_soak_bars" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/example-soak-bars" + data: "plot-vpp-soak-2n-skx" + filter: "'SOAK'" + parameters: + - "throughput" + - "parent" + - "tags" + sort: + - "L2PATCH" + - "L2XCBASE" + - "L2BDMACLRN" + - "IP4FWD" + - "IP6FWD" + layout: + title: "example-30min-bars" + layout: "plot-soak-throughput" + ################################################################################ # Plots VPP HTTP Server Performance - |