# Copyright (c) 2019 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Algorithms to generate plots. """ import re import logging import pandas as pd import plotly.offline as ploff import plotly.graph_objs as plgo from plotly.exceptions import PlotlyError from collections import OrderedDict from copy import deepcopy from utils import mean, stdev COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink", "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black", "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson", "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod", "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon", "MediumSeaGreen", "SeaGreen", "LightSlateGrey"] REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*-') def generate_plots(spec, data): """Generate all plots specified in the specification file. :param spec: Specification read from the specification file. :param data: Data to process. :type spec: Specification :type data: InputData """ logging.info("Generating the plots ...") for index, plot in enumerate(spec.plots): try: logging.info(" Plot nr {0}: {1}".format(index + 1, plot.get("title", ""))) plot["limits"] = spec.configuration["limits"] eval(plot["algorithm"])(plot, data) logging.info(" Done.") except NameError as err: logging.error("Probably algorithm '{alg}' is not defined: {err}". format(alg=plot["algorithm"], err=repr(err))) logging.info("Done.") def plot_performance_box(plot, input_data): """Generate the plot(s) with algorithm: plot_performance_box specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ # Transform the data plot_title = plot.get("title", "") logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot_title)) data = input_data.filter_data(plot) if data is None: logging.error("No data.") return # Prepare the data for the plot y_vals = dict() y_tags = dict() for job in data: for build in job: for test in build: if y_vals.get(test["parent"], None) is None: y_vals[test["parent"]] = list() y_tags[test["parent"]] = test.get("tags", None) try: if test["type"] in ("NDRPDR", ): if "-pdr" in plot_title.lower(): y_vals[test["parent"]].\ append(test["throughput"]["PDR"]["LOWER"]) elif "-ndr" in plot_title.lower(): y_vals[test["parent"]]. \ append(test["throughput"]["NDR"]["LOWER"]) else: continue else: continue except (KeyError, TypeError): y_vals[test["parent"]].append(None) # Sort the tests order = plot.get("sort", None) if order and y_tags: y_sorted = OrderedDict() y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} for tag in order: logging.debug(tag) for suite, tags in y_tags_l.items(): if "not " in tag: tag = tag.split(" ")[-1] if tag.lower() in tags: continue else: if tag.lower() not in tags: continue try: y_sorted[suite] = y_vals.pop(suite) y_tags_l.pop(suite) logging.debug(suite) except KeyError as err: logging.error("Not found: {0}".format(repr(err))) finally: break else: y_sorted = y_vals # Add None to the lists with missing data max_len = 0 nr_of_samples = list() for val in y_sorted.values(): if len(val) > max_len: max_len = len(val) nr_of_samples.append(len(val)) for key, val in y_sorted.items(): if len(val) < max_len: val.extend([None for _ in range(max_len - len(val))]) # Add plot traces traces = list() df = pd.DataFrame(y_sorted) df.head() y_max = list() for i, col in enumerate(df.columns): tst_name = re.sub(REGEX_NIC, "", col.lower().replace('-ndrpdr', ''). replace('2n1l-', '')) name = "{nr}. ({samples:02d} run{plural}) {name}".\ format(nr=(i + 1), samples=nr_of_samples[i], plural='s' if nr_of_samples[i] > 1 else '', name=tst_name) logging.debug(name) traces.append(plgo.Box(x=[str(i + 1) + '.'] * len(df[col]), y=[y / 1000000 if y else None for y in df[col]], name=name, **plot["traces"])) try: val_max = max(df[col]) except ValueError as err: logging.error(repr(err)) continue if val_max: y_max.append(int(val_max / 1000000) + 1) try: # Create plot layout = deepcopy(plot["layout"]) if layout.get("title", None): layout["title"] = "Throughput: {0}". \ format(layout["title"]) if y_max: layout["yaxis"]["range"] = [0, max(y_max)] plpl = plgo.Figure(data=traces, layout=layout) # Export Plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(repr(err).replace("\n", " "))) return def plot_soak_bars(plot, input_data): """Generate the plot(s) with algorithm: plot_soak_bars specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ # Transform the data plot_title = plot.get("title", "") logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot_title)) data = input_data.filter_data(plot) if data is None: logging.error("No data.") return # Prepare the data for the plot y_vals = dict() y_tags = dict() for job in data: for build in job: for test in build: if y_vals.get(test["parent"], None) is None: y_tags[test["parent"]] = test.get("tags", None) try: if test["type"] in ("SOAK", ): y_vals[test["parent"]] = test["throughput"] else: continue except (KeyError, TypeError): y_vals[test["parent"]] = dict() # Sort the tests order = plot.get("sort", None) if order and y_tags: y_sorted = OrderedDict() y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} for tag in order: logging.debug(tag) for suite, tags in y_tags_l.items(): if "not " in tag: tag = tag.split(" ")[-1] if tag.lower() in tags: continue else: if tag.lower() not in tags: continue try: y_sorted[suite] = y_vals.pop(suite) y_tags_l.pop(suite) logging.debug(suite) except KeyError as err: logging.error("Not found: {0}".format(repr(err))) finally: break else: y_sorted = y_vals idx = 0 y_max = 0 traces = list() for test_name, test_data in y_sorted.items(): idx += 1 name = "{nr}. {name}".\ format(nr=idx, name=test_name.lower().replace('-soak', '')) if len(name) > 50: name_lst = name.split('-') name = "" split_name = True for segment in name_lst: if (len(name) + len(segment) + 1) > 50 and split_name: name += "
" split_name = False name += segment + '-' name = name[:-1] y_val = test_data.get("LOWER", None) if y_val: y_val /= 1000000 if y_val > y_max: y_max = y_val time = "No Information" result = "No Information" hovertext = ("{name}
" "Packet Throughput: {val:.2f}Mpps
" "Final Duration: {time}
" "Result: {result}".format(name=name, val=y_val, time=time, result=result)) traces.append(plgo.Bar(x=[str(idx) + '.', ], y=[y_val, ], name=name, text=hovertext, hoverinfo="text")) try: # Create plot layout = deepcopy(plot["layout"]) if layout.get("title", None): layout["title"] = "Packet Throughput: {0}". \ format(layout["title"]) if y_max: layout["yaxis"]["range"] = [0, y_max + 1] plpl = plgo.Figure(data=traces, layout=layout) # Export Plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(repr(err).replace("\n", " "))) return def plot_soak_boxes(plot, input_data): """Generate the plot(s) with algorithm: plot_soak_boxes specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ # Transform the data plot_title = plot.get("title", "") logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot_title)) data = input_data.filter_data(plot) if data is None: logging.error("No data.") return # Prepare the data for the plot y_vals = dict() y_tags = dict() for job in data: for build in job: for test in build: if y_vals.get(test["parent"], None) is None: y_tags[test["parent"]] = test.get("tags", None) try: if test["type"] in ("SOAK", ): y_vals[test["parent"]] = test["throughput"] else: continue except (KeyError, TypeError): y_vals[test["parent"]] = dict() # Sort the tests order = plot.get("sort", None) if order and y_tags: y_sorted = OrderedDict() y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} for tag in order: logging.debug(tag) for suite, tags in y_tags_l.items(): if "not " in tag: tag = tag.split(" ")[-1] if tag.lower() in tags: continue else: if tag.lower() not in tags: continue try: y_sorted[suite] = y_vals.pop(suite) y_tags_l.pop(suite) logging.debug(suite) except KeyError as err: logging.error("Not found: {0}".format(repr(err))) finally: break else: y_sorted = y_vals idx = 0 y_max = 0 traces = list() for test_name, test_data in y_sorted.items(): idx += 1 name = "{nr}. {name}".\ format(nr=idx, name=test_name.lower().replace('-soak', ''). replace('2n1l-', '')) if len(name) > 55: name_lst = name.split('-') name = "" split_name = True for segment in name_lst: if (len(name) + len(segment) + 1) > 55 and split_name: name += "
" split_name = False name += segment + '-' name = name[:-1] y_val = test_data.get("UPPER", None) if y_val: y_val /= 1000000 if y_val > y_max: y_max = y_val y_base = test_data.get("LOWER", None) if y_base: y_base /= 1000000 hovertext = ("Upper bound: {upper:.2f}
" "Lower bound: {lower:.2f}".format(upper=y_val, lower=y_base)) traces.append(plgo.Bar(x=[str(idx) + '.', ], # +0.05 to see the value in case lower == upper y=[y_val - y_base + 0.05, ], base=y_base, name=name, text=hovertext, hoverinfo="text")) try: # Create plot layout = deepcopy(plot["layout"]) if layout.get("title", None): layout["title"] = "Throughput: {0}". \ format(layout["title"]) if y_max: layout["yaxis"]["range"] = [0, y_max + 1] plpl = plgo.Figure(data=traces, layout=layout) # Export Plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(repr(err).replace("\n", " "))) return def plot_latency_error_bars(plot, input_data): """Generate the plot(s) with algorithm: plot_latency_error_bars specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ # Transform the data plot_title = plot.get("title", "") logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot_title)) data = input_data.filter_data(plot) if data is None: logging.error("No data.") return # Prepare the data for the plot y_tmp_vals = dict() y_tags = dict() for job in data: for build in job: for test in build: try: logging.debug("test['latency']: {0}\n". format(test["latency"])) except ValueError as err: logging.warning(repr(err)) if y_tmp_vals.get(test["parent"], None) is None: y_tmp_vals[test["parent"]] = [ list(), # direction1, min list(), # direction1, avg list(), # direction1, max list(), # direction2, min list(), # direction2, avg list() # direction2, max ] y_tags[test["parent"]] = test.get("tags", None) try: if test["type"] in ("NDRPDR", ): if "-pdr" in plot_title.lower(): ttype = "PDR" elif "-ndr" in plot_title.lower(): ttype = "NDR" else: logging.warning("Invalid test type: {0}". format(test["type"])) continue y_tmp_vals[test["parent"]][0].append( test["latency"][ttype]["direction1"]["min"]) y_tmp_vals[test["parent"]][1].append( test["latency"][ttype]["direction1"]["avg"]) y_tmp_vals[test["parent"]][2].append( test["latency"][ttype]["direction1"]["max"]) y_tmp_vals[test["parent"]][3].append( test["latency"][ttype]["direction2"]["min"]) y_tmp_vals[test["parent"]][4].append( test["latency"][ttype]["direction2"]["avg"]) y_tmp_vals[test["parent"]][5].append( test["latency"][ttype]["direction2"]["max"]) else: logging.warning("Invalid test type: {0}". format(test["type"])) continue except (KeyError, TypeError) as err: logging.warning(repr(err)) logging.debug("y_tmp_vals: {0}\n".format(y_tmp_vals)) # Sort the tests order = plot.get("sort", None) if order and y_tags: y_sorted = OrderedDict() y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} for tag in order: logging.debug(tag) for suite, tags in y_tags_l.items(): if "not " in tag: tag = tag.split(" ")[-1] if tag.lower() in tags: continue else: if tag.lower() not in tags: continue try: y_sorted[suite] = y_tmp_vals.pop(suite) y_tags_l.pop(suite) logging.debug(suite) except KeyError as err: logging.error("Not found: {0}".format(repr(err))) finally: break else: y_sorted = y_tmp_vals logging.debug("y_sorted: {0}\n".format(y_sorted)) x_vals = list() y_vals = list() y_mins = list() y_maxs = list() nr_of_samples = list() for key, val in y_sorted.items(): name = re.sub(REGEX_NIC, "", key.replace('-ndrpdr', ''). replace('2n1l-', '')) x_vals.append(name) # dir 1 y_vals.append(mean(val[1]) if val[1] else None) y_mins.append(mean(val[0]) if val[0] else None) y_maxs.append(mean(val[2]) if val[2] else None) nr_of_samples.append(len(val[1]) if val[1] else 0) x_vals.append(name) # dir 2 y_vals.append(mean(val[4]) if val[4] else None) y_mins.append(mean(val[3]) if val[3] else None) y_maxs.append(mean(val[5]) if val[5] else None) nr_of_samples.append(len(val[3]) if val[3] else 0) logging.debug("x_vals :{0}\n".format(x_vals)) logging.debug("y_vals :{0}\n".format(y_vals)) logging.debug("y_mins :{0}\n".format(y_mins)) logging.debug("y_maxs :{0}\n".format(y_maxs)) logging.debug("nr_of_samples :{0}\n".format(nr_of_samples)) traces = list() annotations = list() for idx in range(len(x_vals)): if not bool(int(idx % 2)): direction = "West-East" else: direction = "East-West" hovertext = ("No. of Runs: {nr}
" "Test: {test}
" "Direction: {dir}
".format(test=x_vals[idx], dir=direction, nr=nr_of_samples[idx])) if isinstance(y_maxs[idx], float): hovertext += "Max: {max:.2f}uSec
".format(max=y_maxs[idx]) if isinstance(y_vals[idx], float): hovertext += "Mean: {avg:.2f}uSec
".format(avg=y_vals[idx]) if isinstance(y_mins[idx], float): hovertext += "Min: {min:.2f}uSec".format(min=y_mins[idx]) if isinstance(y_maxs[idx], float) and isinstance(y_vals[idx], float): array = [y_maxs[idx] - y_vals[idx], ] else: array = [None, ] if isinstance(y_mins[idx], float) and isinstance(y_vals[idx], float): arrayminus = [y_vals[idx] - y_mins[idx], ] else: arrayminus = [None, ] logging.debug("y_vals[{1}] :{0}\n".format(y_vals[idx], idx)) logging.debug("array :{0}\n".format(array)) logging.debug("arrayminus :{0}\n".format(arrayminus)) traces.append(plgo.Scatter( x=[idx, ], y=[y_vals[idx], ], name=x_vals[idx], legendgroup=x_vals[idx], showlegend=bool(int(idx % 2)), mode="markers", error_y=dict( type='data', symmetric=False, array=array, arrayminus=arrayminus, color=COLORS[int(idx / 2)] ), marker=dict( size=10, color=COLORS[int(idx / 2)], ), text=hovertext, hoverinfo="text", )) annotations.append(dict( x=idx, y=0, xref="x", yref="y", xanchor="center", yanchor="top", text="E-W" if bool(int(idx % 2)) else "W-E", font=dict( size=16, ), align="center", showarrow=False )) try: # Create plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) layout = deepcopy(plot["layout"]) if layout.get("title", None): layout["title"] = "Latency: {0}".\ format(layout["title"]) layout["annotations"] = annotations plpl = plgo.Figure(data=traces, layout=layout) # Export Plot ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(str(err).replace("\n", " "))) return def plot_throughput_speedup_analysis(plot, input_data): """Generate the plot(s) with algorithm: plot_throughput_speedup_analysis specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ # Transform the data plot_title = plot.get("title", "") logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot_title)) data = input_data.filter_data(plot) if data is None: logging.error("No data.") return y_vals = dict() y_tags = dict() for job in data: for build in job: for test in build: if y_vals.get(test["parent"], None) is None: y_vals[test["parent"]] = {"1": list(), "2": list(), "4": list()} y_tags[test["parent"]] = test.get("tags", None) try: if test["type"] in ("NDRPDR",): if "-pdr" in plot_title.lower(): ttype = "PDR" elif "-ndr" in plot_title.lower(): ttype = "NDR" else: continue if "1C" in test["tags"]: y_vals[test["parent"]]["1"]. \ append(test["throughput"][ttype]["LOWER"]) elif "2C" in test["tags"]: y_vals[test["parent"]]["2"]. \ append(test["throughput"][ttype]["LOWER"]) elif "4C" in test["tags"]: y_vals[test["parent"]]["4"]. \ append(test["throughput"][ttype]["LOWER"]) except (KeyError, TypeError): pass if not y_vals: logging.warning("No data for the plot '{}'". format(plot.get("title", ""))) return y_1c_max = dict() for test_name, test_vals in y_vals.items(): for key, test_val in test_vals.items(): if test_val: avg_val = sum(test_val) / len(test_val) y_vals[test_name][key] = (avg_val, len(test_val)) ideal = avg_val / (int(key) * 1000000.0) if test_name not in y_1c_max or ideal > y_1c_max[test_name]: y_1c_max[test_name] = ideal vals = dict() y_max = list() nic_limit = 0 lnk_limit = 0 pci_limit = plot["limits"]["pci"]["pci-g3-x8"] for test_name, test_vals in y_vals.items(): try: if test_vals["1"][1]: name = re.sub(REGEX_NIC, "", test_name.replace('-ndrpdr', ''). replace('2n1l-', '')) vals[name] = dict() y_val_1 = test_vals["1"][0] / 1000000.0 y_val_2 = test_vals["2"][0] / 1000000.0 if test_vals["2"][0] \ else None y_val_4 = test_vals["4"][0] / 1000000.0 if test_vals["4"][0] \ else None vals[name]["val"] = [y_val_1, y_val_2, y_val_4] vals[name]["rel"] = [1.0, None, None] vals[name]["ideal"] = [y_1c_max[test_name], y_1c_max[test_name] * 2, y_1c_max[test_name] * 4] vals[name]["diff"] = [(y_val_1 - y_1c_max[test_name]) * 100 / y_val_1, None, None] vals[name]["count"] = [test_vals["1"][1], test_vals["2"][1], test_vals["4"][1]] try: # val_max = max(max(vals[name]["val"], vals[name]["ideal"])) val_max = max(vals[name]["val"]) except ValueError as err: logging.error(err) continue if val_max: # y_max.append(int((val_max / 10) + 1) * 10) y_max.append(val_max) if y_val_2: vals[name]["rel"][1] = round(y_val_2 / y_val_1, 2) vals[name]["diff"][1] = \ (y_val_2 - vals[name]["ideal"][1]) * 100 / y_val_2 if y_val_4: vals[name]["rel"][2] = round(y_val_4 / y_val_1, 2) vals[name]["diff"][2] = \ (y_val_4 - vals[name]["ideal"][2]) * 100 / y_val_4 except IndexError as err: logging.warning("No data for '{0}'".format(test_name)) logging.warning(repr(err)) # Limits: if "x520" in test_name: limit = plot["limits"]["nic"]["x520"] elif "x710" in test_name: limit = plot["limits"]["nic"]["x710"] elif "xxv710" in test_name: limit = plot["limits"]["nic"]["xxv710"] elif "xl710" in test_name: limit = plot["limits"]["nic"]["xl710"] elif "x553" in test_name: limit = plot["limits"]["nic"]["x553"] else: limit = 0 if limit > nic_limit: nic_limit = limit mul = 2 if "ge2p" in test_name else 1 if "10ge" in test_name: limit = plot["limits"]["link"]["10ge"] * mul elif "25ge" in test_name: limit = plot["limits"]["link"]["25ge"] * mul elif "40ge" in test_name: limit = plot["limits"]["link"]["40ge"] * mul elif "100ge" in test_name: limit = plot["limits"]["link"]["100ge"] * mul else: limit = 0 if limit > lnk_limit: lnk_limit = limit # Sort the tests order = plot.get("sort", None) if order and y_tags: y_sorted = OrderedDict() y_tags_l = {s: [t.lower() for t in ts] for s, ts in y_tags.items()} for tag in order: for test, tags in y_tags_l.items(): if tag.lower() in tags: name = re.sub(REGEX_NIC, "", test.replace('-ndrpdr', ''). replace('2n1l-', '')) try: y_sorted[name] = vals.pop(name) y_tags_l.pop(test) except KeyError as err: logging.error("Not found: {0}".format(err)) finally: break else: y_sorted = vals traces = list() annotations = list() x_vals = [1, 2, 4] # Limits: try: threshold = 1.1 * max(y_max) # 10% except ValueError as err: logging.error(err) return nic_limit /= 1000000.0 # if nic_limit < threshold: traces.append(plgo.Scatter( x=x_vals, y=[nic_limit, ] * len(x_vals), name="NIC: {0:.2f}Mpps".format(nic_limit), showlegend=False, mode="lines", line=dict( dash="dot", color=COLORS[-1], width=1), hoverinfo="none" )) annotations.append(dict( x=1, y=nic_limit, xref="x", yref="y", xanchor="left", yanchor="bottom", text="NIC: {0:.2f}Mpps".format(nic_limit), font=dict( size=14, color=COLORS[-1], ), align="left", showarrow=False )) # y_max.append(int((nic_limit / 10) + 1) * 10) y_max.append(nic_limit) lnk_limit /= 1000000.0 if lnk_limit < threshold: traces.append(plgo.Scatter( x=x_vals, y=[lnk_limit, ] * len(x_vals), name="Link: {0:.2f}Mpps".format(lnk_limit), showlegend=False, mode="lines", line=dict( dash="dot", color=COLORS[-2], width=1), hoverinfo="none" )) annotations.append(dict( x=1, y=lnk_limit, xref="x", yref="y", xanchor="left", yanchor="bottom", text="Link: {0:.2f}Mpps".format(lnk_limit), font=dict( size=14, color=COLORS[-2], ), align="left", showarrow=False )) # y_max.append(int((lnk_limit / 10) + 1) * 10) y_max.append(lnk_limit) pci_limit /= 1000000.0 if (pci_limit < threshold and (pci_limit < lnk_limit * 0.95 or lnk_limit > lnk_limit * 1.05)): traces.append(plgo.Scatter( x=x_vals, y=[pci_limit, ] * len(x_vals), name="PCIe: {0:.2f}Mpps".format(pci_limit), showlegend=False, mode="lines", line=dict( dash="dot", color=COLORS[-3], width=1), hoverinfo="none" )) annotations.append(dict( x=1, y=pci_limit, xref="x", yref="y", xanchor="left", yanchor="bottom", text="PCIe: {0:.2f}Mpps".format(pci_limit), font=dict( size=14, color=COLORS[-3], ), align="left", showarrow=False )) # y_max.append(int((pci_limit / 10) + 1) * 10) y_max.append(pci_limit) # Perfect and measured: cidx = 0 for name, val in y_sorted.iteritems(): hovertext = list() try: for idx in range(len(val["val"])): htext = "" if isinstance(val["val"][idx], float): htext += "No. of Runs: {1}
" \ "Mean: {0:.2f}Mpps
".format(val["val"][idx], val["count"][idx]) if isinstance(val["diff"][idx], float): htext += "Diff: {0:.0f}%
".format(round(val["diff"][idx])) if isinstance(val["rel"][idx], float): htext += "Speedup: {0:.2f}".format(val["rel"][idx]) hovertext.append(htext) traces.append(plgo.Scatter(x=x_vals, y=val["val"], name=name, legendgroup=name, mode="lines+markers", line=dict( color=COLORS[cidx], width=2), marker=dict( symbol="circle", size=10 ), text=hovertext, hoverinfo="text+name" )) traces.append(plgo.Scatter(x=x_vals, y=val["ideal"], name="{0} perfect".format(name), legendgroup=name, showlegend=False, mode="lines", line=dict( color=COLORS[cidx], width=2, dash="dash"), text=["Perfect: {0:.2f}Mpps".format(y) for y in val["ideal"]], hoverinfo="text" )) cidx += 1 except (IndexError, ValueError, KeyError) as err: logging.warning("No data for '{0}'".format(name)) logging.warning(repr(err)) try: # Create plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) layout = deepcopy(plot["layout"]) if layout.get("title", None): layout["title"] = "Speedup Multi-core: {0}". \ format(layout["title"]) # layout["yaxis"]["range"] = [0, int((max(y_max) / 10) + 1) * 10] layout["yaxis"]["range"] = [0, int(max(y_max) * 1.1)] layout["annotations"].extend(annotations) plpl = plgo.Figure(data=traces, layout=layout) # Export Plot ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(str(err).replace("\n", " "))) return def plot_http_server_performance_box(plot, input_data): """Generate the plot(s) with algorithm: plot_http_server_performance_box specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ # Transform the data logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot.get("title", ""))) data = input_data.filter_data(plot) if data is None: logging.error("No data.") return # Prepare the data for the plot y_vals = dict() for job in data: for build in job: for test in build: if y_vals.get(test["name"], None) is None: y_vals[test["name"]] = list() try: y_vals[test["name"]].append(test["result"]) except (KeyError, TypeError): y_vals[test["name"]].append(None) # Add None to the lists with missing data max_len = 0 nr_of_samples = list() for val in y_vals.values(): if len(val) > max_len: max_len = len(val) nr_of_samples.append(len(val)) for key, val in y_vals.items(): if len(val) < max_len: val.extend([None for _ in range(max_len - len(val))]) # Add plot traces traces = list() df = pd.DataFrame(y_vals) df.head() for i, col in enumerate(df.columns): name = "{nr}. ({samples:02d} run{plural}) {name}".\ format(nr=(i + 1), samples=nr_of_samples[i], plural='s' if nr_of_samples[i] > 1 else '', name=col.lower().replace('-ndrpdr', '')) if len(name) > 50: name_lst = name.split('-') name = "" split_name = True for segment in name_lst: if (len(name) + len(segment) + 1) > 50 and split_name: name += "
" split_name = False name += segment + '-' name = name[:-1] traces.append(plgo.Box(x=[str(i + 1) + '.'] * len(df[col]), y=df[col], name=name, **plot["traces"])) try: # Create plot plpl = plgo.Figure(data=traces, layout=plot["layout"]) # Export Plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(str(err).replace("\n", " "))) return def plot_service_density_heatmap(plot, input_data): """Generate the plot(s) with algorithm: plot_service_density_heatmap specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ REGEX_CN = re.compile(r'^(\d*)R(\d*)C$') REGEX_TEST_NAME = re.compile(r'^.*-(\d+vhost|\d+memif)-' r'(\d+chain|\d+pipe)-' r'(\d+vm|\d+dcr|\d+drc).*$') txt_chains = list() txt_nodes = list() vals = dict() # Transform the data logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot.get("title", ""))) data = input_data.filter_data(plot, continue_on_error=True) if data is None or data.empty: logging.error("No data.") return for job in data: for build in job: for test in build: for tag in test['tags']: groups = re.search(REGEX_CN, tag) if groups: c = str(groups.group(1)) n = str(groups.group(2)) break else: continue groups = re.search(REGEX_TEST_NAME, test["name"]) if groups and len(groups.groups()) == 3: hover_name = "{vhost}-{chain}-{vm}".format( vhost=str(groups.group(1)), chain=str(groups.group(2)), vm=str(groups.group(3))) else: hover_name = "" if vals.get(c, None) is None: vals[c] = dict() if vals[c].get(n, None) is None: vals[c][n] = dict(name=hover_name, vals=list(), nr=None, mean=None, stdev=None) try: if plot["include-tests"] == "MRR": result = test["result"]["receive-rate"].avg elif plot["include-tests"] == "PDR": result = test["throughput"]["PDR"]["LOWER"] elif plot["include-tests"] == "NDR": result = test["throughput"]["NDR"]["LOWER"] else: result = None except TypeError: result = None if result: vals[c][n]["vals"].append(result) if not vals: logging.error("No data.") return for key_c in vals.keys(): txt_chains.append(key_c) for key_n in vals[key_c].keys(): txt_nodes.append(key_n) if vals[key_c][key_n]["vals"]: vals[key_c][key_n]["nr"] = len(vals[key_c][key_n]["vals"]) vals[key_c][key_n]["mean"] = \ round(mean(vals[key_c][key_n]["vals"]) / 1000000, 1) vals[key_c][key_n]["stdev"] = \ round(stdev(vals[key_c][key_n]["vals"]) / 1000000, 1) txt_nodes = list(set(txt_nodes)) txt_chains = sorted(txt_chains, key=lambda chain: int(chain)) txt_nodes = sorted(txt_nodes, key=lambda node: int(node)) chains = [i + 1 for i in range(len(txt_chains))] nodes = [i + 1 for i in range(len(txt_nodes))] data = [list() for _ in range(len(chains))] for c in chains: for n in nodes: try: val = vals[txt_chains[c - 1]][txt_nodes[n - 1]]["mean"] except (KeyError, IndexError): val = None data[c - 1].append(val) # Colorscales: my_green = [[0.0, 'rgb(235, 249, 242)'], [1.0, 'rgb(45, 134, 89)']] my_blue = [[0.0, 'rgb(236, 242, 248)'], [1.0, 'rgb(57, 115, 172)']] my_grey = [[0.0, 'rgb(230, 230, 230)'], [1.0, 'rgb(102, 102, 102)']] hovertext = list() annotations = list() text = ("Test: {name}
" "Runs: {nr}
" "Thput: {val}
" "StDev: {stdev}") for c in range(len(txt_chains)): hover_line = list() for n in range(len(txt_nodes)): if data[c][n] is not None: annotations.append(dict( x=n+1, y=c+1, xref="x", yref="y", xanchor="center", yanchor="middle", text=str(data[c][n]), font=dict( size=14, ), align="center", showarrow=False )) hover_line.append(text.format( name=vals[txt_chains[c]][txt_nodes[n]]["name"], nr=vals[txt_chains[c]][txt_nodes[n]]["nr"], val=data[c][n], stdev=vals[txt_chains[c]][txt_nodes[n]]["stdev"])) hovertext.append(hover_line) traces = [ plgo.Heatmap(x=nodes, y=chains, z=data, colorbar=dict( title=plot.get("z-axis", ""), titleside="right", titlefont=dict( size=16 ), tickfont=dict( size=16, ), tickformat=".1f", yanchor="bottom", y=-0.02, len=0.925, ), showscale=True, colorscale=my_green, text=hovertext, hoverinfo="text") ] for idx, item in enumerate(txt_nodes): # X-axis, numbers: annotations.append(dict( x=idx+1, y=0.05, xref="x", yref="y", xanchor="center", yanchor="top", text=item, font=dict( size=16, ), align="center", showarrow=False )) for idx, item in enumerate(txt_chains): # Y-axis, numbers: annotations.append(dict( x=0.35, y=idx+1, xref="x", yref="y", xanchor="right", yanchor="middle", text=item, font=dict( size=16, ), align="center", showarrow=False )) # X-axis, title: annotations.append(dict( x=0.55, y=-0.15, xref="paper", yref="y", xanchor="center", yanchor="bottom", text=plot.get("x-axis", ""), font=dict( size=16, ), align="center", showarrow=False )) # Y-axis, title: annotations.append(dict( x=-0.1, y=0.5, xref="x", yref="paper", xanchor="center", yanchor="middle", text=plot.get("y-axis", ""), font=dict( size=16, ), align="center", textangle=270, showarrow=False )) updatemenus = list([ dict( x=1.0, y=0.0, xanchor='right', yanchor='bottom', direction='up', buttons=list([ dict( args=[{"colorscale": [my_green, ], "reversescale": False}], label="Green", method="update" ), dict( args=[{"colorscale": [my_blue, ], "reversescale": False}], label="Blue", method="update" ), dict( args=[{"colorscale": [my_grey, ], "reversescale": False}], label="Grey", method="update" ) ]) ) ]) try: layout = deepcopy(plot["layout"]) except KeyError as err: logging.error("Finished with error: No layout defined") logging.error(repr(err)) return layout["annotations"] = annotations layout['updatemenus'] = updatemenus try: # Create plot plpl = plgo.Figure(data=traces, layout=layout) # Export Plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(str(err).replace("\n", " "))) return def plot_service_density_heatmap_compare(plot, input_data): """Generate the plot(s) with algorithm: plot_service_density_heatmap_compare specified in the specification file. :param plot: Plot to generate. :param input_data: Data to process. :type plot: pandas.Series :type input_data: InputData """ REGEX_CN = re.compile(r'^(\d*)R(\d*)C$') REGEX_TEST_NAME = re.compile(r'^.*-(\d+ch|\d+pl)-' r'(\d+vh|\d+mif)-' r'(\d+vm|\d+dcr).*$') REGEX_THREADS = re.compile(r'^(\d+)(VM|DCR)(\d+)T$') txt_chains = list() txt_nodes = list() vals = dict() # Transform the data logging.info(" Creating the data set for the {0} '{1}'.". format(plot.get("type", ""), plot.get("title", ""))) data = input_data.filter_data(plot, continue_on_error=True) if data is None or data.empty: logging.error("No data.") return for job in data: for build in job: for test in build: for tag in test['tags']: groups = re.search(REGEX_CN, tag) if groups: c = str(groups.group(1)) n = str(groups.group(2)) break else: continue groups = re.search(REGEX_TEST_NAME, test["name"]) if groups and len(groups.groups()) == 3: hover_name = "{chain}-{vhost}-{vm}".format( chain=str(groups.group(1)), vhost=str(groups.group(2)), vm=str(groups.group(3))) else: hover_name = "" if vals.get(c, None) is None: vals[c] = dict() if vals[c].get(n, None) is None: vals[c][n] = dict(name=hover_name, vals_r=list(), vals_c=list(), nr_r=None, nr_c=None, mean_r=None, mean_c=None, stdev_r=None, stdev_c=None) try: if plot["include-tests"] == "MRR": result = test["result"]["receive-rate"].avg elif plot["include-tests"] == "PDR": result = test["throughput"]["PDR"]["LOWER"] elif plot["include-tests"] == "NDR": result = test["throughput"]["NDR"]["LOWER"] else: result = None except TypeError: result = None if result: for tag in test['tags']: groups = re.search(REGEX_THREADS, tag) if groups and len(groups.groups()) == 3: if str(groups.group(3)) == \ plot["reference"]["include"]: vals[c][n]["vals_r"].append(result) elif str(groups.group(3)) == \ plot["compare"]["include"]: vals[c][n]["vals_c"].append(result) break if not vals: logging.error("No data.") return for key_c in vals.keys(): txt_chains.append(key_c) for key_n in vals[key_c].keys(): txt_nodes.append(key_n) if vals[key_c][key_n]["vals_r"]: vals[key_c][key_n]["nr_r"] = len(vals[key_c][key_n]["vals_r"]) vals[key_c][key_n]["mean_r"] = \ round(mean(vals[key_c][key_n]["vals_r"]) / 1000000, 1) vals[key_c][key_n]["stdev_r"] = \ round(stdev(vals[key_c][key_n]["vals_r"]) / 1000000, 1) if vals[key_c][key_n]["vals_c"]: vals[key_c][key_n]["nr_c"] = len(vals[key_c][key_n]["vals_c"]) vals[key_c][key_n]["mean_c"] = \ round(mean(vals[key_c][key_n]["vals_c"]) / 1000000, 1) vals[key_c][key_n]["stdev_c"] = \ round(stdev(vals[key_c][key_n]["vals_c"]) / 1000000, 1) txt_nodes = list(set(txt_nodes)) txt_chains = sorted(txt_chains, key=lambda chain: int(chain)) txt_nodes = sorted(txt_nodes, key=lambda node: int(node)) chains = [i + 1 for i in range(len(txt_chains))] nodes = [i + 1 for i in range(len(txt_nodes))] data_r = [list() for _ in range(len(chains))] data_c = [list() for _ in range(len(chains))] diff = [list() for _ in range(len(chains))] for c in chains: for n in nodes: try: val_r = vals[txt_chains[c - 1]][txt_nodes[n - 1]]["mean_r"] except (KeyError, IndexError): val_r = None data_r[c - 1].append(val_r) try: val_c = vals[txt_chains[c - 1]][txt_nodes[n - 1]]["mean_c"] except (KeyError, IndexError): val_c = None data_c[c - 1].append(val_c) if val_c is not None and val_r: diff[c - 1].append(round((val_c - val_r) * 100 / val_r, 1)) else: diff[c - 1].append(None) # Colorscales: my_green = [[0.0, 'rgb(235, 249, 242)'], [1.0, 'rgb(45, 134, 89)']] my_blue = [[0.0, 'rgb(236, 242, 248)'], [1.0, 'rgb(57, 115, 172)']] my_grey = [[0.0, 'rgb(230, 230, 230)'], [1.0, 'rgb(102, 102, 102)']] hovertext = list() annotations = list() annotations_r = list() annotations_c = list() annotations_diff = list() text = ("Test: {name}" "
{title_r}: {text_r}" "
{title_c}: {text_c}{text_diff}") text_r = "Thput: {val_r}; StDev: {stdev_r}; Runs: {nr_r}" text_c = "Thput: {val_c}; StDev: {stdev_c}; Runs: {nr_c}" text_diff = "
Relative Difference {title_c} vs. {title_r}: {diff}%" for c in range(len(txt_chains)): hover_line = list() for n in range(len(txt_nodes)): point = dict( x=n + 1, y=c + 1, xref="x", yref="y", xanchor="center", yanchor="middle", text="", font=dict( size=14, ), align="center", showarrow=False ) point_text_r = "Not present" point_text_c = "Not present" point_text_diff = "" try: point_r = data_r[c][n] if point_r is not None: point_text_r = text_r.format( val_r=point_r, stdev_r=vals[txt_chains[c]][txt_nodes[n]]["stdev_r"], nr_r=vals[txt_chains[c]][txt_nodes[n]]["nr_r"]) except KeyError: point_r = None point["text"] = "" if point_r is None else point_r annotations_r.append(deepcopy(point)) try: point_c = data_c[c][n] if point_c is not None: point_text_c = text_c.format( val_c=point_c, stdev_c=vals[txt_chains[c]][txt_nodes[n]]["stdev_c"], nr_c=vals[txt_chains[c]][txt_nodes[n]]["nr_c"]) except KeyError: point_c = None point["text"] = "" if point_c is None else point_c annotations_c.append(deepcopy(point)) try: point_d = diff[c][n] if point_d is not None: point_text_diff = text_diff.format( title_r=plot["reference"]["name"], title_c=plot["compare"]["name"], diff=point_d) except KeyError: point_d = None point["text"] = "" if point_d is None else point_d annotations_diff.append(deepcopy(point)) try: name = vals[txt_chains[c]][txt_nodes[n]]["name"] except KeyError: continue hover_line.append(text.format( name=name, title_r=plot["reference"]["name"], text_r=point_text_r, title_c=plot["compare"]["name"], text_c=point_text_c, text_diff=point_text_diff )) hovertext.append(hover_line) traces = [ plgo.Heatmap(x=nodes, y=chains, z=data_r, visible=True, colorbar=dict( title=plot.get("z-axis", ""), titleside="right", titlefont=dict( size=16 ), tickfont=dict( size=16, ), tickformat=".1f", yanchor="bottom", y=-0.02, len=0.925, ), showscale=True, colorscale=my_green, reversescale=False, text=hovertext, hoverinfo="text"), plgo.Heatmap(x=nodes, y=chains, z=data_c, visible=False, colorbar=dict( title=plot.get("z-axis", ""), titleside="right", titlefont=dict( size=16 ), tickfont=dict( size=16, ), tickformat=".1f", yanchor="bottom", y=-0.02, len=0.925, ), showscale=True, colorscale=my_blue, reversescale=False, text=hovertext, hoverinfo="text"), plgo.Heatmap(x=nodes, y=chains, z=diff, name="Diff", visible=False, colorbar=dict( title="Relative Difference {name_c} vs. {name_r} [%]". format(name_c=plot["compare"]["name"], name_r=plot["reference"]["name"]), titleside="right", titlefont=dict( size=16 ), tickfont=dict( size=16, ), tickformat=".1f", yanchor="bottom", y=-0.02, len=0.925, ), showscale=True, colorscale=my_grey, reversescale=False, text=hovertext, hoverinfo="text") ] for idx, item in enumerate(txt_nodes): # X-axis, numbers: annotations.append(dict( x=idx+1, y=0.05, xref="x", yref="y", xanchor="center", yanchor="top", text=item, font=dict( size=16, ), align="center", showarrow=False )) for idx, item in enumerate(txt_chains): # Y-axis, numbers: annotations.append(dict( x=0.35, y=idx+1, xref="x", yref="y", xanchor="right", yanchor="middle", text=item, font=dict( size=16, ), align="center", showarrow=False )) # X-axis, title: annotations.append(dict( x=0.55, y=-0.15, xref="paper", yref="y", xanchor="center", yanchor="bottom", text=plot.get("x-axis", ""), font=dict( size=16, ), align="center", showarrow=False )) # Y-axis, title: annotations.append(dict( x=-0.1, y=0.5, xref="x", yref="paper", xanchor="center", yanchor="middle", text=plot.get("y-axis", ""), font=dict( size=16, ), align="center", textangle=270, showarrow=False )) updatemenus = list([ dict( active=0, x=1.0, y=0.0, xanchor='right', yanchor='bottom', direction='up', buttons=list([ dict( label=plot["reference"]["name"], method="update", args=[ { "visible": [True, False, False] }, { "colorscale": [my_green, ], "reversescale": False, "annotations": annotations + annotations_r, }, ] ), dict( label=plot["compare"]["name"], method="update", args=[ { "visible": [False, True, False] }, { "colorscale": [my_blue, ], "reversescale": False, "annotations": annotations + annotations_c, }, ] ), dict( label="Diff", method="update", args=[ { "visible": [False, False, True] }, { "colorscale": [my_grey, ], "reversescale": False, "annotations": annotations + annotations_diff, }, ] ), ]) ) ]) try: layout = deepcopy(plot["layout"]) except KeyError as err: logging.error("Finished with error: No layout defined") logging.error(repr(err)) return layout["annotations"] = annotations + annotations_r layout['updatemenus'] = updatemenus try: # Create plot plpl = plgo.Figure(data=traces, layout=layout) # Export Plot logging.info(" Writing file '{0}{1}'.". format(plot["output-file"], plot["output-file-type"])) ploff.plot(plpl, show_link=False, auto_open=False, filename='{0}{1}'.format(plot["output-file"], plot["output-file-type"])) except PlotlyError as err: logging.error(" Finished with error: {}". format(str(err).replace("\n", " "))) return