From 67db7dd16cba6db668234b10b22f4ff9c8d58a92 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Tue, 27 Oct 2020 10:21:58 +0100 Subject: PAL: Add processing of PPS and CPS tests Change-Id: I7b15c69c923d47e83c2dc586cdc03ed4ebaeec74 Signed-off-by: Tibor Frank (cherry picked from commit 7bcbdcb30d2eea8fe7e1fb60696e39abef897920) --- resources/tools/presentation/generator_plots.py | 6 +- resources/tools/presentation/input_data_parser.py | 127 ++++++--- resources/tools/presentation/specification.yaml | 302 ++++++++++++++++++++++ 3 files changed, 396 insertions(+), 39 deletions(-) (limited to 'resources/tools/presentation') diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py index cae334ade9..42f450e688 100644 --- a/resources/tools/presentation/generator_plots.py +++ b/resources/tools/presentation/generator_plots.py @@ -386,8 +386,8 @@ def plot_perf_box_name(plot, input_data): if y_vals.get(test[u"parent"], None) is None: y_vals[test[u"parent"]] = list() try: - if test[u"type"] in (u"NDRPDR", ): - test_type = u"NDRPDR" + if test[u"type"] in (u"NDRPDR", u"CPS"): + test_type = test[u"type"] if u"-pdr" in plot_title: ttype = u"PDR" @@ -477,7 +477,7 @@ def plot_perf_box_name(plot, input_data): # Create plot layout = deepcopy(plot[u"layout"]) if layout.get(u"title", None): - if test_type in (u"HOSTSTACK", ): + if test_type in (u"HOSTSTACK", u"CPS"): layout[u"title"] = f"Bandwidth: {layout[u'title']}" else: layout[u"title"] = f"Throughput: {layout[u'title']}" diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 1a8b5bf98d..6eb60ec284 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -229,9 +229,17 @@ class ExecutionChecker(ResultVisitor): r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n' r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n' ) + REGEX_CPS_MSG_INFO = re.compile( + r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n' + r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*' + ) + REGEX_PPS_MSG_INFO = re.compile( + r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n' + r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*' + ) REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]') - # TODO: Remove when not needed + # Needed for CPS and PPS tests REGEX_NDRPDR_LAT_BASE = re.compile( r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n' r'LATENCY.*\[\'(.*)\', \'(.*)\'\]' @@ -244,18 +252,7 @@ class ExecutionChecker(ResultVisitor): r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' r'Latency.*\[\'(.*)\', \'(.*)\'\]' ) - # TODO: Remove when not needed - REGEX_NDRPDR_LAT_LONG = re.compile( - r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n' - r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n' - r'Latency.*\[\'(.*)\', \'(.*)\'\]' - ) + REGEX_VERSION_VPP = re.compile( r"(return STDOUT Version:\s*|" r"VPP Version:\s*|VPP version:\s*)(.*)" @@ -271,8 +268,7 @@ class ExecutionChecker(ResultVisitor): r'tx\s(\d*),\srx\s(\d*)' ) REGEX_BMRR = re.compile( - r'Maximum Receive Rate trial results' - r' in packets per second: \[(.*)\]' + r'Maximum Receive Rate trial results .*: \[(.*)\]' ) REGEX_RECONF_LOSS = re.compile( r'Packets lost due to reconfig: (\d*)' @@ -392,12 +388,56 @@ class ExecutionChecker(ResultVisitor): except (AttributeError, IndexError, ValueError, KeyError): return u"Test Failed." + def _get_data_from_cps_test_msg(self, msg): + """Get info from message of NDRPDR CPS tests. + + :param msg: Message to be processed. + :type msg: str + :returns: Processed message or "Test Failed." if a problem occurs. + :rtype: str + """ + + groups = re.search(self.REGEX_CPS_MSG_INFO, msg) + if not groups or groups.lastindex != 2: + return u"Test Failed." + + try: + return ( + f"1. {(float(groups.group(1)) / 1e6):5.2f}\n" + f"2. {(float(groups.group(2)) / 1e6):5.2f}" + ) + except (AttributeError, IndexError, ValueError, KeyError): + return u"Test Failed." + + def _get_data_from_pps_test_msg(self, msg): + """Get info from message of NDRPDR PPS tests. + + :param msg: Message to be processed. + :type msg: str + :returns: Processed message or "Test Failed." if a problem occurs. + :rtype: str + """ + + groups = re.search(self.REGEX_PPS_MSG_INFO, msg) + if not groups or groups.lastindex != 4: + return u"Test Failed." + + try: + return ( + f"1. {(float(groups.group(1)) / 1e6):5.2f} " + f"{float(groups.group(2)):5.2f}\n" + f"2. {(float(groups.group(3)) / 1e6):5.2f} " + f"{float(groups.group(4)):5.2f}" + ) + except (AttributeError, IndexError, ValueError, KeyError): + return u"Test Failed." + def _get_data_from_perf_test_msg(self, msg): """Get info from message of NDRPDR performance tests. :param msg: Message to be processed. :type msg: str - :returns: Processed message or original message if a problem occurs. + :returns: Processed message or "Test Failed." if a problem occurs. :rtype: str """ @@ -816,10 +856,7 @@ class ExecutionChecker(ResultVisitor): }, } - # TODO: Rewrite when long and base are not needed - groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg) - if groups is None: - groups = re.search(self.REGEX_NDRPDR_LAT, msg) + groups = re.search(self.REGEX_NDRPDR_LAT, msg) if groups is None: groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg) if groups is None: @@ -1060,9 +1097,18 @@ class ExecutionChecker(ResultVisitor): if test.status == u"PASS": if u"NDRPDR" in tags: - test_result[u"msg"] = self._get_data_from_perf_test_msg( - test.message).replace(u'\n', u' |br| ').\ - replace(u'\r', u'').replace(u'"', u"'") + if u"TCP_PPS" in tags or u"UDP_PPS" in tags: + test_result[u"msg"] = self._get_data_from_pps_test_msg( + test.message).replace(u'\n', u' |br| '). \ + replace(u'\r', u'').replace(u'"', u"'") + elif u"TCP_CPS" in tags or u"UDP_CPS" in tags: + test_result[u"msg"] = self._get_data_from_cps_test_msg( + test.message).replace(u'\n', u' |br| '). \ + replace(u'\r', u'').replace(u'"', u"'") + else: + test_result[u"msg"] = self._get_data_from_perf_test_msg( + test.message).replace(u'\n', u' |br| ').\ + replace(u'\r', u'').replace(u'"', u"'") elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags: test_result[u"msg"] = self._get_data_from_mrr_test_msg( test.message).replace(u'\n', u' |br| ').\ @@ -1107,28 +1153,24 @@ class ExecutionChecker(ResultVisitor): return if test.status == u"PASS": + logging.info(self._test_id) + logging.info(tags) if u"DEVICETEST" in tags: test_result[u"type"] = u"DEVICETEST" elif u"NDRPDR" in tags: - test_result[u"type"] = u"NDRPDR" + if u"TCP_CPS" in tags or u"UDP_CPS" in tags: + test_result[u"type"] = u"CPS" + else: + test_result[u"type"] = u"NDRPDR" test_result[u"throughput"], test_result[u"status"] = \ self._get_ndrpdr_throughput(test.message) test_result[u"gbps"], test_result[u"status"] = \ self._get_ndrpdr_throughput_gbps(test.message) test_result[u"latency"], test_result[u"status"] = \ self._get_ndrpdr_latency(test.message) - elif u"SOAK" in tags: - test_result[u"type"] = u"SOAK" - test_result[u"throughput"], test_result[u"status"] = \ - self._get_plr_throughput(test.message) - elif u"HOSTSTACK" in tags: - test_result[u"type"] = u"HOSTSTACK" - test_result[u"result"], test_result[u"status"] = \ - self._get_hoststack_data(test.message, tags) - elif u"TCP" in tags: - test_result[u"type"] = u"TCP" - groups = re.search(self.REGEX_TCP, test.message) - test_result[u"result"] = int(groups.group(2)) + logging.info(test_result[u"throughput"]) + logging.info(test_result[u"gbps"]) + logging.info(test_result[u"latency"]) elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags: if u"MRR" in tags: test_result[u"type"] = u"MRR" @@ -1150,6 +1192,19 @@ class ExecutionChecker(ResultVisitor): groups = re.search(self.REGEX_MRR, test.message) test_result[u"result"][u"receive-rate"] = \ float(groups.group(3)) / float(groups.group(1)) + logging.info(test_result[u"result"][u"receive-rate"]) + elif u"SOAK" in tags: + test_result[u"type"] = u"SOAK" + test_result[u"throughput"], test_result[u"status"] = \ + self._get_plr_throughput(test.message) + elif u"HOSTSTACK" in tags: + test_result[u"type"] = u"HOSTSTACK" + test_result[u"result"], test_result[u"status"] = \ + self._get_hoststack_data(test.message, tags) + elif u"TCP" in tags: + test_result[u"type"] = u"TCP" + groups = re.search(self.REGEX_TCP, test.message) + test_result[u"result"] = int(groups.group(2)) elif u"RECONF" in tags: test_result[u"type"] = u"RECONF" test_result[u"result"] = None diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 744aaa1f25..d3111523dc 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -1838,6 +1838,116 @@ hoverlabel: namelength: -1 + plot-throughput-gbps: + titlefont: + size: 16 + xaxis: + title: "Test Cases [Index]" + titlefont: + size: 14 + autorange: True + fixedrange: False + gridcolor: "rgb(230, 230, 230)" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickmode: "linear" + tickfont: + size: 14 + zeroline: False + yaxis: + title: "Packet Throughput [Gbps]" + titlefont: + size: 14 + gridcolor: "rgb(230, 230, 230)" + hoverformat: ".4r" + tickformat: ".3r" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickfont: + size: 14 + zeroline: False + range: [0,10] + autosize: False + margin: + t: 50 + b: 0 + l: 80 + r: 20 + showlegend: True + legend: + orientation: "h" + font: + size: 14 + width: 700 + height: 900 + paper_bgcolor: "#fff" + plot_bgcolor: "#fff" + hoverlabel: + namelength: -1 + + plot-cps: + titlefont: + size: 16 + xaxis: + title: "Test Cases [Index]" + titlefont: + size: 14 + autorange: True + fixedrange: False + gridcolor: "rgb(230, 230, 230)" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickmode: "linear" + tickfont: + size: 14 + zeroline: False + yaxis: + title: "Connections Per Second [cps]" + titlefont: + size: 14 + gridcolor: "rgb(230, 230, 230)" + hoverformat: ".4r" + tickformat: ".3s" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickfont: + size: 14 + zeroline: False + range: [0,10] + autosize: False + margin: + t: 50 + b: 0 + l: 80 + r: 20 + showlegend: True + legend: + orientation: "h" + font: + size: 14 + width: 700 + height: 900 + paper_bgcolor: "#fff" + plot_bgcolor: "#fff" + hoverlabel: + namelength: -1 + plot-throughput-speedup-analysis: titlefont: size: 16 @@ -1934,6 +2044,198 @@ }, ] + plot-throughput-speedup-analysis-gbps: + titlefont: + size: 16 + xaxis: + title: "Number of Cores [Qty]" + titlefont: + size: 14 + autorange: True + fixedrange: False + gridcolor: "rgb(230, 230, 230)" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + tickfont: + size: 14 + zeroline: False + yaxis: + title: "Packet Throughput [Gbps]" + titlefont: + size: 14 + type: "linear" + gridcolor: "rgb(230, 230, 230)" + hoverformat: ".4s" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickformat: ".4s" + tickfont: + size: 14 + zeroline: True + rangemode: "tozero" + range: [0,100] + legend: + orientation: "h" + font: + size: 14 + xanchor: "left" + yanchor: "top" + x: 0 + y: -0.2 + bgcolor: "rgba(255, 255, 255, 0)" + bordercolor: "rgba(255, 255, 255, 0)" + traceorder: "normal" # "grouped" does not work: bug https://github.com/plotly/plotly.js/issues/1913 + autosize: False + margin: + 't': 50 + 'b': 150 + 'l': 85 + 'r': 10 + showlegend: True + width: 700 + height: 700 + paper_bgcolor: "#fff" + plot_bgcolor: "#fff" + hoverlabel: + namelength: -1 + annotations: [ + { + text: "_ _ __", + align: "left", + showarrow: False, + xref: "paper", + yref: "paper", + xanchor: "left", + yanchor: "top", + x: 0, + y: -0.13, + font: { + family: "Consolas, Courier New", + size: 13 + }, + }, + { + text: " Perfect Measured", + align: "left", + showarrow: False, + xref: "paper", + yref: "paper", + xanchor: "left", + yanchor: "top", + x: 0, + y: -0.15, + font: { + family: "Consolas, Courier New", + size: 13 + }, + }, + ] + + plot-throughput-speedup-analysis-cps: + titlefont: + size: 16 + xaxis: + title: "Number of Cores [Qty]" + titlefont: + size: 14 + autorange: True + fixedrange: False + gridcolor: "rgb(230, 230, 230)" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + tickfont: + size: 14 + zeroline: False + yaxis: + title: "Connections Per Second [cps]" + titlefont: + size: 14 + type: "linear" + gridcolor: "rgb(230, 230, 230)" + hoverformat: ".4s" + linecolor: "rgb(220, 220, 220)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(220, 220, 220)" + tickformat: ".4s" + tickfont: + size: 14 + zeroline: True + rangemode: "tozero" + range: [0,100] + legend: + orientation: "h" + font: + size: 14 + xanchor: "left" + yanchor: "top" + x: 0 + y: -0.2 + bgcolor: "rgba(255, 255, 255, 0)" + bordercolor: "rgba(255, 255, 255, 0)" + traceorder: "normal" # "grouped" does not work: bug https://github.com/plotly/plotly.js/issues/1913 + autosize: False + margin: + 't': 50 + 'b': 150 + 'l': 85 + 'r': 10 + showlegend: True + width: 700 + height: 700 + paper_bgcolor: "#fff" + plot_bgcolor: "#fff" + hoverlabel: + namelength: -1 + annotations: [ + { + text: "_ _ __", + align: "left", + showarrow: False, + xref: "paper", + yref: "paper", + xanchor: "left", + yanchor: "top", + x: 0, + y: -0.13, + font: { + family: "Consolas, Courier New", + size: 13 + }, + }, + { + text: " Perfect Measured", + align: "left", + showarrow: False, + xref: "paper", + yref: "paper", + xanchor: "left", + yanchor: "top", + x: 0, + y: -0.15, + font: { + family: "Consolas, Courier New", + size: 13 + }, + }, + ] + plot-latency: titlefont: size: 16 -- cgit 1.2.3-korg