diff options
| -rw-r--r-- | resources/tools/presentation/doc/pal_lld.rst | 140 | ||||
| -rw-r--r-- | resources/tools/presentation/generator_plots.py | 98 | ||||
| -rw-r--r-- | resources/tools/presentation/specification.yaml | 127 |
3 files changed, 338 insertions, 27 deletions
diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst index 9158b889b9..12318ccfe9 100644 --- a/resources/tools/presentation/doc/pal_lld.rst +++ b/resources/tools/presentation/doc/pal_lld.rst @@ -83,6 +83,8 @@ the type: - type: "environment" - + type: "configuration" + - type: "debug" - type: "static" @@ -123,6 +125,7 @@ This section has the following parts: - build-dirs - a list of the directories where the results are stored. The structure of the section "Environment" is as follows (example): + :: - @@ -223,6 +226,108 @@ will be automatically changed to DIR[WORKING,DATA]: "_tmp/data" +Section: Configuration +'''''''''''''''''''''' + +This section specifies the groups of parameters which are repeatedly used in the +elements defined later in the specification file. It has the following parts: + + - data sets - Specification of data sets used later in element's specifications + to define the input data. + - plot layouts - Specification of plot layouts used later in plots' + specifications to define the plot layout. + +The structure of the section "Configuration" is as follows (example): + +:: + + - + type: "configuration" + data-sets: + plot-vpp-throughput-latency: + csit-vpp-perf-1710-all: + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + vpp-perf-results: + csit-vpp-perf-1710-all: + - 20 + - 23 + plot-layouts: + plot-throughput: + xaxis: + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + title: "Indexed Test Cases" + zeroline: False + yaxis: + gridcolor: "rgb(238, 238, 238)'" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + title: "Packets Per Second [pps]" + zeroline: False + boxmode: "group" + boxgroupgap: 0.5 + autosize: False + margin: + t: 50 + b: 20 + l: 50 + r: 20 + showlegend: True + legend: + orientation: "h" + width: 700 + height: 1000 + +The definitions from this sections are used in the elements, e.g.: + +:: + + - + type: "plot" + title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + algorithm: "plot_performance_box" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc" + data: + "plot-vpp-throughput-latency" + filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + traces: + hoverinfo: "x+y" + boxpoints: "outliers" + whiskerwidth: 0 + layout: + title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" + layout: + "plot-throughput" + + Section: Debug mode ''''''''''''''''''' @@ -1037,16 +1142,45 @@ Data analytics part implements: - etc. +Throughput Speedup Analysis - Multi-Core Speedup Ratio +'''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Throughput Speedup Analysis (TSA) calculates a speedup factor for 1, 2, 4 cores +which is defined as: + +:: + + throughput + speedup factor = ----------------- + 1-core-throughput + +A bar plot displays the speedup factor (normalized throughput for 64B/78B on 1 +core). The plot displays number of cores on the X-axis and the speedup factor on +the Y-axis. + +For better comparision, there can be displayed more than one set of data in a +plot. So, in general: + + - graph type: grouped bars; + - graph X-axis: (testcase index, number of cores); + - graph Y-axis: speedup factor. + +The data displayed is a subset of existing performance tests with 1core, 2core, +4core. + +:TODO: Specify the data model for TSA. + + Advanced data analytics ``````````````````````` As the next steps, advanced data analytics (ADA) will be implemented using machine learning (ML) and artificial intelligence (AI). -TODO: +:TODO: - - describe the concept of ADA. - - add specification. + - describe the concept of ADA. + - add specification. Data presentation diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py index 9ef8574907..5580cbdf1c 100644 --- a/resources/tools/presentation/generator_plots.py +++ b/resources/tools/presentation/generator_plots.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -219,3 +219,99 @@ def plot_latency_box(plot, input_data): return logging.info(" Done.") + + +def plot_throughput_speedup_analysis(plot, input_data): + """Generate the plot(s) with algorithm: plot_throughput_speedup_analysis + specified in the specification file. + + :param plot: Plot to generate. + :param input_data: Data to process. + :type plot: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the plot {0} ...". + format(plot.get("title", ""))) + + # Transform the data + data = input_data.filter_data(plot) + if data is None: + logging.error("No data.") + return + + throughput = dict() + for job in data: + for build in job: + for test in build: + if throughput.get(test["parent"], None) is None: + throughput[test["parent"]] = {"1": list(), + "2": list(), + "4": list()} + try: + if "1T1C" in test["tags"]: + throughput[test["parent"]]["1"].\ + append(test["throughput"]["value"]) + elif "2T2C" in test["tags"]: + throughput[test["parent"]]["2"]. \ + append(test["throughput"]["value"]) + elif "4T4C" in test["tags"]: + throughput[test["parent"]]["4"]. \ + append(test["throughput"]["value"]) + except (KeyError, TypeError): + pass + + for test_name, test_vals in throughput.items(): + for key, test_val in test_vals.items(): + if test_val: + throughput[test_name][key] = sum(test_val) / len(test_val) + + print(throughput) + + names = ['1 core', '2 cores', '4 cores'] + x_vals = list() + y_vals_1 = list() + y_vals_2 = list() + y_vals_4 = list() + + for test_name, test_vals in throughput.items(): + if test_vals["1"]: + x_vals.append("-".join(test_name.split('-')[:-1])) + y_vals_1.append(1) + if test_vals["2"]: + y_vals_2.append( + round(float(test_vals["2"]) / float(test_vals["1"]), 2)) + else: + y_vals_2.append(None) + if test_vals["4"]: + y_vals_4.append( + round(float(test_vals["4"]) / float(test_vals["1"]), 2)) + else: + y_vals_4.append(None) + + y_vals = [y_vals_1, y_vals_2, y_vals_4] + + y_vals_zipped = zip(names, y_vals) + traces = list() + for val in y_vals_zipped: + traces.append(plgo.Bar(x=x_vals, + y=val[1], + name=val[0])) + + try: + # Create plot + logging.info(" Writing file '{0}{1}'.". + format(plot["output-file"], plot["output-file-type"])) + plpl = plgo.Figure(data=traces, layout=plot["layout"]) + + # Export Plot + ploff.plot(plpl, + show_link=False, auto_open=False, + filename='{0}{1}'.format(plot["output-file"], + plot["output-file-type"])) + except PlotlyError as err: + logging.error(" Finished with error: {}". + format(str(err).replace("\n", " "))) + return + + logging.info(" Done.") diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 96493ed47e..63f7605119 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -87,6 +87,10 @@ - type: "configuration" data-sets: + plot-throughput-speedup-analysis: +# csit-vpp-perf-1801-all: +# - 1 +# - 2 performance-improvements: csit-vpp-perf-1707-all: - 9 @@ -257,6 +261,54 @@ orientation: "h" width: 700 height: 1000 + plot-throughput-speedup-analysis: + xaxis: + # title: "Test Cases" + autorange: True + autotick: False + fixedrange: False + gridcolor: "rgb(238, 238, 238)" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + tickmode: "linear" + tickangle: 270 + zeroline: False + yaxis: + title: "Throughput speedup factor" + gridcolor: "rgb(238, 238, 238)" + hoverformat: ".4s" + linecolor: "rgb(238, 238, 238)" + linewidth: 1 + range: [] + showgrid: True + showline: True + showticklabels: True + tickcolor: "rgb(238, 238, 238)" + zeroline: False + legend: + orientation: "h" + xanchor: "center" + yanchor: "top" + x: 0.5 + y: 1 + bgcolor: "rgba(255, 255, 255, 0)" + bordercolor: "rgba(255, 255, 255, 0)" + barmode: "group" + bargap: 0.15 + bargroupgap: 0.1 + autosize: False + margin: + 't': 50 + 'b': 250 + 'l': 50 + 'r': 20 + showlegend: True + width: 700 + height: 1000 - type: "debug" @@ -265,32 +317,39 @@ extract: "robot-plugin/output.xml" # Only for zip builds: # The files must be in the directory DIR[WORKING,DATA] - csit-dpdk-perf-1707-all: - - - build: 10 - file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml" + csit-vpp-perf-1801-all: - - build: 9 - file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml" - csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: + build: 1 + file: "{DIR[WORKING,DATA]}/output.xml" - build: 2 - file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml" - csit-vpp-functional-1707-ubuntu1604-virl: - - - build: lastSuccessfulBuild - file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml" - hc2vpp-csit-integration-1707-ubuntu1604: - - - build: lastSuccessfulBuild - file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml" - csit-vpp-perf-1707-all: - - - build: 16 - file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml" - - - build: 17 - file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml" + file: "{DIR[WORKING,DATA]}/output.xml" +# csit-dpdk-perf-1707-all: +# - +# build: 10 +# file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml" +# - +# build: 9 +# file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml" +# csit-nsh_sfc-verify-func-1707-ubuntu1604-virl: +# - +# build: 2 +# file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml" +# csit-vpp-functional-1707-ubuntu1604-virl: +# - +# build: lastSuccessfulBuild +# file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml" +# hc2vpp-csit-integration-1707-ubuntu1604: +# - +# build: lastSuccessfulBuild +# file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml" +# csit-vpp-perf-1707-all: +# - +# build: 16 +# file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml" +# - +# build: 17 +# file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml" - type: "static" @@ -812,6 +871,28 @@ ### P L O T S ### ################################################################################ +# Plot Throughput Speedup Analysis + +- + type: "plot" + title: "TODO Throughput Speedup Analysis" + algorithm: "plot_throughput_speedup_analysis" + output-file-type: ".html" + output-file: "{DIR[STATIC,VPP]}/tsa-64B-l2" + data: + "plot-throughput-speedup-analysis" # CHANGE + # Keep this formatting, the filter is enclosed with " (quotation mark) and + # each tag is enclosed with ' (apostrophe). + filter: "('64B' or '78B') and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD' or 'IP4FWD' or 'IP6FWD') and not 'VHOST'" + parameters: + - "throughput" + - "parent" + - "tags" + layout: + title: "TODO Throughput Speedup Analysis ..." + layout: + "plot-throughput-speedup-analysis" + # Plot packets per second # VPP L2 sel1 |