aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2018-01-16 13:07:02 +0100
committerTibor Frank <tifrank@cisco.com>2018-01-19 13:50:20 +0000
commitaef6479f9bdc6ef2cf3acddb7c75bdb6c039b186 (patch)
tree674ad9ef42998e58210727ea58947c2774cfb64c
parent8215cf7caa6f6fc1d78971f78a934499f3998a4c (diff)
CSIT-887: Throughput Speedup Analysis
- CSIT-894: LLD - CSIT-900: Plot layout - CSIT 888: Data model - CSIT-889: Algorithm Change-Id: I569107b474d216533568c1c731c50dcf7b706cd4 Signed-off-by: Tibor Frank <tifrank@cisco.com> (cherry picked from commit 0f06aa38489e4242ee2b51bc23f79682a8f658f9)
-rw-r--r--resources/tools/presentation/doc/pal_lld.rst140
-rw-r--r--resources/tools/presentation/generator_plots.py98
-rw-r--r--resources/tools/presentation/specification.yaml127
3 files changed, 338 insertions, 27 deletions
diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst
index 9158b889b9..12318ccfe9 100644
--- a/resources/tools/presentation/doc/pal_lld.rst
+++ b/resources/tools/presentation/doc/pal_lld.rst
@@ -83,6 +83,8 @@ the type:
-
type: "environment"
-
+ type: "configuration"
+ -
type: "debug"
-
type: "static"
@@ -123,6 +125,7 @@ This section has the following parts:
- build-dirs - a list of the directories where the results are stored.
The structure of the section "Environment" is as follows (example):
+
::
-
@@ -223,6 +226,108 @@ will be automatically changed to
DIR[WORKING,DATA]: "_tmp/data"
+Section: Configuration
+''''''''''''''''''''''
+
+This section specifies the groups of parameters which are repeatedly used in the
+elements defined later in the specification file. It has the following parts:
+
+ - data sets - Specification of data sets used later in element's specifications
+ to define the input data.
+ - plot layouts - Specification of plot layouts used later in plots'
+ specifications to define the plot layout.
+
+The structure of the section "Configuration" is as follows (example):
+
+::
+
+ -
+ type: "configuration"
+ data-sets:
+ plot-vpp-throughput-latency:
+ csit-vpp-perf-1710-all:
+ - 11
+ - 12
+ - 13
+ - 14
+ - 15
+ - 16
+ - 17
+ - 18
+ - 19
+ - 20
+ vpp-perf-results:
+ csit-vpp-perf-1710-all:
+ - 20
+ - 23
+ plot-layouts:
+ plot-throughput:
+ xaxis:
+ autorange: True
+ autotick: False
+ fixedrange: False
+ gridcolor: "rgb(238, 238, 238)"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ tickmode: "linear"
+ title: "Indexed Test Cases"
+ zeroline: False
+ yaxis:
+ gridcolor: "rgb(238, 238, 238)'"
+ hoverformat: ".4s"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ range: []
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ title: "Packets Per Second [pps]"
+ zeroline: False
+ boxmode: "group"
+ boxgroupgap: 0.5
+ autosize: False
+ margin:
+ t: 50
+ b: 20
+ l: 50
+ r: 20
+ showlegend: True
+ legend:
+ orientation: "h"
+ width: 700
+ height: 1000
+
+The definitions from this sections are used in the elements, e.g.:
+
+::
+
+ -
+ type: "plot"
+ title: "VPP Performance 64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc"
+ algorithm: "plot_performance_box"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc"
+ data:
+ "plot-vpp-throughput-latency"
+ filter: "'64B' and ('BASE' or 'SCALE') and 'NDRDISC' and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ traces:
+ hoverinfo: "x+y"
+ boxpoints: "outliers"
+ whiskerwidth: 0
+ layout:
+ title: "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc"
+ layout:
+ "plot-throughput"
+
+
Section: Debug mode
'''''''''''''''''''
@@ -1037,16 +1142,45 @@ Data analytics part implements:
- etc.
+Throughput Speedup Analysis - Multi-Core Speedup Ratio
+''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+Throughput Speedup Analysis (TSA) calculates a speedup factor for 1, 2, 4 cores
+which is defined as:
+
+::
+
+ throughput
+ speedup factor = -----------------
+ 1-core-throughput
+
+A bar plot displays the speedup factor (normalized throughput for 64B/78B on 1
+core). The plot displays number of cores on the X-axis and the speedup factor on
+the Y-axis.
+
+For better comparision, there can be displayed more than one set of data in a
+plot. So, in general:
+
+ - graph type: grouped bars;
+ - graph X-axis: (testcase index, number of cores);
+ - graph Y-axis: speedup factor.
+
+The data displayed is a subset of existing performance tests with 1core, 2core,
+4core.
+
+:TODO: Specify the data model for TSA.
+
+
Advanced data analytics
```````````````````````
As the next steps, advanced data analytics (ADA) will be implemented using
machine learning (ML) and artificial intelligence (AI).
-TODO:
+:TODO:
- - describe the concept of ADA.
- - add specification.
+ - describe the concept of ADA.
+ - add specification.
Data presentation
diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index 9ef8574907..5580cbdf1c 100644
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -219,3 +219,99 @@ def plot_latency_box(plot, input_data):
return
logging.info(" Done.")
+
+
+def plot_throughput_speedup_analysis(plot, input_data):
+ """Generate the plot(s) with algorithm: plot_throughput_speedup_analysis
+ specified in the specification file.
+
+ :param plot: Plot to generate.
+ :param input_data: Data to process.
+ :type plot: pandas.Series
+ :type input_data: InputData
+ """
+
+ logging.info(" Generating the plot {0} ...".
+ format(plot.get("title", "")))
+
+ # Transform the data
+ data = input_data.filter_data(plot)
+ if data is None:
+ logging.error("No data.")
+ return
+
+ throughput = dict()
+ for job in data:
+ for build in job:
+ for test in build:
+ if throughput.get(test["parent"], None) is None:
+ throughput[test["parent"]] = {"1": list(),
+ "2": list(),
+ "4": list()}
+ try:
+ if "1T1C" in test["tags"]:
+ throughput[test["parent"]]["1"].\
+ append(test["throughput"]["value"])
+ elif "2T2C" in test["tags"]:
+ throughput[test["parent"]]["2"]. \
+ append(test["throughput"]["value"])
+ elif "4T4C" in test["tags"]:
+ throughput[test["parent"]]["4"]. \
+ append(test["throughput"]["value"])
+ except (KeyError, TypeError):
+ pass
+
+ for test_name, test_vals in throughput.items():
+ for key, test_val in test_vals.items():
+ if test_val:
+ throughput[test_name][key] = sum(test_val) / len(test_val)
+
+ print(throughput)
+
+ names = ['1 core', '2 cores', '4 cores']
+ x_vals = list()
+ y_vals_1 = list()
+ y_vals_2 = list()
+ y_vals_4 = list()
+
+ for test_name, test_vals in throughput.items():
+ if test_vals["1"]:
+ x_vals.append("-".join(test_name.split('-')[:-1]))
+ y_vals_1.append(1)
+ if test_vals["2"]:
+ y_vals_2.append(
+ round(float(test_vals["2"]) / float(test_vals["1"]), 2))
+ else:
+ y_vals_2.append(None)
+ if test_vals["4"]:
+ y_vals_4.append(
+ round(float(test_vals["4"]) / float(test_vals["1"]), 2))
+ else:
+ y_vals_4.append(None)
+
+ y_vals = [y_vals_1, y_vals_2, y_vals_4]
+
+ y_vals_zipped = zip(names, y_vals)
+ traces = list()
+ for val in y_vals_zipped:
+ traces.append(plgo.Bar(x=x_vals,
+ y=val[1],
+ name=val[0]))
+
+ try:
+ # Create plot
+ logging.info(" Writing file '{0}{1}'.".
+ format(plot["output-file"], plot["output-file-type"]))
+ plpl = plgo.Figure(data=traces, layout=plot["layout"])
+
+ # Export Plot
+ ploff.plot(plpl,
+ show_link=False, auto_open=False,
+ filename='{0}{1}'.format(plot["output-file"],
+ plot["output-file-type"]))
+ except PlotlyError as err:
+ logging.error(" Finished with error: {}".
+ format(str(err).replace("\n", " ")))
+ return
+
+ logging.info(" Done.")
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 96493ed47e..63f7605119 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -87,6 +87,10 @@
-
type: "configuration"
data-sets:
+ plot-throughput-speedup-analysis:
+# csit-vpp-perf-1801-all:
+# - 1
+# - 2
performance-improvements:
csit-vpp-perf-1707-all:
- 9
@@ -257,6 +261,54 @@
orientation: "h"
width: 700
height: 1000
+ plot-throughput-speedup-analysis:
+ xaxis:
+ # title: "Test Cases"
+ autorange: True
+ autotick: False
+ fixedrange: False
+ gridcolor: "rgb(238, 238, 238)"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ tickmode: "linear"
+ tickangle: 270
+ zeroline: False
+ yaxis:
+ title: "Throughput speedup factor"
+ gridcolor: "rgb(238, 238, 238)"
+ hoverformat: ".4s"
+ linecolor: "rgb(238, 238, 238)"
+ linewidth: 1
+ range: []
+ showgrid: True
+ showline: True
+ showticklabels: True
+ tickcolor: "rgb(238, 238, 238)"
+ zeroline: False
+ legend:
+ orientation: "h"
+ xanchor: "center"
+ yanchor: "top"
+ x: 0.5
+ y: 1
+ bgcolor: "rgba(255, 255, 255, 0)"
+ bordercolor: "rgba(255, 255, 255, 0)"
+ barmode: "group"
+ bargap: 0.15
+ bargroupgap: 0.1
+ autosize: False
+ margin:
+ 't': 50
+ 'b': 250
+ 'l': 50
+ 'r': 20
+ showlegend: True
+ width: 700
+ height: 1000
-
type: "debug"
@@ -265,32 +317,39 @@
extract: "robot-plugin/output.xml" # Only for zip
builds:
# The files must be in the directory DIR[WORKING,DATA]
- csit-dpdk-perf-1707-all:
- -
- build: 10
- file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml"
+ csit-vpp-perf-1801-all:
-
- build: 9
- file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml"
- csit-nsh_sfc-verify-func-1707-ubuntu1604-virl:
+ build: 1
+ file: "{DIR[WORKING,DATA]}/output.xml"
-
build: 2
- file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml"
- csit-vpp-functional-1707-ubuntu1604-virl:
- -
- build: lastSuccessfulBuild
- file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml"
- hc2vpp-csit-integration-1707-ubuntu1604:
- -
- build: lastSuccessfulBuild
- file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml"
- csit-vpp-perf-1707-all:
- -
- build: 16
- file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml"
- -
- build: 17
- file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml"
+ file: "{DIR[WORKING,DATA]}/output.xml"
+# csit-dpdk-perf-1707-all:
+# -
+# build: 10
+# file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__10.xml"
+# -
+# build: 9
+# file: "{DIR[WORKING,DATA]}/csit-dpdk-perf-1707-all__9.xml"
+# csit-nsh_sfc-verify-func-1707-ubuntu1604-virl:
+# -
+# build: 2
+# file: "{DIR[WORKING,DATA]}/csit-nsh_sfc-verify-func-1707-ubuntu1604-virl-2.xml"
+# csit-vpp-functional-1707-ubuntu1604-virl:
+# -
+# build: lastSuccessfulBuild
+# file: "{DIR[WORKING,DATA]}/csit-vpp-functional-1707-ubuntu1604-virl-lastSuccessfulBuild.xml"
+# hc2vpp-csit-integration-1707-ubuntu1604:
+# -
+# build: lastSuccessfulBuild
+# file: "{DIR[WORKING,DATA]}/hc2vpp-csit-integration-1707-ubuntu1604-lastSuccessfulBuild.xml"
+# csit-vpp-perf-1707-all:
+# -
+# build: 16
+# file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__16__output.xml"
+# -
+# build: 17
+# file: "{DIR[WORKING,DATA]}/csit-vpp-perf-1707-all__17__output.xml"
-
type: "static"
@@ -812,6 +871,28 @@
### P L O T S ###
################################################################################
+# Plot Throughput Speedup Analysis
+
+-
+ type: "plot"
+ title: "TODO Throughput Speedup Analysis"
+ algorithm: "plot_throughput_speedup_analysis"
+ output-file-type: ".html"
+ output-file: "{DIR[STATIC,VPP]}/tsa-64B-l2"
+ data:
+ "plot-throughput-speedup-analysis" # CHANGE
+ # Keep this formatting, the filter is enclosed with " (quotation mark) and
+ # each tag is enclosed with ' (apostrophe).
+ filter: "('64B' or '78B') and 'BASE' and 'NDRDISC' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD' or 'IP4FWD' or 'IP6FWD') and not 'VHOST'"
+ parameters:
+ - "throughput"
+ - "parent"
+ - "tags"
+ layout:
+ title: "TODO Throughput Speedup Analysis ..."
+ layout:
+ "plot-throughput-speedup-analysis"
+
# Plot packets per second
# VPP L2 sel1