CSIT-913: Continuous Trending, Analysis and Change Detection

 - CSIT-915: LLD
 - CSIT-917: Functions to evaluate the results according to the PASS / FAIL criteria
 - CSIT-918: Sphinx configuration
 - CSIT-948: Statistical functions
 - CSIT-949: Data models for trending plots
 - CSIT-950: Code trending plots
 - CSIT-951: Static content
 - CSIT-984: PAL Specification file
 - CSIT-996: Download data from nexus

Change-Id: Icb9305945bb0f142135bb177cb8781ba0096280e
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 400 insertions, 0 deletions

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
new file mode 100644
index 0000000000..51cdc6324b
--- /dev/null
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -0,0 +1,400 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the specification of parameters for "Continuous Performance Trending
+# and Analysis" feature provided by PAL.
+
+-
+  type: "environment"
+  configuration:
+    # Debug mode:
+    # - Skip:
+    #   - Download of input data files
+    # - Do:
+    #   - Read data from given zip / xml files
+    #   - Set the configuration as it is done in normal mode
+    # If the section "type: debug" is missing, CFG[DEBUG] is set to 0.
+    CFG[DEBUG]: 0
+
+  paths:
+    # Top level directories:
+    ## Working directory
+    DIR[WORKING]: "_tmp"
+    ## Build directories
+    DIR[BUILD,HTML]: "_build"
+    ## Static .rst files
+    DIR[RST]: "../../../docs/cpta"
+
+    # Static html content
+    DIR[STATIC]: "{DIR[BUILD,HTML]}/_static"
+    DIR[STATIC,VPP]: "{DIR[STATIC]}/vpp"
+    # DIR[STATIC,DPDK]: "{DIR[STATIC]}/dpdk"
+    DIR[STATIC,ARCH]: "{DIR[STATIC]}/archive"
+
+    # Working directories
+    ## Input data files (.zip, .xml)
+    DIR[WORKING,DATA]: "{DIR[WORKING]}/data"
+    ## Static source files from git
+    DIR[WORKING,SRC]: "{DIR[WORKING]}/src"
+    DIR[WORKING,SRC,STATIC]: "{DIR[WORKING,SRC]}/_static"
+
+    # .css patch file
+    DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css"
+    DIR[CSS_PATCH_FILE2]: "{DIR[WORKING,SRC,STATIC]}/theme_overrides.css"
+
+  urls:
+    URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
+    URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+    URL[NEXUS]: "https://docs.fd.io/csit"
+    DIR[NEXUS]: "report/_static/archive"
+
+  make-dirs:
+  # List the directories which are created while preparing the environment.
+  # All directories MUST be defined in "paths" section.
+  - "DIR[WORKING,DATA]"
+  - "DIR[WORKING,SRC,STATIC]"
+  - "DIR[BUILD,HTML]"
+  - "DIR[STATIC,VPP]"
+  - "DIR[STATIC,ARCH]"
+  build-dirs:
+  # List the directories where the results (build) is stored.
+  # All directories MUST be defined in "paths" section.
+  - "DIR[BUILD,HTML]"
+
+-
+  type: "configuration"
+
+  data-sets:
+# TODO: Specify input data, this is only an example:
+    plot-performance-trending:
+      csit-vpp-perf-mrr-daily-master:
+        start: 9
+        end: 14 # "lastSuccessfulBuild"  # take all from the 'start'
+
+  plot-layouts:
+    plot-cpta:
+      title: ""
+      autosize: False
+      showlegend: True
+      width: 1100
+      height: 800
+      yaxis:
+        showticklabels: True
+        title: "Throughput [Mpps]"
+        hoverformat: ".4s"
+        range: []
+        gridcolor: "rgb(238, 238, 238)"
+        linecolor: "rgb(238, 238, 238)"
+        showline: True
+        zeroline: False
+        tickcolor: "rgb(238, 238, 238)"
+        linewidth: 1
+        showgrid: True
+      xaxis:
+        showticklabels: True
+        title: "VPP Performance Trending Job ID"
+        autorange: True
+        showgrid: True
+        gridcolor: "rgb(238, 238, 238)"
+        linecolor: "rgb(238, 238, 238)"
+        fixedrange: False
+        zeroline: False
+        tickcolor: "rgb(238, 238, 238)"
+        showline: True
+        linewidth: 1
+        autotick: True
+      margin:
+        r: 20
+        b: 50
+        t: 50
+        l: 70
+      legend:
+        orientation: "h"
+        traceorder: "normal"
+#        tracegroupgap: 10
+#        bordercolor: "rgb(238, 238, 238)"
+#        borderwidth: 1
+
+-
+  type: "debug"
+  general:
+    input-format: "xml"  # zip or xml
+    extract: "robot-plugin/output.xml"  # Only for zip
+  builds:
+    # The files must be in the directory DIR[WORKING,DATA]
+    csit-vpp-perf-mrr-daily-master:
+    -
+      build: 1
+      file: "{DIR[WORKING,DATA]}/output_mrr_1.xml"
+    -
+      build: 2
+      file: "{DIR[WORKING,DATA]}/output_mrr_2.xml"
+    -
+      build: 3
+      file: "{DIR[WORKING,DATA]}/output_mrr_3.xml"
+    -
+      build: 4
+      file: "{DIR[WORKING,DATA]}/output_mrr_4.xml"
+    -
+      build: 5
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 6
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 7
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 8
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 9
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 10
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 11
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+    -
+      build: 12
+      file: "{DIR[WORKING,DATA]}/output_mrr_5.xml"
+
+-
+  type: "static"
+  src-path: "{DIR[RST]}"
+  dst-path: "{DIR[WORKING,SRC]}"
+
+-
+  type: "input"  # Ignored in debug mode
+  general:
+    file-name: "output.xml.log.gz"
+    file-format: ".gz"
+    download-path: "{job}/{build}/archives/{filename}"
+    extract: "output.xml"
+#    file-name: "robot-plugin.zip"
+#    file-format: ".zip"
+#    download-path: "{job}/{build}/robot/report/*zip*/{filename}"
+#    extract: "robot-plugin/output.xml"
+  builds:
+    csit-vpp-perf-mrr-daily-master:
+      start: 9
+      end: 14 #"lastSuccessfulBuild"  # take all from the 'start'
+#    csit-vpp-perf-check-master:
+#      start: 22
+#      end: 22
+
+-
+  type: "output"
+  output:
+#   "report"
+    "CPTA"  # Continuous Performance Trending and Analysis
+  format:
+    html:
+    - full
+    pdf:
+    - minimal
+
+################################################################################
+###                                 C P T A                                  ###
+################################################################################
+
+# Plots VPP Continuous Performance Trending and Analysis
+-
+  type: "cpta"
+  title: "Continuous Performance Trending and Analysis"
+  algorithm: "cpta"
+  output-file-type: ".html"
+  output-file: "{DIR[STATIC,VPP]}/cpta"
+  plots:
+
+# L2
+
+    - title: "VPP 1T1C L2 64B Packet Throughput - {period} Trending"
+      output-file-name: "l2"
+      data: "plot-performance-trending"
+      filter: "'MRR' and '64B' and ('BASE' or 'SCALE') and '1T1C' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST' and not 'MEMIF'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+# IPv4
+
+    - title: "VPP 1T1C IPv4 64B Packet Throughput - {period} Trending"
+      output-file-name: "ip4"
+      data: "plot-performance-trending"
+      filter: "'MRR' and '64B' and ('BASE' or 'SCALE' or 'FEATURE') and '1T1C' and 'IP4FWD' and not 'IPSEC' and not 'VHOST'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+# IPv6
+
+    - title: "VPP 1T1C IPv6 78B Packet Throughput - {period} Trending"
+      output-file-name: "ip6"
+      data: "plot-performance-trending"
+      filter: "'MRR' and '78B' and ('BASE' or 'SCALE' or 'FEATURE') and '1T1C' and 'IP6FWD' and not 'IPSEC' and not 'VHOST'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+# Container memif
+
+    - title: "VPP 1T1C L2 Container memif 64B Packet Throughput - {period} Trending"
+      output-file-name: "container-memif-l2-1t1c-x520"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-X520-DA2' and 'MRR' and '64B' and 'BASE' and '1T1C' and 'MEMIF' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 2T2C L2 Container memif 64B Packet Throughput - {period} Trending"
+      output-file-name: "container-memif-l2-2t2c-x520"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-X520-DA2' and 'MRR' and '64B' and 'BASE' and '2T2C' and 'MEMIF' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 1T1C L2 Container memif 64B Packet Throughput - {period} Trending"
+      output-file-name: "container-memif-l2-1t1c-xl710"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-XL710' and 'MRR' and '64B' and 'BASE' and '1T1C' and 'MEMIF' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 2T2C L2 Container memif 64B Packet Throughput - {period} Trending"
+      output-file-name: "container-memif-l2-2t2c-xl710"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-XL710' and 'MRR' and '64B' and 'BASE' and '2T2C' and 'MEMIF' and ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD') and not 'VHOST'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+# VM vhost
+
+    - title: "VPP 1T1C VM vhost ethip4 64B Packet Throughput - {period} Trending"
+      output-file-name: "vm-vhost-ethip4-1t1c-x520"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-X520-DA2' and '64B' and 'MRR' and '1T1C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 2T2C VM vhost ethip4 64B Packet Throughput - {period} Trending"
+      output-file-name: "vm-vhost-ethip4-2t2c-x520"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-X520-DA2' and '64B' and 'MRR' and '2T2C' and 'VHOST' and not ('L2BDMACSTAT' or 'L2BDMACLRN' or 'L2XCFWD')"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 1T1C VM vhost eth 64B Packet Throughput - {period} Trending"
+      output-file-name: "vm-vhost-eth-1t1c-x520"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-X520-DA2' and '64B' and 'MRR' and '1T1C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'"
+
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 2T2C VM vhost eth 64B Packet Throughput - {period} Trending"
+      output-file-name: "vm-vhost-eth-2t2c-x520"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-X520-DA2' and '64B' and 'MRR' and '2T2C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 1T1C VM vhost eth 64B Packet Throughput - {period} Trending"
+      output-file-name: "vm-vhost-eth-1t1c-xl710"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-XL710' and '64B' and 'MRR' and '1T1C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'"
+
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"
+
+    - title: "VPP 2T2C VM vhost eth 64B Packet Throughput - {period} Trending"
+      output-file-name: "vm-vhost-eth-2t2c-xl710"
+      data: "plot-performance-trending"
+      filter: "'NIC_Intel-XL710' and '64B' and 'MRR' and '2T2C' and 'VHOST' and not 'VXLAN' and not 'IP4FWD' and not 'DOT1Q' and not '2VM'"
+      parameters:
+      - "result"
+      - "name"
+      periods:
+      - 1
+      - 5
+      - 30
+      layout: "plot-cpta"