PAL: Refactor the processing of spec and download

Change-Id: I204fdc3593a3702366727238f2b469d3d4320c2b
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 265 insertions, 0 deletions

diff --git a/resources/tools/presentation/specifications/trending/environment.yaml b/resources/tools/presentation/specifications/trending/environment.yaml
new file mode 100644
index 0000000000..dfa9f680c2
--- /dev/null
+++ b/resources/tools/presentation/specifications/trending/environment.yaml
@@ -0,0 +1,265 @@
+################################################################################
+###                          E N V I R O N M E N T                           ###
+################################################################################
+
+- type: "environment"
+
+  spec-files:
+    - "specifications/trending/data_sets.yaml"
+    - "specifications/trending/layouts.yaml"
+    - "specifications/trending/elements.yaml"
+
+  paths:
+    # Top level directories:
+    ## Working directory
+    DIR[WORKING]: "_tmp"
+    ## Build directories
+    DIR[BUILD,HTML]: "_build"
+    ## Static .rst files
+    DIR[RST]: "../../../docs/cpta"
+
+    # Static html content
+    DIR[STATIC]: "{DIR[BUILD,HTML]}/_static"
+    DIR[STATIC,VPP]: "{DIR[STATIC]}/vpp"
+    DIR[STATIC,ARCH]: "{DIR[STATIC]}/archive"
+
+    # Working directories
+    ## Input data files (.zip, .xml)
+    DIR[WORKING,DATA]: "{DIR[WORKING]}/data"
+    ## Static source files from git
+    DIR[WORKING,SRC]: "{DIR[WORKING]}/src"
+    DIR[WORKING,SRC,STATIC]: "{DIR[WORKING,SRC]}/_static"
+
+    # .css patch file
+    DIR[CSS_PATCH_FILE]: "{DIR[STATIC]}/theme_overrides.css"
+    DIR[CSS_PATCH_FILE2]: "{DIR[WORKING,SRC,STATIC]}/theme_overrides.css"
+
+  urls:
+    URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
+
+  make-dirs:
+  # List the directories which are created while preparing the environment.
+  # All directories MUST be defined in "paths" section.
+  - "DIR[WORKING,DATA]"
+  - "DIR[WORKING,SRC,STATIC]"
+  - "DIR[BUILD,HTML]"
+  - "DIR[STATIC,VPP]"
+  - "DIR[STATIC,ARCH]"
+  build-dirs:
+  # List the directories where the results (build) is stored.
+  # All directories MUST be defined in "paths" section.
+  - "DIR[BUILD,HTML]"
+
+  testbeds:
+    "10.30.51.45": "LF-2n-SKX-21"
+    "10.30.51.53": "LF-2n-SKX-22"
+    "10.30.51.55": "LF-2n-SKX-23"
+    "10.30.51.57": "LF-2n-SKX-24"
+    "10.32.8.19": "LF-2n-CLX-27"
+    "10.32.8.21": "LF-2n-CLX-28"
+    "10.32.8.23": "LF-2n-CLX-29"
+    "10.30.51.16": "LF-3n-HSW-01"
+    "10.30.51.20": "LF-3n-HSW-02"
+    "10.30.51.24": "LF-3n-HSW-03"
+    "10.30.51.48": "LF-3n-SKX-31"
+    "10.30.51.60": "LF-3n-SKX-32"
+    "10.30.51.49": "LF-3n-TSH-33"
+    "10.30.51.49": "LF-2n-TX2-34"
+    "10.32.8.25": "LF-2n-ZN2-210"
+    "10.32.8.10": "LF-DNV"
+
+  # Data sources are used in this order:
+  data-sources:
+    # JSON from S3
+    - type: "json"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output.json.gz"
+      file-format: ".gz"
+      enabled: False
+    # XML
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+
+  archive-inputs: False
+
+  reverse-input: True  # Needed for trending, not important for the report
+
+  # Number of days from now to the past. Only files generated in this
+  # time period are used.
+  # Only for trending
+  time-period: 180
+
+  mapping-file: ""
+
+  ignore-list: ""
+
+  alerting:
+
+    alerts:
+
+# As Jenkins slave is not configured to send emails, this is now only as
+# a working example:
+#
+#      # Send the list of failed tests vie email.
+#      # Pre-requisites:
+#      # - SMTP server is installed on the Jenkins slave
+#      # - SMTP server is configured to send emails. Default configuration is
+#      #   sufficient.
+#      email-failed-tests:
+#        # Title is used in logs and also as the email subject.
+#        title: "Trending: Failed Tests"
+#        # Type of alert.
+#        type: "failed-tests"
+#        # How to send the alert. The used way must be specified in the
+#        # configuration part.
+#        way: "email"
+#        # Data to be included in the alert.
+#        # Here is used the list of tables generated by the function
+#        # "table_failed_tests_html".
+#        include:
+#        - "failed-tests-3n-hsw"
+#        - "failed-tests-3n-skx"
+#        - "failed-tests-2n-skx"
+#        # This url is used in the tables instead of the original one. The aim
+#        # is to make the links usable also from the email.
+#        url: "https://docs.fd.io/csit/master/trending/trending"
+
+      # Jenkins job sends the email with failed tests.
+      # Pre-requisites:
+      # - Jenkins job is configured to send emails in "Post-build Actions" -->
+      #   "Editable Email Notification".
+      jenkins-send-failed-tests:
+        title: "Trending: Failed Tests"
+        type: "failed-tests"
+        way: "jenkins"
+        include:
+          - "last-failed-tests-vpp-2n-skx-mrr"
+          - "last-failed-tests-vpp-3n-skx-mrr"
+          - "last-failed-tests-vpp-2n-clx-mrr"
+          - "last-failed-tests-vpp-2n-zn2-mrr"
+          - "last-failed-tests-vpp-3n-hsw-mrr"
+          - "last-failed-tests-vpp-3n-tsh-mrr"
+          - "last-failed-tests-vpp-2n-tx2-mrr"
+          - "last-failed-tests-vpp-2n-dnv-mrr"
+          - "last-failed-tests-vpp-3n-dnv-mrr"
+          - "last-failed-tests-dpdk-2n-skx-mrr"
+          - "last-failed-tests-dpdk-3n-skx-mrr"
+          - "last-failed-tests-dpdk-2n-clx-mrr"
+          - "last-failed-tests-dpdk-3n-hsw-mrr"
+          - "last-failed-tests-dpdk-3n-tsh-mrr"
+          - "last-failed-tests-dpdk-2n-tx2-mrr"
+          - "last-failed-tests-vpp-2n-skx-ndrpdr"
+          - "last-failed-tests-vpp-3n-skx-ndrpdr"
+          - "last-failed-tests-vpp-2n-clx-ndrpdr"
+          - "last-failed-tests-vpp-3n-hsw-ndrpdr"
+        urls:
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-clx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-zn2"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-tsh"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-tx2"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-dnv"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-dnv"
+          - "https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-mrr-weekly-master-2n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-mrr-weekly-master-3n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-mrr-weekly-master-2n-clx"
+          - "https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-mrr-weekly-master"
+          - "https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-mrr-weekly-master-3n-tsh"
+          - "https://jenkins.fd.io/view/csit/job/csit-dpdk-perf-mrr-weekly-master-2n-tx2"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-2n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-3n-skx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-2n-clx"
+          - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-ndrpdr-weekly-master-3n-hsw"
+        url-details: "https://docs.fd.io/csit/master/trending/introduction/failures.html"
+
+    configurations:
+      # Configuration of the email notifications.
+      email:
+        # SMTP server
+        server: "localhost"
+        # List of recipients.
+        address-to:
+        - "csit-report@lists.fd.io"
+        # Sender
+        address-from: "testuser@testserver.com"
+
+      # Configuration of notifications sent by Jenkins.
+      jenkins:
+        # The directory in the workspace where the generated data is stored and
+        # then read by Jenkins job.
+        output-dir: "_build/_static/vpp"
+        # The name of the output files. ASCII text and HTML formats are
+        # generated.
+        output-file: "trending-failed-tests"
+        # If defined, all output files are zipped into this file.
+        zip-output: "failed-tests.tar.gz"
+
+################################################################################
+###                               S T A T I C                                ###
+################################################################################
+
+- type: "static"
+  src-path: "{DIR[RST]}"
+  dst-path: "{DIR[WORKING,SRC]}"
+
+################################################################################
+###                               O U T P U T                                ###
+################################################################################
+
+- type: "output"
+  arch-file-format:  # moved from input, TODO: change it in the code
+    - ".gz"
+    - ".zip"
+  output: "trending"
+  format:
+    html:
+    - full