PAL: Convert XML to JSON

Change-Id: I24f0ddc412d4353ba244c58a3068b5b0ea4349e3
Signed-off-by: Tibor Frank <tifrank@cisco.com>

4 files changed, 148 insertions, 4 deletions

diff --git a/resources/tools/presentation/specifications/converter/environment.yaml b/resources/tools/presentation/specifications/converter/environment.yaml
new file mode 100644
index 0000000000..1f57445638
--- /dev/null
+++ b/resources/tools/presentation/specifications/converter/environment.yaml
@@ -0,0 +1,124 @@
+################################################################################
+###                          E N V I R O N M E N T                           ###
+################################################################################
+
+- type: "environment"
+
+  spec-files:
+    - "specifications/converter/input.yaml"  # Only for converter XML --> JSON
+
+  paths:
+    # Top level directories:
+    ## Working directory
+    DIR[WORKING]: "_tmp"
+    ## Build directories
+    DIR[BUILD,JSON]: "_build"
+
+    # Working directories
+    ## Input data files (.zip, .xml)
+    DIR[WORKING,DATA]: "{DIR[WORKING]}/data"
+
+  # Data sources are used in this order:
+  data-sources:
+    # JSON from S3
+    - type: "json"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output.json.gz"
+      file-format: ".gz"
+      enabled: False
+    # XML
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/archives/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml"
+      url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+      path: "{job}/{build}/{filename}"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    # XML from docs.nexus
+    - type: "xml-docs"
+      url: "https://docs.fd.io/csit"
+      path: "report/_static/archive"
+      file-name: "output_info.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml-docs"
+      url: "https://docs.fd.io/csit"
+      path: "report/_static/archive"
+      file-name: "output.xml.gz"
+      file-format: ".gz"
+      enabled: True
+    - type: "xml-docs"
+      url: "https://docs.fd.io/csit"
+      path: "report/_static/archive"
+      file-name: "robot-plugin.zip"
+      file-format: ".zip"
+      enabled: True
+
+  make-dirs:
+  # List the directories which are created while preparing the environment.
+  # All directories MUST be defined in "paths" section.
+    - "DIR[WORKING,DATA]"
+
+  remove-dirs:
+  # List the directories which are deleted while cleaning the environment.
+  # All directories MUST be defined in "paths" section.
+    - "DIR[WORKING,DATA]"
+
+  build-dirs:
+  # List the directories where the results (build) is stored.
+  # All directories MUST be defined in "paths" section.
+    - "DIR[BUILD,JSON]"
+
+################################################################################
+###                               O U T P U T                                ###
+################################################################################
+
+- type: "output"
+  output: "convert-xml-to-json"
+  # type: flat | structured
+  # - flat - all .gz files in one directory
+  # - structured - .gz files in directories structured as job/build/*.gz
+  structure: "tree"  # Use flat or tree
+  use-template: "json/template_0.1.0.json"
diff --git a/resources/tools/presentation/specifications/converter/input.yaml b/resources/tools/presentation/specifications/converter/input.yaml
new file mode 100644
index 0000000000..0cf765030d
--- /dev/null
+++ b/resources/tools/presentation/specifications/converter/input.yaml
@@ -0,0 +1,21 @@
+################################################################################
+###                      I N P U T   X M L   F I L E S                       ###
+################################################################################
+
+# This is only an example for converter XML --> JSON
+
+- type: "input"
+
+  # 3n-hsw
+
+  builds:
+    csit-vpp-perf-report-iterative-2101-3n-hsw:
+      - 65  # rls2101.rel NDRPDR reconf iter env 6
+      - 69  # rls2101.rel Hoststack iter env 6
+      - 64  # rls2101.rel NDRPDR iter env 6
+      - 63  # rls2101.rel MRR iter env 6
+    csit-vpp-perf-report-iterative-2101-2n-skx:
+      - 94  # rls2101.rel NDRPDR iter env 6
+      - 68  # rls2101.rel soak env 6
+    csit-vpp-device-2101-ubuntu1804-1n-skx:
+      - 358  # rls2101.rel VPP DEV env 6
diff --git a/resources/tools/presentation/specifications/report/environment.yaml b/resources/tools/presentation/specifications/report/environment.yaml
index 0e946046b4..10d61f56e4 100644
--- a/resources/tools/presentation/specifications/report/environment.yaml
+++ b/resources/tools/presentation/specifications/report/environment.yaml
@@ -193,7 +193,7 @@
       file-format: ".zip"
       enabled: True
 
-  archive-inputs: True
+  archive-inputs: False
 
   mapping-file: ""
 
@@ -224,7 +224,6 @@
 
   reverse-input: False  # Needed for trending, not important for the report
 
-  # TODO: Change in code needed, it was in type: "configuration"
   limits:
     nic:
       x520: 24460000
@@ -258,7 +257,7 @@
 ################################################################################
 
 - type: "output"
-  arch-file-format:  # moved from input, TODO: change it in the code
+  arch-file-format:
     - ".gz"
     - ".zip"
   output: "report"
diff --git a/resources/tools/presentation/specifications/trending/environment.yaml b/resources/tools/presentation/specifications/trending/environment.yaml
index dfa9f680c2..95eaa7b606 100644
--- a/resources/tools/presentation/specifications/trending/environment.yaml
+++ b/resources/tools/presentation/specifications/trending/environment.yaml
@@ -256,7 +256,7 @@
 ################################################################################
 
 - type: "output"
-  arch-file-format:  # moved from input, TODO: change it in the code
+  arch-file-format:
     - ".gz"
     - ".zip"
   output: "trending"