From c763cfcb064e4f4acf6b8309b08d3800b9bd5331 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Mon, 3 May 2021 14:22:31 +0200 Subject: PAL: Convert XML to JSON Change-Id: I24f0ddc412d4353ba244c58a3068b5b0ea4349e3 Signed-off-by: Tibor Frank --- .../specifications/converter/environment.yaml | 124 +++++++++++++++++++++ .../specifications/converter/input.yaml | 21 ++++ .../specifications/report/environment.yaml | 5 +- .../specifications/trending/environment.yaml | 2 +- 4 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 resources/tools/presentation/specifications/converter/environment.yaml create mode 100644 resources/tools/presentation/specifications/converter/input.yaml (limited to 'resources/tools/presentation/specifications') diff --git a/resources/tools/presentation/specifications/converter/environment.yaml b/resources/tools/presentation/specifications/converter/environment.yaml new file mode 100644 index 0000000000..1f57445638 --- /dev/null +++ b/resources/tools/presentation/specifications/converter/environment.yaml @@ -0,0 +1,124 @@ +################################################################################ +### E N V I R O N M E N T ### +################################################################################ + +- type: "environment" + + spec-files: + - "specifications/converter/input.yaml" # Only for converter XML --> JSON + + paths: + # Top level directories: + ## Working directory + DIR[WORKING]: "_tmp" + ## Build directories + DIR[BUILD,JSON]: "_build" + + # Working directories + ## Input data files (.zip, .xml) + DIR[WORKING,DATA]: "{DIR[WORKING]}/data" + + # Data sources are used in this order: + data-sources: + # JSON from S3 + - type: "json" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output.json.gz" + file-format: ".gz" + enabled: False + # XML + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/archives/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml" + url: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + path: "{job}/{build}/{filename}" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + # XML from docs.nexus + - type: "xml-docs" + url: "https://docs.fd.io/csit" + path: "report/_static/archive" + file-name: "output_info.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml-docs" + url: "https://docs.fd.io/csit" + path: "report/_static/archive" + file-name: "output.xml.gz" + file-format: ".gz" + enabled: True + - type: "xml-docs" + url: "https://docs.fd.io/csit" + path: "report/_static/archive" + file-name: "robot-plugin.zip" + file-format: ".zip" + enabled: True + + make-dirs: + # List the directories which are created while preparing the environment. + # All directories MUST be defined in "paths" section. + - "DIR[WORKING,DATA]" + + remove-dirs: + # List the directories which are deleted while cleaning the environment. + # All directories MUST be defined in "paths" section. + - "DIR[WORKING,DATA]" + + build-dirs: + # List the directories where the results (build) is stored. + # All directories MUST be defined in "paths" section. + - "DIR[BUILD,JSON]" + +################################################################################ +### O U T P U T ### +################################################################################ + +- type: "output" + output: "convert-xml-to-json" + # type: flat | structured + # - flat - all .gz files in one directory + # - structured - .gz files in directories structured as job/build/*.gz + structure: "tree" # Use flat or tree + use-template: "json/template_0.1.0.json" diff --git a/resources/tools/presentation/specifications/converter/input.yaml b/resources/tools/presentation/specifications/converter/input.yaml new file mode 100644 index 0000000000..0cf765030d --- /dev/null +++ b/resources/tools/presentation/specifications/converter/input.yaml @@ -0,0 +1,21 @@ +################################################################################ +### I N P U T X M L F I L E S ### +################################################################################ + +# This is only an example for converter XML --> JSON + +- type: "input" + + # 3n-hsw + + builds: + csit-vpp-perf-report-iterative-2101-3n-hsw: + - 65 # rls2101.rel NDRPDR reconf iter env 6 + - 69 # rls2101.rel Hoststack iter env 6 + - 64 # rls2101.rel NDRPDR iter env 6 + - 63 # rls2101.rel MRR iter env 6 + csit-vpp-perf-report-iterative-2101-2n-skx: + - 94 # rls2101.rel NDRPDR iter env 6 + - 68 # rls2101.rel soak env 6 + csit-vpp-device-2101-ubuntu1804-1n-skx: + - 358 # rls2101.rel VPP DEV env 6 diff --git a/resources/tools/presentation/specifications/report/environment.yaml b/resources/tools/presentation/specifications/report/environment.yaml index 0e946046b4..10d61f56e4 100644 --- a/resources/tools/presentation/specifications/report/environment.yaml +++ b/resources/tools/presentation/specifications/report/environment.yaml @@ -193,7 +193,7 @@ file-format: ".zip" enabled: True - archive-inputs: True + archive-inputs: False mapping-file: "" @@ -224,7 +224,6 @@ reverse-input: False # Needed for trending, not important for the report - # TODO: Change in code needed, it was in type: "configuration" limits: nic: x520: 24460000 @@ -258,7 +257,7 @@ ################################################################################ - type: "output" - arch-file-format: # moved from input, TODO: change it in the code + arch-file-format: - ".gz" - ".zip" output: "report" diff --git a/resources/tools/presentation/specifications/trending/environment.yaml b/resources/tools/presentation/specifications/trending/environment.yaml index dfa9f680c2..95eaa7b606 100644 --- a/resources/tools/presentation/specifications/trending/environment.yaml +++ b/resources/tools/presentation/specifications/trending/environment.yaml @@ -256,7 +256,7 @@ ################################################################################ - type: "output" - arch-file-format: # moved from input, TODO: change it in the code + arch-file-format: - ".gz" - ".zip" output: "trending" -- cgit 1.2.3-korg