CSIT-891: Add data sources for 1801 report

Change-Id: I320737f2edbf74caa8f12177dcc7186a88920a9a
Signed-off-by: Tibor Frank <tifrank@cisco.com>

3 files changed, 228 insertions, 195 deletions

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index d57d1545ea..7dae834b4d 100644
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -15,6 +15,8 @@
 Download all data.
 """
 
+import re
+
 import logging
 
 from os import rename, remove
@@ -34,6 +36,8 @@ CHUNK_SIZE = 512
 # Separator used in file names
 SEPARATOR = "__"
 
+REGEX_RELEASE = re.compile(r'(\D*)(\d{4})(\D*)')
+
 
 def download_data_files(spec):
     """Download all data specified in the specification file in the section
@@ -59,21 +63,42 @@ def download_data_files(spec):
             url = "{0}/{1}".format(url, full_name)
             new_name = join(
                 spec.environment["paths"]["DIR[WORKING,DATA]"],
-                "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR,
+                "{job}{sep}{build}{sep}{name}".format(job=job,
+                                                      sep=SEPARATOR,
                                                       build=build["build"],
                                                       name=file_name))
-
-            logging.info("Downloading the file '{0}' to '{1}'.".
-                         format(url, new_name))
+            logging.info(
+                "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
 
             status = "failed"
             try:
                 response = get(url, stream=True)
                 code = response.status_code
                 if code != codes["OK"]:
-                    logging.error("{0}: {1}".format(code, responses[code]))
+                    logging.warning(
+                        "Jenkins: {0}: {1}.".format(code, responses[code]))
+                    logging.info("Trying to download from Nexus:")
                     spec.set_input_state(job, build["build"], "not found")
-                    continue
+                    if code == codes["not_found"]:
+                        release = re.search(REGEX_RELEASE, job).group(2)
+                        nexus_file_name = "{job}{sep}{build}{sep}{name}".\
+                            format(job=job, sep=SEPARATOR, build=build["build"],
+                                   name=file_name)
+                        url = "{url}/rls{release}/{dir}/{file}".\
+                            format(url=spec.environment["urls"]["URL[NEXUS]"],
+                                   release=release,
+                                   dir=spec.environment["urls"]["DIR[NEXUS]"],
+                                   file=nexus_file_name)
+                        logging.info("Downloading the file '{0}' to '{1}' ...".
+                                     format(url, new_name))
+                        response = get(url, stream=True)
+                        code = response.status_code
+                        if code != codes["OK"]:
+                            logging.error(
+                                "Nexus: {0}: {1}".format(code, responses[code]))
+                            spec.set_input_state(
+                                job, build["build"], "not found")
+                            continue
 
                 file_handle = open(new_name, "wb")
                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
diff --git a/resources/tools/presentation/pal.py b/resources/tools/presentation/pal.py
index 6d613e339c..4c5de010d4 100644
--- a/resources/tools/presentation/pal.py
+++ b/resources/tools/presentation/pal.py
@@ -83,39 +83,39 @@ def main():
         logging.critical("Finished with error.")
         sys.exit(1)
 
-    try:
-        env = Environment(spec.environment, args.force)
-        env.set_environment()
-
-        if spec.is_debug:
-            if spec.debug["input-format"] == "zip":
-                unzip_files(spec)
-        else:
-            download_data_files(spec)
-
-        prepare_static_content(spec)
-
-        data = InputData(spec)
-        data.read_data()
-
-        generate_tables(spec, data)
-        generate_plots(spec, data)
-        generate_files(spec, data)
-        generate_report(args.release, spec)
-
-        logging.info("Successfully finished.")
-
-    except (KeyError, ValueError, PresentationError) as err:
-        logging.info("Finished with an error.")
-        logging.critical(str(err))
-    except Exception as err:
-        logging.info("Finished with an unexpected error.")
-        logging.critical(str(err))
-
-    finally:
-        if spec is not None and not spec.is_debug:
-            clean_environment(spec.environment)
-        sys.exit(1)
+    # try:
+    env = Environment(spec.environment, args.force)
+    env.set_environment()
+
+    if spec.is_debug:
+        if spec.debug["input-format"] == "zip":
+            unzip_files(spec)
+    else:
+        download_data_files(spec)
+
+    prepare_static_content(spec)
+
+    data = InputData(spec)
+    data.read_data()
+
+    generate_tables(spec, data)
+    generate_plots(spec, data)
+    generate_files(spec, data)
+    generate_report(args.release, spec)
+
+    logging.info("Successfully finished.")
+
+    # except (KeyError, ValueError, PresentationError) as err:
+    #     logging.info("Finished with an error.")
+    #     logging.critical(str(err))
+    # except Exception as err:
+    #     logging.info("Finished with an unexpected error.")
+    #     logging.critical(str(err))
+    #
+    # finally:
+    #     if spec is not None and not spec.is_debug:
+    #         clean_environment(spec.environment)
+    #     sys.exit(1)
 
 
 if __name__ == '__main__':
diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 339bf5c2c8..4b61ddb19e 100644
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -61,6 +61,8 @@
   urls:
     URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
     URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+    URL[NEXUS]: "https://docs.fd.io/csit"
+    DIR[NEXUS]: "report/_static/archive"
 
   make-dirs:
   # List the directories which are created while preparing the environment.
@@ -88,38 +90,38 @@
   type: "configuration"
   data-sets:
 # TODO: Add the data sources
-    plot-vpp-http-server-performance:
-      csit-vpp-perf-1801-all:
-      - 1
-      - 2
-      - 3
-      - 4
+#    plot-vpp-http-server-performance:
+#      csit-vpp-perf-1801-all:
+#      - 1
+#      - 2
+#      - 3
+#      - 4
 # TODO: Add the data sources
-    vpp-meltdown-impact:
-      csit-vpp-perf-1707-all:
-      - 9
-      - 10
-      - 13
-      csit-vpp-perf-1710-all:
-      - 11
-      - 12
-      - 13
+#    vpp-meltdown-impact:
+#      csit-vpp-perf-1707-all:
+#      - 9
+#      - 10
+#      - 13
+#      csit-vpp-perf-1710-all:
+#      - 11
+#      - 12
+#      - 13
 # TODO: Add the data sources
-    vpp-spectre-impact:
-      csit-vpp-perf-1707-all:
-      - 9
-      - 10
-      - 13
-      csit-vpp-perf-1710-all:
-      - 11
-      - 12
-      - 13
+#    vpp-spectre-impact:
+#      csit-vpp-perf-1707-all:
+#      - 9
+#      - 10
+#      - 13
+#      csit-vpp-perf-1710-all:
+#      - 11
+#      - 12
+#      - 13
 # TODO: Add the data sources
     plot-throughput-speedup-analysis:
       csit-vpp-perf-1801-all:
-      - 13
-      - 14
-      - 21
+      - 122
+      - 126
+      - 129
     performance-improvements:
       csit-vpp-perf-1707-all:
       - 9
@@ -145,9 +147,9 @@
       - 20
 # TODO: Add the data sources
       csit-vpp-perf-1801-all:
-      - 13
-      - 14
-      - 21
+      - 124
+      - 127
+      - 128
       csit-ligato-perf-1710-all:
       - 5
       - 7
@@ -166,7 +168,9 @@
 # TODO:
     vpp-perf-results:
       csit-vpp-perf-1801-all:
-      - 12
+      - 122
+      - 126
+      - 129
     vpp-func-results:
       csit-vpp-functional-1801-ubuntu1604-virl:
       - "lastSuccessfulBuild"
@@ -180,18 +184,15 @@
     hc-func-results:
       csit-hc2vpp-verify-func-1801-ubuntu1604:
       - "lastSuccessfulBuild"
-# TODO: Will we use this one?
-#      hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604:
-#      - lastSuccessfulBuild
     nsh-func-results:
       csit-nsh_sfc-verify-func-1801-ubuntu1604-virl:
       - 1
 # TODO:
     plot-vpp-throughput-latency:
       csit-vpp-perf-1801-all:
-      - 13
-      - 14
-      - 21
+      - 124
+      - 127
+      - 128
     plot-dpdk-throughput-latency:
       csit-dpdk-perf-1801-all:
       - 1
@@ -201,7 +202,6 @@
       - 6
       - 7
       - 8
-      - 9
       - 10
       - 12
 # TODO:
@@ -483,13 +483,14 @@
     - 18
     - 19
     - 20
-    - 23
 # TODO:
     csit-vpp-perf-1801-all:
-    - 12 # full
-    - 13 # sel
-    - 14 # sel
-    - 21 # sel
+    - 122  # full
+    - 126  # full
+    - 129  # full
+    - 124  # sel
+    - 127  # sel
+    - 128  # sel
     csit-ligato-perf-1710-all:
     - 5
     - 7
@@ -501,6 +502,16 @@
     - 13
     - 16
     - 17
+    csit-dpdk-perf-1801-all:
+    - 1
+    - 4
+    - 5
+    - 7
+    - 8
+    - 10
+    - 12
+    - 16
+    - 17
 # TODO:
     csit-ligato-perf-1801-all:
     - 1
@@ -523,9 +534,6 @@
     - 1
     csit-hc2vpp-verify-func-1801-ubuntu1604:
     - lastSuccessfulBuild
-# TODO: Will we use this one?
-#    hc2vpp-csit-integration-odl-carbon-1801-ubuntu1604:
-#    - lastSuccessfulBuild
 
 -
   type: "output"
@@ -539,73 +547,73 @@
 ###                               T A B L E S                                ###
 ################################################################################
 
--
-  type: "table"
-  title: "Performance Impact of Meltdown Patches"
-  algorithm: "table_performance_comparison"
-  output-file-ext: ".csv"
-# TODO: specify dir
-  output-file: "{DIR[STATIC,VPP]}/meltdown-impact"
-  reference:
-    title: "No Meltdown"
-# TODO: specify data sources
-    data:
-      csit-vpp-perf-1707-all:
-      - 9
-      - 10
-      - 13
-  compare:
-    title: "Meltdown Patches Applied"
-# TODO: specify data sources
-    data:
-      csit-vpp-perf-1710-all:
-      - 11
-      - 12
-      - 13
-  data:
-    "vpp-meltdown-impact"
-  filter: "all"
-  parameters:
-  - "name"
-  - "parent"
-  - "throughput"
-  # Number of the best and the worst tests presented in the table. Use 0 (zero)
-  # to present all tests.
-  nr-of-tests-shown: 20
-
--
-  type: "table"
-  title: "Performance Impact of Spectre Patches"
-  algorithm: "table_performance_comparison"
-  output-file-ext: ".csv"
-# TODO: specify dir
-  output-file: "{DIR[STATIC,VPP]}/meltdown-spectre-impact"
-  reference:
-    title: "No Spectre"
-# TODO: specify data sources
-    data:
-      csit-vpp-perf-1707-all:
-      - 9
-      - 10
-      - 13
-  compare:
-    title: "Spectre Patches Applied"
-# TODO: specify data sources
-    data:
-      csit-vpp-perf-1710-all:
-      - 11
-      - 12
-      - 13
-  data:
-    "vpp-spectre-impact"
-  filter: "all"
-  parameters:
-  - "name"
-  - "parent"
-  - "throughput"
-  # Number of the best and the worst tests presented in the table. Use 0 (zero)
-  # to present all tests.
-  nr-of-tests-shown: 20
+#-
+#  type: "table"
+#  title: "Performance Impact of Meltdown Patches"
+#  algorithm: "table_performance_comparison"
+#  output-file-ext: ".csv"
+## TODO: specify dir
+#  output-file: "{DIR[STATIC,VPP]}/meltdown-impact"
+#  reference:
+#    title: "No Meltdown"
+## TODO: specify data sources
+#    data:
+#      csit-vpp-perf-1707-all:
+#      - 9
+#      - 10
+#      - 13
+#  compare:
+#    title: "Meltdown Patches Applied"
+## TODO: specify data sources
+#    data:
+#      csit-vpp-perf-1710-all:
+#      - 11
+#      - 12
+#      - 13
+#  data:
+#    "vpp-meltdown-impact"
+#  filter: "all"
+#  parameters:
+#  - "name"
+#  - "parent"
+#  - "throughput"
+#  # Number of the best and the worst tests presented in the table. Use 0 (zero)
+#  # to present all tests.
+#  nr-of-tests-shown: 20
+#
+#-
+#  type: "table"
+#  title: "Performance Impact of Spectre Patches"
+#  algorithm: "table_performance_comparison"
+#  output-file-ext: ".csv"
+## TODO: specify dir
+#  output-file: "{DIR[STATIC,VPP]}/meltdown-spectre-impact"
+#  reference:
+#    title: "No Spectre"
+## TODO: specify data sources
+#    data:
+#      csit-vpp-perf-1707-all:
+#      - 9
+#      - 10
+#      - 13
+#  compare:
+#    title: "Spectre Patches Applied"
+## TODO: specify data sources
+#    data:
+#      csit-vpp-perf-1710-all:
+#      - 11
+#      - 12
+#      - 13
+#  data:
+#    "vpp-spectre-impact"
+#  filter: "all"
+#  parameters:
+#  - "name"
+#  - "parent"
+#  - "throughput"
+#  # Number of the best and the worst tests presented in the table. Use 0 (zero)
+#  # to present all tests.
+#  nr-of-tests-shown: 20
 
 -
   type: "table"
@@ -1051,49 +1059,49 @@
 ################################################################################
 
 # Plots VPP HTTP Server Performance
--
-  type: "plot"
-  title: "VPP HTTP Server Performance"
-  algorithm: "plot_http_server_performance_box"
-  output-file-type: ".html"
-  output-file: "{DIR[STATIC,VPP]}/http-server-performance-cps"
-  data:
-    "plot-vpp-http-server-performance"
-  # Keep this formatting, the filter is enclosed with " (quotation mark) and
-  # each tag is enclosed with ' (apostrophe).
-  filter: "'HTTP' and 'TCP_CPS'"
-  parameters:
-  - "result"
-  - "name"
-  traces:
-    hoverinfo: "x+y"
-    boxpoints: "outliers"
-    whiskerwidth: 0
-  layout:
-    title: "VPP HTTP Server Performance"
-    layout:
-      "plot-cps"
-
--
-  type: "plot"
-  title: "VPP HTTP Server Performance"
-  algorithm: "plot_http_server_performance_box"
-  output-file-type: ".html"
-  output-file: "{DIR[STATIC,VPP]}/http-server-performance-rps"
-  data:
-    "plot-vpp-http-server-performance"
-  filter: "'HTTP' and 'TCP_RPS'"
-  parameters:
-  - "result"
-  - "name"
-  traces:
-    hoverinfo: "x+y"
-    boxpoints: "outliers"
-    whiskerwidth: 0
-  layout:
-    title: "VPP HTTP Server Performance"
-    layout:
-      "plot-rps"
+#-
+#  type: "plot"
+#  title: "VPP HTTP Server Performance"
+#  algorithm: "plot_http_server_performance_box"
+#  output-file-type: ".html"
+#  output-file: "{DIR[STATIC,VPP]}/http-server-performance-cps"
+#  data:
+#    "plot-vpp-http-server-performance"
+#  # Keep this formatting, the filter is enclosed with " (quotation mark) and
+#  # each tag is enclosed with ' (apostrophe).
+#  filter: "'HTTP' and 'TCP_CPS'"
+#  parameters:
+#  - "result"
+#  - "name"
+#  traces:
+#    hoverinfo: "x+y"
+#    boxpoints: "outliers"
+#    whiskerwidth: 0
+#  layout:
+#    title: "VPP HTTP Server Performance"
+#    layout:
+#      "plot-cps"
+#
+#-
+#  type: "plot"
+#  title: "VPP HTTP Server Performance"
+#  algorithm: "plot_http_server_performance_box"
+#  output-file-type: ".html"
+#  output-file: "{DIR[STATIC,VPP]}/http-server-performance-rps"
+#  data:
+#    "plot-vpp-http-server-performance"
+#  filter: "'HTTP' and 'TCP_RPS'"
+#  parameters:
+#  - "result"
+#  - "name"
+#  traces:
+#    hoverinfo: "x+y"
+#    boxpoints: "outliers"
+#    whiskerwidth: 0
+#  layout:
+#    title: "VPP HTTP Server Performance"
+#    layout:
+#      "plot-rps"
 
 # Plot Throughput Speedup Analysis