Use 'lftools deploy archives' for log upload

- This is a pre-requisite for using amazon S3
  for log storage.
- Requires docker images built by updated
  docker scripts to include packages required
  by 'lftools' or other project specific
  docker images to be upgraded with same
  packages. This change only updates jobs
  which are using fdiotools docker images.
- Add FD.io specific global macros to avoid
  inflating docker images with packages
  required by unneeded global-jjb scripts.
- Sort VPP job parameters alphabetically
- Dump installed python3 packages in
  setup_executor_env.sh

Change-Id: Ifddd8f2b70b10acf834ffc837b27ef5453213a86
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

1 files changed, 128 insertions, 131 deletions

diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index 900bf55e0..4d31e89e6 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -1,3 +1,16 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ---
 # PROJECT DEFINITIONS
 #
@@ -52,6 +65,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -83,6 +97,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - '1908':
           branch: 'stable/1908'
@@ -125,6 +140,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -158,6 +174,7 @@
       - 'vpp-debug-verify-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '4'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -194,6 +211,7 @@
 #          comment-trigger-value: 'vpp-beta-merge'
 #    project: 'vpp'
 #    make-parallel-jobs: '4'
+#    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 #    stream:
 #      - master:
 #          branch: 'master'
@@ -234,6 +252,7 @@
       - 'vpp-merge-{stream}-{os}-{executor-arch}'
     project: 'vpp'
     make-parallel-jobs: '16'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -281,6 +300,7 @@
 #
 #    project: 'vpp'
 #    make-parallel-jobs: '16'
+#    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 #    stream:
 #      - master:
 #          branch: 'master'
@@ -317,6 +337,7 @@
           comment-trigger-value: 'beta-merge'
     project: 'vpp'
     make-parallel-jobs: '16'
+    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
     stream:
       - master:
           branch: 'master'
@@ -434,7 +455,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -442,17 +462,20 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -538,8 +561,8 @@
           - ../scripts/vpp/commitmsg.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
+
 # [end] VPP-CHECKSTYLE-VERIFY JOB TEMPLATE
 
 # VPP-VERIFY JOB TEMPLATE
@@ -549,7 +572,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -557,25 +579,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -607,8 +628,7 @@
           - ../scripts/vpp/build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-VERIFY JOB TEMPLATE
 
 # VPP-DEBUG-VERIFY JOB TEMPLATE
@@ -618,7 +638,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -626,21 +645,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: 'ubuntu1804'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
       - stream-parameter:
           stream: '{stream}'
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -674,8 +696,7 @@
           - ../scripts/vpp/debug-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-DEBUG-VERIFY JOB TEMPLATE
 
 # VPP-MERGE JOB TEMPLATE
@@ -691,27 +712,22 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
-      - maven-project-parameter:
-          maven: 'mvn33-new'
-      - maven-exec:
-          maven-version: 'mvn33-new'
-      - stream-parameter:
-          stream: '{stream}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
+      - stream-parameter:
+          stream: '{stream}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -758,8 +774,7 @@
           - ../scripts/packagecloud_push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-MERGE JOB TEMPLATE
 
 # VPP-BETA-VERIFY JOB TEMPLATE
@@ -769,7 +784,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -777,25 +791,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -828,8 +841,7 @@
           - ../scripts/vpp/build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-BETA-VERIFY JOB TEMPLATE
 
 # VPP-BETA-MERGE JOB TEMPLATE
@@ -845,27 +857,22 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
-      - maven-project-parameter:
-          maven: 'mvn33-new'
-      - maven-exec:
-          maven-version: 'mvn33-new'
-      - stream-parameter:
-          stream: '{stream}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
+      - stream-parameter:
+          stream: '{stream}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -907,16 +914,12 @@
       - shell:
           !include-raw-escape:
           - ../scripts/vpp/build.sh
-      - provide-maven-settings:
-          settings-file: 'vpp-settings'
-          global-settings-file: 'global-settings'
       - shell:
           !include-raw-escape:
           - ../scripts/packagecloud_push.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-BETA-MERGE JOB TEMPLATE
 
 # VPP-GCC-VERIFY JOB TEMPLATE
@@ -926,7 +929,6 @@
     project-type: freestyle
     node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core'
 
     build-discarder:
       daysToKeep: '{build-days-to-keep}'
@@ -934,25 +936,24 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter:
+          refspec: '{branch-head-refspec}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.{repo-os-part}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
-      - gerrit-refspec-parameter:
-          refspec: '{branch-head-refspec}'
+      - workspace-archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
 
     scm:
       - gerrit-trigger-scm:
@@ -984,8 +985,7 @@
           - ../scripts/vpp/gcc-build.sh
 
     publishers:
-      - fdio-infra-shiplogs:
-          maven-version: 'mvn36'
+      - fdio-infra-publish
 # [end] VPP-GCC-VERIFY JOB TEMPLATE
 
 # VPP-CSIT-VERIFY-DEVICE JOB TEMPLATE
@@ -1011,27 +1011,26 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
+      - archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
+      - gerrit-csit-refspec-parameter
+      - gerrit-event-comment-text-parameter
+      - gerrit-event-type-parameter
       - gerrit-parameter:
           branch: '{branch}'
+      - gerrit-refspec-parameter
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
-      - gerrit-refspec-parameter
-      - gerrit-event-type-parameter
-      - gerrit-event-comment-text-parameter
-      - gerrit-csit-refspec-parameter
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -1100,31 +1099,30 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
+      - archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
+      - csit-perf-trial-duration-parameter:
+          csit-perf-trial-duration: "10.0"
+      - csit-perf-trial-multiplicity-parameter:
+          csit-perf-trial-multiplicity: "5"
+      - gerrit-csit-refspec-parameter
+      - gerrit-event-comment-text-parameter
+      - gerrit-event-type-parameter
+      - gerrit-refspec-parameter
       - gerrit-parameter:
           branch: '{branch}'
+      - make-parallel-jobs-parameter:
+          make-parallel-jobs: '{make-parallel-jobs}'
       - os-parameter:
           os: '{os}'
+      - project-parameter:
+          project: '{project}'
       - repo-name-parameter:
           repo-name: '{repo-stream-part}.ubuntu.bionic.main'
-      - gerrit-refspec-parameter
-      - gerrit-event-type-parameter
-      - gerrit-event-comment-text-parameter
-      - gerrit-csit-refspec-parameter
-      - csit-perf-trial-duration-parameter:
-          csit-perf-trial-duration: "10.0"
-      - csit-perf-trial-multiplicity-parameter:
-          csit-perf-trial-multiplicity: "5"
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - make-parallel-jobs-parameter:
-          make-parallel-jobs: '{make-parallel-jobs}'
 
     scm:
       - gerrit-trigger-scm:
@@ -1161,7 +1159,8 @@
       - robot-report:
           output-path: 'csit_current/0/'
 
-      - lf-infra-publish
+      - fdio-infra-shiplogs:
+          maven-version: 'mvn36'
 # [end] VPP-CSIT-VERIFY-PERF JOB TEMPLATE
 
 # VPP-CSIT-VERIFY-API-CRC JOB TEMPLATE
@@ -1171,7 +1170,6 @@
     project-type: freestyle
     node: '{os}-us'
     concurrent: true
-    archive-artifacts: ''
     latest-only: false
 
     build-discarder:
@@ -1180,23 +1178,22 @@
       artifactDaysToKeep: '{build-artifact-days-to-keep}'
       artifactNumToKeep: '{build-artifact-num-to-keep}'
 
+    # Please keep parameters in alphabetical order
     parameters:
-      - project-parameter:
-          project: '{project}'
+      - archive-artifacts-parameter:
+          artifacts: '{archive-artifacts}'
+      # Not sure whether not failing has any useful usage,
+      # but it does not hurt to have some flexibility for future.
+      - csit-fail-on-crc-mismatch-parameter:
+          fail-on-crc-mismatch: 'True'
+      - gerrit-csit-refspec-parameter
       - gerrit-parameter:
           branch: '{branch}'
       - gerrit-refspec-parameter
-      - gerrit-csit-refspec-parameter
+      - project-parameter:
+          project: '{project}'
       - stream-parameter:
           stream: '{stream}'
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      # Not sure whether not failing has any useful usage,
-      # but it does not hurt to have some flexibility for future.
-      - csit-fail-on-crc-mismatch-parameter:
-          fail-on-crc-mismatch: 'True'
 
     scm:
       - gerrit-trigger-scm: