Use Ubuntu-20.04 for vpp-csit-* job executor images

- refactor vpp-csit job names to conform to the vpp
  job nomenclature
- add ubuntu-20.04 for CSIT image builder install
  requirements
- fix executor class verification
- add option to build_executor_docker_image.sh to
  dump Dockerfile without building image(s)
- refactor Dockerfile generation to support
  executor classes
- add support for csit_shim and csit_dut docker
  image generation from csit ansible repo
- refactor docker installation
- refactor locale support
- remove old static csit Dockerfiles
- remove unused jenkins labels
- use csit-builder-{os}-prod-{executor_arch} label for
  vpp csit device test
- run tox in csit_pip_cache() to include pylint
  requirements in python cache to avoid downloading
  from pypi during job execution.

Change-Id: I5f4ffc587c6b8c43cb874dfd8c615ce56d3b97a4
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

1 files changed, 87 insertions, 37 deletions

diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index c44a4e0f3..8031d4240 100644
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -228,14 +228,14 @@
         os: 'ubuntu2004'
 # [end] VPP-AARCH64 PROJECT
 
-# VPP-CSIT-VERIFY PROJECT
+# VPP-CSIT-VERIFY-DEVICE PROJECT
 - project:
-    name: vpp-csit-verify
+    name: vpp-csit-verify-device
     jobs:
       - 'vpp-csit-verify-device-perpatch':
           make-parallel-jobs: '32'
           exclude:
-            - device-node-arch: '1n-tx2'
+            - testbed-arch: '1n-tx2'
       - 'vpp-csit-verify-device-periodic':
           make-parallel-jobs: '32'
           periodicity: 'H * * * *'
@@ -244,12 +244,8 @@
           exclude:
             - stream: '2009'
             - stream: '2101'
-            - device-node-arch: '1n-skx'
-      - 'vpp-csit-verify-perf-{stream}-{node-arch}'
+            - testbed-arch: '1n-skx'
     project: 'vpp'
-    os: ubuntu1804
-    device-executor: 'vpp-csit-device'
-    executor: '{os}-us'
     skip-vote: 'false'
     stream:
       - master:
@@ -264,26 +260,68 @@
           branch: 'stable/2101'
           branch-refspec: ''
           repo-stream-part: 'stable.2101'
-    device-node-arch:
-      - 1n-skx
+    testbed-arch:
+      - 1n-skx:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
       - 1n-tx2:
-          device-executor: 'vpp-csit-arm-ubuntu18'
-    node-arch:
-      - 2n-clx
-      - 2n-skx
-      - 2n-zn2
-      - 3n-skx
-      - 3n-hsw
-      - 2n-dnv
-      - 3n-dnv
+          os: 'ubuntu2004'
+          executor-arch: 'aarch64'
+
+# [end] VPP-CSIT-VERIFY-DEVICE PROJECT
+
+# VPP-CSIT-VERIFY-PERF PROJECT
+- project:
+    name: vpp-csit-verify-perf
+    jobs:
+      - 'vpp-csit-verify-perf-{stream}-{os}-{executor-arch}-{testbed-arch}'
+    project: 'vpp'
+    skip-vote: 'false'
+    stream:
+      - master:
+          branch: 'master'
+          branch-refspec: ''
+          repo-stream-part: 'master'
+      - '2009':
+          branch: 'stable/2009'
+          branch-refspec: ''
+          repo-stream-part: 'stable.2009'
+      - '2101':
+          branch: 'stable/2101'
+          branch-refspec: ''
+          repo-stream-part: 'stable.2101'
+    testbed-arch:
+      - 2n-clx:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
+      - 2n-skx:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
+      - 2n-zn2:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
+      - 3n-skx:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
+      - 3n-hsw:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
+      - 2n-dnv:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
+      - 3n-dnv:
+          os: 'ubuntu2004'
+          executor-arch: 'x86_64'
       - 3n-tsh:
-          executor: 'ubuntu1804arm-s'
+          os: 'ubuntu2004'
+          executor-arch: 'aarch64'
           make-parallel-jobs: '16'
       - 2n-tx2:
-          executor: 'ubuntu1804arm-s'
+          os: 'ubuntu2004'
+          executor-arch: 'aarch64'
           make-parallel-jobs: '16'
 
-# [end] VPP-CSIT-VERIFY PROJECT
+# [end] VPP-CSIT-VERIFY-PERF PROJECT
 
 # VPP-CSIT-API-CRC PROJECT
 #
@@ -292,9 +330,9 @@
 - project:
     name: vpp-csit-api-crc
     jobs:
-      - 'vpp-csit-verify-api-crc-{stream}'
+      - 'vpp-csit-verify-api-crc-{stream}-{os}-{executor-arch}'
     project: 'vpp'
-    executor: 'vpp-csit-device'
+    executor-arch: 'x86_64'
     skip-vote: 'false'
     stream:
       - master:
@@ -308,7 +346,7 @@
           branch: 'stable/2101'
           branch-refspec: ''
           repo-stream-part: 'stable.2101'
-    os: ubuntu1804
+    os: ubuntu2004
 # [end] VPP-CSIT-API-CRC PROJECT
 
 # LF RELENG VPP PROJECTS
@@ -737,18 +775,22 @@
 
 # VPP-CSIT-VERIFY-DEVICE-PERIODIC JOB TEMPLATE
 - job-template:
-    name: 'vpp-csit-verify-device-{stream}-{device-node-arch}'
+    name: 'vpp-csit-verify-device-{stream}-{os}-{executor-arch}-{testbed-arch}'
     id: 'vpp-csit-verify-device-periodic'
     description: |
       <ul>
           <li>executor
               <ul>
-                  <li>{device-executor}
+                  <li>csit-builder-{os}-prod-{executor-arch}
+              </ul>
+          <li>testbed architecture
+              <ul>
+                  <li>{testbed-arch}
               </ul>
       </ul>
 
     project-type: freestyle
-    node: '{device-executor}'
+    node: 'csit-builder-{os}-prod-{executor-arch}'
     concurrent: true
     archive-artifacts: '**/csit_current/**/*.*'
     latest-only: false
@@ -822,18 +864,22 @@
 
 # VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE
 - job-template:
-    name: 'vpp-csit-verify-device-{stream}-{device-node-arch}'
+    name: 'vpp-csit-verify-device-{stream}-{os}-{executor-arch}-{testbed-arch}'
     id: 'vpp-csit-verify-device-perpatch'
     description: |
       <ul>
           <li>executor
               <ul>
-                  <li>{device-executor}
+                  <li>csit-builder-{os}-prod-{executor-arch}
+              </ul>
+          <li>testbed architecture
+              <ul>
+                  <li>{testbed-arch}
               </ul>
       </ul>
 
     project-type: freestyle
-    node: '{device-executor}'
+    node: 'csit-builder-{os}-prod-{executor-arch}'
     concurrent: true
     archive-artifacts: '**/csit_current/**/*.*'
     latest-only: false
@@ -906,12 +952,16 @@
 
 # VPP-CSIT-VERIFY-PERF JOB TEMPLATE
 - job-template:
-    name: 'vpp-csit-verify-perf-{stream}-{node-arch}'
+    name: 'vpp-csit-verify-perf-{stream}-{os}-{executor-arch}-{testbed-arch}'
     description: |
       <ul>
           <li>executor
               <ul>
-                  <li>{executor}
+                  <li>builder-{os}-prod-{executor-arch}
+              </ul>
+          <li>testbed architecture
+              <ul>
+                  <li>{testbed-arch}
               </ul>
           <li>Objective
           Contrary to csit-vpp job, this also measures the parent performance,
@@ -920,7 +970,7 @@
       </ul>
 
     project-type: freestyle
-    node: '{executor}'
+    node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
     archive-artifacts: >
       **/csit_current/**/*.*
@@ -973,7 +1023,7 @@
       - gerrit-trigger-csit-perftest:
           project: '{project}'
           branch: '{branch}'
-          trigger-word: 'perftest-{node-arch}'
+          trigger-word: 'perftest-{testbed-arch}'
           skip-vote: true
 
     builders:
@@ -1000,10 +1050,10 @@
 
 # VPP-CSIT-VERIFY-API-CRC JOB TEMPLATE
 - job-template:
-    name: 'vpp-csit-verify-api-crc-{stream}'
+    name: 'vpp-csit-verify-api-crc-{stream}-{os}-{executor-arch}'
 
     project-type: freestyle
-    node: '{os}-us'
+    node: 'builder-{os}-prod-{executor-arch}'
     concurrent: true
     latest-only: false