Use 'lftools deploy archives' for log upload

- This is a pre-requisite for using amazon S3
  for log storage.
- Requires docker images built by updated
  docker scripts to include packages required
  by 'lftools' or other project specific
  docker images to be upgraded with same
  packages. This change only updates jobs
  which are using fdiotools docker images.
- Add FD.io specific global macros to avoid
  inflating docker images with packages
  required by unneeded global-jjb scripts.
- Sort VPP job parameters alphabetically
- Dump installed python3 packages in
  setup_executor_env.sh

Change-Id: Ifddd8f2b70b10acf834ffc837b27ef5453213a86
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

6 files changed, 24 insertions, 13 deletions

diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh
index 7e58ac87c..918a6ff14 100755
--- a/docker/scripts/dbld_lfit_requirements.sh
+++ b/docker/scripts/dbld_lfit_requirements.sh
@@ -60,11 +60,5 @@ perl -i -p -e "s/$jenkins_uid\:/0\:/g" /etc/passwd
 cp $DOCKER_CIMAN_ROOT/global-jjb/jenkins-init-scripts/lf-env.sh /root
 chmod 644 /root/lf-env.sh
 
-# Install lftools[openstack] -- from global-jjb/shell/python-tools-install.sh
-pinned_version=""
-if [ "$OS_NAME" = "debian-9" } ; then
-    # debian-9 does not have osc-lib==2.2.0 available breaking docker image
-    # build so pin the version of lftools which does not pin osc-lib==2.2.0
-    pinned_version="==0.34.1"
-fi
-python3 -m pip install --no-deps lftools[openstack]$pinned_version
+# Install lftools for log / artifact upload.
+python3 -m pip install lftools
diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh
index d2aa70db4..0c3711534 100644
--- a/docker/scripts/lib_apt.sh
+++ b/docker/scripts/lib_apt.sh
@@ -112,6 +112,8 @@ ENV PATH=$PATH:$DOCKER_CIMAN_ROOT/docker/scripts
 #       facter
 #       python3-pip
 #       python3-venv
+#   for lftools:
+#       xmlstarlet
 #   from global-jjb/packer/provision/baseline.sh:
 #       unzip
 #       xz-utils
@@ -147,6 +149,7 @@ RUN apt-get update -q \\
         enchant \\
         emacs \\
         facter \\
+        gawk \\
         gfortran \\
         git \\
         git-review \\
@@ -173,6 +176,7 @@ RUN apt-get update -q \\
         tree \\
         vim \\
         wget \\
+        xmlstarlet \\
         xz-utils \\
     && rm -r /var/lib/apt/lists/*
 
diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh
index 424aa7858..17dcc144a 100644
--- a/docker/scripts/lib_common.sh
+++ b/docker/scripts/lib_common.sh
@@ -179,7 +179,7 @@ export EXECUTOR_CLASS_ARCH="$EXECUTOR_DEFAULT_CLASS-$OS_ARCH"
 export EXECUTOR_CLASSES="$EXECUTOR_DEFAULT_CLASS csit csit_dut csit_shim"
 export EXECUTOR_ARCHS="aarch64 x86_64"
 declare -A EXECUTOR_CLASS_ARCH_OS_NAMES
-EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="centos-7 centos-8 ubuntu-18.04 ubuntu-20.04"
+EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="centos-8 ubuntu-18.04 ubuntu-20.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit-aarch64"]="ubuntu-18.04"
 EXECUTOR_CLASS_ARCH_OS_NAMES["csit-x86_64"]="ubuntu-18.04"
diff --git a/docker/scripts/lib_dnf.sh b/docker/scripts/lib_dnf.sh
index 41915e154..639e7f073 100644
--- a/docker/scripts/lib_dnf.sh
+++ b/docker/scripts/lib_dnf.sh
@@ -77,6 +77,9 @@ COPY . .
 # Install baseline packages (minimum build & utils).
 #
 # ci-management global-jjb requirements:
+#   for lftools:
+#       libxml2-devel
+#       xmlstarlet
 #   for lf-env.sh:
 #       facter
 #   from global-jjb/packer/provision/baseline.sh:
@@ -101,13 +104,16 @@ COPY . .
 #   lapack-devel    for python numpy/scipy (CSIT/aarch64)
 #   openblas-devel  for python numpy/scipy (CSIT/aarch64)
 #
-RUN dnf update -y \\
+RUN export LC_ALL=C.UTF8 \\
+    && dnf update -y \\
     && dnf install -y \\
         dnf-plugins-core \\
         epel-release \\
-    && dnf config-manager --set-enabled PowerTools --set-enabled epel \\
+    && dnf config-manager --set-enabled \$(dnf repolist all 2> /dev/null | grep -i powertools | cut -d' ' -f1) --set-enabled epel \\
+    && dnf repolist all \\
     && dnf clean all
-RUN dnf update -y \\
+RUN export LC_ALL=C.UTF8 \\
+    && dnf update -y \\
     && dnf install -y \\
         dnf-utils \\
         doxygen \\
@@ -124,6 +130,7 @@ RUN dnf update -y \\
         lapack-devel \\
         libffi-devel \\
         libpcap-devel \\
+        libxml2-devel \\
         make \\
         mawk \\
         mock \\
@@ -140,6 +147,7 @@ RUN dnf update -y \\
         unzip \\
         vim \\
         wget \\
+        xmlstarlet \\
         xz \\
     && dnf clean all
 
diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh
index 38245c7b7..72bec646f 100644
--- a/docker/scripts/lib_vpp.sh
+++ b/docker/scripts/lib_vpp.sh
@@ -81,7 +81,7 @@ docker_build_setup_vpp() {
 # Branches must be listed in chronological order -- oldest stable branch
 # first and master last.
 declare -A VPP_BRANCHES
-VPP_BRANCHES["centos-7"]="stable/1908 stable/2001 stable/2005 stable/2009 master"
+VPP_BRANCHES["centos-7"]="stable/1908 stable/2001 stable/2005 stable/2009"
 VPP_BRANCHES["centos-8"]="stable/2009 master"
 VPP_BRANCHES["debian-9"]="stable/2009 master"
 VPP_BRANCHES["debian-10"]="stable/2009 master"
diff --git a/docker/scripts/lib_yum.sh b/docker/scripts/lib_yum.sh
index 95a728e6b..89ac3a691 100644
--- a/docker/scripts/lib_yum.sh
+++ b/docker/scripts/lib_yum.sh
@@ -70,6 +70,9 @@ COPY . .
 # Install baseline packages (minimum build & utils).
 #
 # ci-management global-jjb requirements:
+#   for lftools:
+#       libxml2-devel
+#       xmlstarlet
 #   for lf-env.sh:
 #       facter
 #   from global-jjb/packer/provision/baseline.sh:
@@ -117,6 +120,7 @@ RUN yum update -y \\
         liblapack-devel \\
         libopenblas-devel \\
         libpcap-devel \\
+        libxml2-devel \\
         make \\
         mawk \\
         mock \\
@@ -131,6 +135,7 @@ RUN yum update -y \\
         unzip \\
         vim \\
         wget \\
+        xmlstarlet \\
         xz \\
     && yum clean all