diff options
92 files changed, 1907 insertions, 835 deletions
diff --git a/docker/scripts/build_executor_docker_image.sh b/docker/scripts/build_executor_docker_image.sh index d0af78be3..2eac97df7 100755 --- a/docker/scripts/build_executor_docker_image.sh +++ b/docker/scripts/build_executor_docker_image.sh @@ -143,3 +143,5 @@ for executor_os_name in $os_names ; do fi fi done + +echo -e "\n$(basename $BASH_SOURCE) COMPLETE!\nHave a great day! :D" diff --git a/docker/scripts/dbld_lfit_requirements.sh b/docker/scripts/dbld_lfit_requirements.sh index 7e58ac87c..a3c7661d1 100755 --- a/docker/scripts/dbld_lfit_requirements.sh +++ b/docker/scripts/dbld_lfit_requirements.sh @@ -1,6 +1,6 @@ #! /bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -60,11 +60,7 @@ perl -i -p -e "s/$jenkins_uid\:/0\:/g" /etc/passwd cp $DOCKER_CIMAN_ROOT/global-jjb/jenkins-init-scripts/lf-env.sh /root chmod 644 /root/lf-env.sh -# Install lftools[openstack] -- from global-jjb/shell/python-tools-install.sh -pinned_version="" -if [ "$OS_NAME" = "debian-9" } ; then - # debian-9 does not have osc-lib==2.2.0 available breaking docker image - # build so pin the version of lftools which does not pin osc-lib==2.2.0 - pinned_version="==0.34.1" -fi -python3 -m pip install --no-deps lftools[openstack]$pinned_version +# Install lftools & boto3 for log / artifact upload. +source /root/lf-env.sh +lf-activate-venv lftools +python3 -m pip install boto3 diff --git a/docker/scripts/lib_apt.sh b/docker/scripts/lib_apt.sh index d2aa70db4..449b008e7 100644 --- a/docker/scripts/lib_apt.sh +++ b/docker/scripts/lib_apt.sh @@ -112,10 +112,13 @@ ENV PATH=$PATH:$DOCKER_CIMAN_ROOT/docker/scripts # facter # python3-pip # python3-venv -# from global-jjb/packer/provision/baseline.sh: +# for lftools: +# xmlstarlet +# libxml2-dev +# libxslt-dev +# from packer/provision/baseline.sh: # unzip # xz-utils -# puppet # git # git-review # libxml2-dev @@ -137,6 +140,7 @@ ENV PATH=$PATH:$DOCKER_CIMAN_ROOT/docker/scripts # liblapack-dev for python numpy/scipy (CSIT/aarch64) # libopenblas-dev for python numpy/scipy (CSIT/aarch64) # libpcap-dev for python pypcap install (CSIT) +# sshpass for CSIT jobs # RUN apt-get update -q \\ && apt-get install -y -qq \\ @@ -147,6 +151,8 @@ RUN apt-get update -q \\ enchant \\ emacs \\ facter \\ + gawk \\ + gdb \\ gfortran \\ git \\ git-review \\ @@ -168,11 +174,13 @@ RUN apt-get update -q \\ python3-venv \\ rsync \\ ruby-dev \\ + sshpass \\ sudo \\ traceroute \\ tree \\ vim \\ wget \\ + xmlstarlet \\ xz-utils \\ && rm -r /var/lib/apt/lists/* diff --git a/docker/scripts/lib_common.sh b/docker/scripts/lib_common.sh index 424aa7858..3449bf398 100644 --- a/docker/scripts/lib_common.sh +++ b/docker/scripts/lib_common.sh @@ -124,23 +124,6 @@ remove_pyc_files_and_pycache_dirs() { -print -exec rm -rf {} \; 2>/dev/null || true } -# Get the refspec for the specified project branch at HEAD -# -# Arguments: -# $1 - branch -# $2 - project (Optional: defaults to 'vpp') -get_gerrit_refspec() { - local branch=${1:-"master"} - local project=${2:-"vpp"} - local query="$(ssh -p 29418 gerrit.fd.io gerrit query status:merged project:$project branch:$branch limit:1 --format=JSON --current-patch-set | tr ',' '\n' | grep refs | cut -d'"' -f4)" - - if [ -z "$query" ] ; then - echo "ERROR: Invalid project ($1) or branch ($2)" - else - echo "$query" - fi -} - # Well-known filename variables export APT_DEBIAN_DOCKER_GPGFILE="docker.linux.debian.gpg" export APT_UBUNTU_DOCKER_GPGFILE="docker.linux.ubuntu.gpg" @@ -179,7 +162,7 @@ export EXECUTOR_CLASS_ARCH="$EXECUTOR_DEFAULT_CLASS-$OS_ARCH" export EXECUTOR_CLASSES="$EXECUTOR_DEFAULT_CLASS csit csit_dut csit_shim" export EXECUTOR_ARCHS="aarch64 x86_64" declare -A EXECUTOR_CLASS_ARCH_OS_NAMES -EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="centos-7 centos-8 ubuntu-18.04 ubuntu-20.04" +EXECUTOR_CLASS_ARCH_OS_NAMES["builder-aarch64"]="centos-8 ubuntu-18.04 ubuntu-20.04" EXECUTOR_CLASS_ARCH_OS_NAMES["builder-x86_64"]="centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04" EXECUTOR_CLASS_ARCH_OS_NAMES["csit-aarch64"]="ubuntu-18.04" EXECUTOR_CLASS_ARCH_OS_NAMES["csit-x86_64"]="ubuntu-18.04" diff --git a/docker/scripts/lib_csit.sh b/docker/scripts/lib_csit.sh index 9958faba2..a903ab0d8 100644 --- a/docker/scripts/lib_csit.sh +++ b/docker/scripts/lib_csit.sh @@ -147,14 +147,8 @@ csit_pip_cache() { # Virtualenv version is pinned in common.sh in newer csit branches. # (note: xargs removes leading/trailing spaces) - # - # TODO: pip3 install virtualenv==20.0.20 installs a version of virtualenv - # which hardcodes python3 in the shebang line. This breaks branches - # containing python2 code (eg. oper1908-*). - # Restore when 19.08 is no longer supported or is updated to override - # the shebang in virtualenv (e.g. 'python2.7 virtualenv env') - # install_virtualenv="$(grep 'virtualenv' $csit_bash_function_dir/common.sh | grep pip | grep install | cut -d'|' -f1 | xargs)" - # $install_virtualenv + install_virtualenv="$(grep 'virtualenv' $csit_bash_function_dir/common.sh | grep pip | grep install | cut -d'|' -f1 | xargs)" + $install_virtualenv git checkout -q -- . echo_log " Completed $description!" diff --git a/docker/scripts/lib_dnf.sh b/docker/scripts/lib_dnf.sh index 41915e154..e3fd768af 100644 --- a/docker/scripts/lib_dnf.sh +++ b/docker/scripts/lib_dnf.sh @@ -77,13 +77,16 @@ COPY . . # Install baseline packages (minimum build & utils). # # ci-management global-jjb requirements: +# for lftools: +# libxml2-devel +# libxslt-devel +# xmlstarlet # for lf-env.sh: # facter -# from global-jjb/packer/provision/baseline.sh: +# from packer/provision/baseline.sh: # deltarpm # unzip # xz -# puppet # python3-pip # git # git-review @@ -98,22 +101,27 @@ COPY . . # enchant for 'make docs' (VPP) # libffi-devel for python cffi install (Ubuntu20.04/VPP/aarch64) # libpcap-devel for python pypcap install (CSIT) -# lapack-devel for python numpy/scipy (CSIT/aarch64) -# openblas-devel for python numpy/scipy (CSIT/aarch64) +# lapack-devel for python numpy/scipy (CSIT/aarch64) +# openblas-devel for python numpy/scipy (CSIT/aarch64) +# sshpass for CSIT # -RUN dnf update -y \\ +RUN export LC_ALL=C.UTF8 \\ + && dnf update -y \\ && dnf install -y \\ dnf-plugins-core \\ epel-release \\ - && dnf config-manager --set-enabled PowerTools --set-enabled epel \\ + && dnf config-manager --set-enabled \$(dnf repolist all 2> /dev/null | grep -i powertools | cut -d' ' -f1) --set-enabled epel \\ + && dnf repolist all \\ && dnf clean all -RUN dnf update -y \\ +RUN export LC_ALL=C.UTF8 \\ + && dnf update -y \\ && dnf install -y \\ dnf-utils \\ doxygen \\ enchant \\ emacs \\ facter \\ + gdb \\ git \\ git-review \\ graphviz \\ @@ -124,6 +132,8 @@ RUN dnf update -y \\ lapack-devel \\ libffi-devel \\ libpcap-devel \\ + libxml2-devel \\ + libxslt-devel \\ make \\ mawk \\ mock \\ @@ -131,15 +141,16 @@ RUN dnf update -y \\ perl \\ perl-XML-XPath \\ python3-pip \\ - puppet \\ rake \\ rsync \\ ruby-devel \\ + sshpass \\ sudo \\ tree \\ unzip \\ vim \\ wget \\ + xmlstarlet \\ xz \\ && dnf clean all diff --git a/docker/scripts/lib_vpp.sh b/docker/scripts/lib_vpp.sh index 38245c7b7..5ed538a88 100644 --- a/docker/scripts/lib_vpp.sh +++ b/docker/scripts/lib_vpp.sh @@ -1,7 +1,7 @@ # lib_vpp.sh - Docker build script VPP library. # For import only. -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -80,11 +80,18 @@ docker_build_setup_vpp() { # Branches must be listed in chronological order -- oldest stable branch # first and master last. +# +# Note: CI Jobs for each architecture are maintained in +# .../ci-management/jjb/vpp/vpp.yaml +# All OS's and branches are included in the 'os' and 'stream' +# definitions respectively, then the exclude list maintained +# to create an enumerated set of jobs jobs that match the +# definitions here. declare -A VPP_BRANCHES -VPP_BRANCHES["centos-7"]="stable/1908 stable/2001 stable/2005 stable/2009 master" -VPP_BRANCHES["centos-8"]="stable/2009 master" -VPP_BRANCHES["debian-9"]="stable/2009 master" -VPP_BRANCHES["debian-10"]="stable/2009 master" -VPP_BRANCHES["ubuntu-18.04"]="stable/1908 stable/2001 stable/2005 stable/2009 master" -VPP_BRANCHES["ubuntu-20.04"]="stable/2009 master" +VPP_BRANCHES["centos-7"]="stable/2009" +VPP_BRANCHES["centos-8"]="stable/2009 stable/2101 master" +VPP_BRANCHES["debian-9"]="stable/2009" +VPP_BRANCHES["debian-10"]="stable/2009 stable/2101 master" +VPP_BRANCHES["ubuntu-18.04"]="stable/2009 stable/2101 master" +VPP_BRANCHES["ubuntu-20.04"]="stable/2009 stable/2101 master" export VPP_BRANCHES diff --git a/docker/scripts/lib_yum.sh b/docker/scripts/lib_yum.sh index 95a728e6b..d4da28972 100644 --- a/docker/scripts/lib_yum.sh +++ b/docker/scripts/lib_yum.sh @@ -70,13 +70,16 @@ COPY . . # Install baseline packages (minimum build & utils). # # ci-management global-jjb requirements: +# for lftools: +# libxml2-devel +# libxslt-devel +# xmlstarlet # for lf-env.sh: # facter -# from global-jjb/packer/provision/baseline.sh: +# from packer/provision/baseline.sh: # deltarpm # unzip # xz -# puppet # python3-pip # git # git-review @@ -93,6 +96,7 @@ COPY . . # libpcap-devel for python pypcap install (CSIT) # liblapack-devel for python numpy/scipy (CSIT/aarch64) # libopenblas-devel for python numpy/scipy (CSIT/aarch64) +# sshpass for CSIT # RUN yum update -y \\ && yum install -y \\ @@ -106,6 +110,7 @@ RUN yum update -y \\ enchant \\ emacs \\ facter \\ + gdb \\ git \\ git-review \\ graphviz \\ @@ -117,20 +122,23 @@ RUN yum update -y \\ liblapack-devel \\ libopenblas-devel \\ libpcap-devel \\ + libxml2-devel \\ + libxslt-devel \\ make \\ mawk \\ mock \\ perl \\ perl-XML-XPath \\ python3-pip \\ - puppet \\ rake \\ ruby-devel \\ + sshpass \\ sudo \\ tree \\ unzip \\ vim \\ wget \\ + xmlstarlet \\ xz \\ && yum clean all diff --git a/extras/bash/sandbox_test_functions.sh b/extras/bash/sandbox_test_functions.sh new file mode 100644 index 000000000..afccf9a21 --- /dev/null +++ b/extras/bash/sandbox_test_functions.sh @@ -0,0 +1,118 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bash function to set up jenkins sandbox environment +# +# See LF Sandbox documentation: +# https://docs.releng.linuxfoundation.org/en/latest/jenkins-sandbox.html +# +# Prerequisites: +# 1. Create jenkins sandbox token and add it to your local jenkins.ini file +# Either specify the location of the init file in $JENKINS_INI or +# JENKINS_INI will be initialized to either +# ~/.config/jenkins_jobs/jenkins.ini +# $WS_ROOT/jenkins.ini +# 2. Clone ci-management workspace from gerrit.fd.io +# 3. export WS_ROOT=<local ci-management workspace> +jjb-sandbox-env() +{ + local jjb_version=${JJB_VERSION:-"3.5.0"} + + if [ -z "$WS_ROOT" ] ; then + echo "ERROR: WS_ROOT is not set!" + return + elif [ ! -d "$WS_ROOT/jjb" ] ; then + echo "ERROR: WS_ROOT is not set to a ci-management workspace:" + echo " '$WS_ROOT'" + return + fi + + if [ -n "$(declare -f deactivate)" ]; then + echo "Deactivating Python Virtualenv!" + deactivate + fi + + if [ -z "$JENKINS_INI" ] ; then + local user_jenkins_ini="/home/$USER/.config/jenkins_jobs/jenkins.ini" + if [ -f "$user_jenkins_ini" ] ; then + export JENKINS_INI=$user_jenkins_ini + elif [ -f "$WS_ROOT/jenkins.ini" ] ; then + export JENKINS_INI="$WS_ROOT/jenkins.ini" + else + echo "ERROR: Unable to find 'jenkins.ini'!" + return + fi + echo "Exporting JENKINS_INI=$JENKINS_INI" + elif [ ! -f "$JENKINS_INI" ] ; then + echo "ERROR: file specified in JENKINS_INI ($JENKINS_INI) not found!" + return + fi + + if [ -n "$(declare -f deactivate)" ]; then + echo "Deactivating Python Virtualenv!" + deactivate + fi + cd $WS_ROOT + git submodule update --init --recursive + + local VENV_DIR=$WS_ROOT/venv + rm -rf $VENV_DIR \ + && python3 -m venv $VENV_DIR \ + && source $VENV_DIR/bin/activate \ + && pip3 install wheel \ + && pip3 install jenkins-job-builder==$jjb_version + + alias jjsb='jenkins-jobs --conf $JENKINS_INI' + function jjsb-test() { + if [ -z "$(which jenkins-jobs 2>&1)" ] ; then + echo "jenkins-jobs not found! Run jjb-sandbox-env to activate." + return + fi + if [ -z "$1" ] ; then + echo "Usage: $FUNCNAME <jenkins-job-name>" + return + fi + which jenkins-jobs \ + && jenkins-jobs --conf $JENKINS_INI test $WS_ROOT/jjb $@ + } + function jjsb-update() { + if [ -z "$(which jenkins-jobs 2>&1)" ] ; then + echo "jenkins-jobs not found! Run jjb-sandbox-env to activate." + return + fi + if [ -z "$1" ] ; then + echo "Usage: $FUNCNAME <jenkins-job-name>" + return + fi + which jenkins-jobs \ + && jenkins-jobs --conf $JENKINS_INI update $WS_ROOT/jjb $@ + } + jenkins-jobs --version +} + +# Get the refspec for the specified project branch at HEAD +# +# Arguments: +# $1 - branch +# $2 - project (Optional: defaults to 'vpp') +get_gerrit_refspec() { + local branch=${1:-"master"} + local project=${2:-"vpp"} + local query="$(ssh -p 29418 gerrit.fd.io gerrit query status:merged project:$project branch:$branch limit:1 --format=JSON --current-patch-set | tr ',' '\n' | grep refs | cut -d'"' -f4)" + + if [ -z "$query" ] ; then + echo "ERROR: Invalid project ($1) or branch ($2)" + else + echo "$query" + fi +} diff --git a/global-jjb b/global-jjb -Subproject 413bdb2e77dcc2055e05476930f13874a2fae24 +Subproject e5840e0c645226926475e6dfabd9f16606325ab diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-aarch64.yaml new file mode 100644 index 000000000..95b632ffb --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-centos7:prod-aarch64" + labels: "builder-centos7-prod-aarch64" + prefix: "builder-centos7-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-x86_64.yaml new file mode 100644 index 000000000..a5d77a7be --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-centos7:prod-x86_64" + labels: "builder-centos7-prod-x86_64" + memory: 14000 + prefix: "builder-centos7-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-aarch64.yaml new file mode 100644 index 000000000..2458aa9b6 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-centos7:sandbox-aarch64" + labels: "builder-centos7-sandbox-aarch64" + prefix: "builder-centos7-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-x86_64.yaml new file mode 100644 index 000000000..efa35ed82 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-centos7:sandbox-x86_64" + labels: "builder-centos7-sandbox-x86_64" + memory: 14000 + prefix: "builder-centos7-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-aarch64.yaml new file mode 100644 index 000000000..364e6a117 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-centos7:test-aarch64" + labels: "builder-centos7-test-aarch64" + prefix: "builder-centos7-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-x86_64.yaml new file mode 100644 index 000000000..529c2a1f9 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-centos7:test-x86_64" + labels: "builder-centos7-test-x86_64" + memory: 14000 + prefix: "builder-centos7-test-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-aarch64.yaml new file mode 100644 index 000000000..4e60a6dea --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-centos8:prod-aarch64" + labels: "builder-centos8-prod-aarch64" + prefix: "builder-centos8-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-x86_64.yaml new file mode 100644 index 000000000..09ddccc2b --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-centos8:prod-x86_64" + labels: "builder-centos8-prod-x86_64" + memory: 14000 + prefix: "builder-centos8-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-aarch64.yaml new file mode 100644 index 000000000..8a0d72a34 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-centos8:sandbox-aarch64" + labels: "builder-centos8-sandbox-aarch64" + prefix: "builder-centos8-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-x86_64.yaml new file mode 100644 index 000000000..447ceb5cc --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-centos8:sandbox-x86_64" + labels: "builder-centos8-sandbox-x86_64" + memory: 14000 + prefix: "builder-centos8-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-aarch64.yaml new file mode 100644 index 000000000..0ae69f641 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-centos8:test-aarch64" + labels: "builder-centos8-test-aarch64" + prefix: "builder-centos8-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-x86_64.yaml new file mode 100644 index 000000000..8868e31ec --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-centos8:test-x86_64" + labels: "builder-centos8-test-x86_64" + memory: 14000 + prefix: "builder-centos8-test-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-aarch64.yaml new file mode 100644 index 000000000..2145da043 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-debian10:prod-aarch64" + labels: "builder-debian10-prod-aarch64" + prefix: "builder-debian10-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-x86_64.yaml new file mode 100644 index 000000000..b3444f83d --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-debian10:prod-x86_64" + labels: "builder-debian10-prod-x86_64" + memory: 14000 + prefix: "builder-debian10-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-aarch64.yaml new file mode 100644 index 000000000..6cb75294e --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-debian10:sandbox-aarch64" + labels: "builder-debian10-sandbox-aarch64" + prefix: "builder-debian10-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-x86_64.yaml new file mode 100644 index 000000000..424e5cf56 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-debian10:sandbox-x86_64" + labels: "builder-debian10-sandbox-x86_64" + memory: 14000 + prefix: "builder-debian10-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-aarch64.yaml new file mode 100644 index 000000000..92b9937a7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-debian10:test-aarch64" + labels: "builder-debian10-test-aarch64" + prefix: "builder-debian10-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-x86_64.yaml new file mode 100644 index 000000000..359323db7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-debian10:test-x86_64" + labels: "builder-debian10-test-x86_64" + memory: 14000 + prefix: "builder-debian10-test-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-aarch64.yaml new file mode 100644 index 000000000..c247c766f --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-debian9:prod-aarch64" + labels: "builder-debian9-prod-aarch64" + prefix: "builder-debian9-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-x86_64.yaml new file mode 100644 index 000000000..1314af8b2 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-debian9:prod-x86_64" + labels: "builder-debian9-prod-x86_64" + memory: 14000 + prefix: "builder-debian9-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-aarch64.yaml new file mode 100644 index 000000000..af1283571 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-debian9:sandbox-aarch64" + labels: "builder-debian9-sandbox-aarch64" + prefix: "builder-debian9-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-x86_64.yaml new file mode 100644 index 000000000..86847e130 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-debian9:sandbox-x86_64" + labels: "builder-debian9-sandbox-x86_64" + memory: 14000 + prefix: "builder-debian9-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-aarch64.yaml new file mode 100644 index 000000000..ff8e981d5 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-debian9:test-aarch64" + labels: "builder-debian9-test-aarch64" + prefix: "builder-debian9-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-x86_64.yaml new file mode 100644 index 000000000..6042ed3d7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-debian9:test-x86_64" + labels: "builder-debian9-test-x86_64" + memory: 14000 + prefix: "builder-debian9-test-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-aarch64.yaml new file mode 100644 index 000000000..b857ebed7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-ubuntu1804:prod-aarch64" + labels: "builder-ubuntu1804-prod-aarch64" + prefix: "builder-ubuntu1804-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-x86_64.yaml new file mode 100644 index 000000000..d7c4ef613 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-ubuntu1804:prod-x86_64" + labels: "builder-ubuntu1804-prod-x86_64" + memory: 14000 + prefix: "builder-ubuntu1804-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-aarch64.yaml new file mode 100644 index 000000000..3be0f2b90 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-ubuntu1804:sandbox-aarch64" + labels: "builder-ubuntu1804-sandbox-aarch64" + prefix: "builder-ubuntu1804-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-x86_64.yaml new file mode 100644 index 000000000..c28e5f894 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-ubuntu1804:sandbox-x86_64" + labels: "builder-ubuntu1804-sandbox-x86_64" + memory: 14000 + prefix: "builder-ubuntu1804-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-aarch64.yaml new file mode 100644 index 000000000..2d4481fa2 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-ubuntu1804:test-aarch64" + labels: "builder-ubuntu1804-test-aarch64" + prefix: "builder-ubuntu1804-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-x86_64.yaml new file mode 100644 index 000000000..3341b2530 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-ubuntu1804:test-x86_64" + labels: "builder-ubuntu1804-test-x86_64" + memory: 14000 + prefix: "builder-ubuntu1804-test-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-aarch64.yaml new file mode 100644 index 000000000..71bd7c8b7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-ubuntu2004:prod-aarch64" + labels: "builder-ubuntu2004-prod-aarch64" + prefix: "builder-ubuntu2004-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-x86_64.yaml new file mode 100644 index 000000000..2560e4507 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-ubuntu2004:prod-x86_64" + labels: "builder-ubuntu2004-prod-x86_64" + memory: 14000 + prefix: "builder-ubuntu2004-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-aarch64.yaml new file mode 100644 index 000000000..9f23d1965 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-ubuntu2004:sandbox-aarch64" + labels: "builder-ubuntu2004-sandbox-aarch64" + prefix: "builder-ubuntu2004-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-x86_64.yaml new file mode 100644 index 000000000..53426ed8b --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-ubuntu2004:sandbox-x86_64" + labels: "builder-ubuntu2004-sandbox-x86_64" + memory: 14000 + prefix: "builder-ubuntu2004-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-aarch64.yaml new file mode 100644 index 000000000..31b815c91 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-aarch64.yaml @@ -0,0 +1,5 @@ +--- +templates: + - image: "fdiotools/builder-ubuntu2004:test-aarch64" + labels: "builder-ubuntu2004-test-aarch64" + prefix: "builder-ubuntu2004-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-x86_64.yaml new file mode 100644 index 000000000..b35cef803 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "fdiotools/builder-ubuntu2004:test-x86_64" + labels: "builder-ubuntu2004-test-x86_64" + memory: 14000 + prefix: "builder-ubuntu2004-test-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-prod-aarch64.yaml new file mode 100644 index 000000000..e45c768fd --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-prod-aarch64.yaml @@ -0,0 +1,12 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + image: "fdiotools/csit_dut-ubuntu1804:prod-aarch64" + labels: "csit_dut-ubuntu1804-prod-aarch64" + prefix: "csit_dut-ubuntu1804-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-prod-x86_64.yaml new file mode 100644 index 000000000..9196f2704 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csit" + cpu: 10000 + image: "fdiotools/csit_dut-ubuntu1804:prod-x86_64" + labels: "csit_dut-ubuntu1804-prod-x86_64" + memory: 18000 + prefix: "csit_dut-ubuntu1804-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-sandbox-aarch64.yaml new file mode 100644 index 000000000..574375826 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-sandbox-aarch64.yaml @@ -0,0 +1,12 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + image: "fdiotools/csit_dut-ubuntu1804:sandbox-aarch64" + labels: "csit_dut-ubuntu1804-sandbox-aarch64" + prefix: "csit_dut-ubuntu1804-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-sandbox-x86_64.yaml new file mode 100644 index 000000000..93297357a --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csit" + cpu: 10000 + image: "fdiotools/csit_dut-ubuntu1804:sandbox-x86_64" + labels: "csit_dut-ubuntu1804-sandbox-x86_64" + memory: 18000 + prefix: "csit_dut-ubuntu1804-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-test-aarch64.yaml new file mode 100644 index 000000000..12137e60d --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_dut-ubuntu1804-test-aarch64.yaml @@ -0,0 +1,12 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + image: "fdiotools/csit_dut-ubuntu1804:test-aarch64" + labels: "csit_dut-ubuntu1804-test-aarch64" + prefix: "csit_dut-ubuntu1804-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-prod-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-prod-aarch64.yaml new file mode 100644 index 000000000..e86a8bb7c --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-prod-aarch64.yaml @@ -0,0 +1,12 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + image: "fdiotools/csit_shim-ubuntu1804:prod-aarch64" + labels: "csit_shim-ubuntu1804-prod-aarch64" + prefix: "csit_shim-ubuntu1804-prod-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-prod-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-prod-x86_64.yaml new file mode 100644 index 000000000..7730c8f82 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-prod-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csit" + cpu: 10000 + image: "fdiotools/csit_shim-ubuntu1804:prod-x86_64" + labels: "csit_shim-ubuntu1804-prod-x86_64" + memory: 18000 + prefix: "csit_shim-ubuntu1804-prod-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-sandbox-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-sandbox-aarch64.yaml new file mode 100644 index 000000000..5dc33b6bf --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-sandbox-aarch64.yaml @@ -0,0 +1,12 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + image: "fdiotools/csit_shim-ubuntu1804:sandbox-aarch64" + labels: "csit_shim-ubuntu1804-sandbox-aarch64" + prefix: "csit_shim-ubuntu1804-sandbox-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-sandbox-x86_64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-sandbox-x86_64.yaml new file mode 100644 index 000000000..5adf5f04a --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-sandbox-x86_64.yaml @@ -0,0 +1,14 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csit" + cpu: 10000 + image: "fdiotools/csit_shim-ubuntu1804:sandbox-x86_64" + labels: "csit_shim-ubuntu1804-sandbox-x86_64" + memory: 18000 + prefix: "csit_shim-ubuntu1804-sandbox-x86_64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-test-aarch64.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-test-aarch64.yaml new file mode 100644 index 000000000..778c534fb --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/csit_shim-ubuntu1804-test-aarch64.yaml @@ -0,0 +1,12 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + image: "fdiotools/csit_shim-ubuntu1804:test-aarch64" + labels: "csit_shim-ubuntu1804-test-aarch64" + prefix: "csit_shim-ubuntu1804-test-aarch64" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml new file mode 100644 index 000000000..33a3349d1 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml @@ -0,0 +1,31 @@ +--- +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + datacenters: "yul1" + cpu: 6000 + memory: 10000 + disk: 3000 + priority: 50 + idleTerminationInMinutes: 10 + numExecutors: 1 + prefixCmd: "" + forcePull: true + hostVolumes: "" + privileged: true + reusable: false + useRawExec: false + network: "bridge" + region: "global" + remoteFs: "/w" + username: "fdionomad" + password: "hidden" + capAdd: "" + capDrop: "" + securityOpt: "" + extraHosts: "" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml new file mode 100644 index 000000000..ca90d3148 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/main.yaml @@ -0,0 +1,13 @@ +--- +jenkins: + clouds: + - nomad: + jenkinsUrl: "https://jenkins.fd.io/" + name: "FDIONOMAD" + nomadUrl: "http://10.30.51.32:4646" + jenkinsTunnel: "" + prune: false + nomadACLCredentialsId: "" + workerTimeout: 5 + workerUrl: "https://jenkins.fd.io/jnlpJars/slave.jar" + templates: diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml new file mode 100644 index 000000000..45dd5e0f0 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/sandbox.yaml @@ -0,0 +1,13 @@ +--- +jenkins: + clouds: + - nomad: + jenkinsUrl: "https://jenkins.fd.io/sandbox" + name: "FDIONOMAD" + nomadUrl: "http://10.30.51.32:4646" + jenkinsTunnel: "" + prune: false + nomadACLCredentialsId: "" + workerTimeout: 5 + workerUrl: "https://jenkins.fd.io/sandbox/jnlpJars/slave.jar" + templates: diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804-l.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804-l.yaml new file mode 100644 index 000000000..747f61ad9 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804-l.yaml @@ -0,0 +1,13 @@ +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 20000 + image: "snergster/vpp-ubuntu18" + labels: "ubuntu1804-l" + memory: 32000 + prefix: "prod-amd" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804-us.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804-us.yaml new file mode 100644 index 000000000..b40b58e63 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804-us.yaml @@ -0,0 +1,13 @@ +templates: + - constraints: + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + - ltarget: "^${node.class}" + operand: "=" + rtarget: "builder" + cpu: 14000 + image: "snergster/vpp-ubuntu18" + labels: "ubuntu1804-us" + memory: 14000 + prefix: "prod-amd" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804arm-s.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804arm-s.yaml new file mode 100644 index 000000000..5f3afcc05 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804arm-s.yaml @@ -0,0 +1,5 @@ +templates: + - image: "snergster/vpp-arm-ubuntu18" + labels: "ubuntu1804arm-s" + memory: 16000 + prefix: "prod-arm" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804arm-us.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804arm-us.yaml new file mode 100644 index 000000000..f898934a6 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/ubuntu1804arm-us.yaml @@ -0,0 +1,4 @@ +templates: + - image: "snergster/vpp-arm-ubuntu18" + labels: "ubuntu1804arm-us" + prefix: "prod-arm" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-arm-ubuntu18.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-arm-ubuntu18.yaml new file mode 100644 index 000000000..9ab4631a4 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-arm-ubuntu18.yaml @@ -0,0 +1,11 @@ +templates: + - constraints: + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csitarm" + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "arm64" + image: "snergster/vpp-arm-ubuntu18" + labels: "vpp-csit-arm-ubuntu18" + prefix: "prod-csit-arm" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-device.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-device.yaml new file mode 100644 index 000000000..ef4f9eda7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-device.yaml @@ -0,0 +1,13 @@ +templates: + - constraints: + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csit" + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + cpu: 10000 + image: "snergster/vpp-ubuntu18" + labels: "vpp-csit-device vpp-csit-ubuntu18" + memory: 18000 + prefix: "prod-csit" diff --git a/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-ubuntu18.yaml b/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-ubuntu18.yaml new file mode 100644 index 000000000..ef4f9eda7 --- /dev/null +++ b/jenkins-config/clouds/nomad/FDIONOMAD/vpp-csit-ubuntu18.yaml @@ -0,0 +1,13 @@ +templates: + - constraints: + - ltarget: "^${node.class}" + operand: "=" + rtarget: "csit" + - ltarget: "^${attr.cpu.arch}" + operand: "=" + rtarget: "amd64" + cpu: 10000 + image: "snergster/vpp-ubuntu18" + labels: "vpp-csit-device vpp-csit-ubuntu18" + memory: 18000 + prefix: "prod-csit" diff --git a/jjb/cicn/cicn.yaml b/jjb/cicn/cicn.yaml index d526ea6f3..ff296d130 100644 --- a/jjb/cicn/cicn.yaml +++ b/jjb/cicn/cicn.yaml @@ -1,3 +1,16 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - project: name: cicn-libparc-x86_64 @@ -11,8 +24,8 @@ repo-os-part: "ubuntu.bionic.main" - ubuntu2004: repo-os-part: "ubuntu.focal.main" - - centos7: - repo-os-part: "centos7" + - centos8: + repo-os-part: "centos8" executor-arch: "x86_64" @@ -96,10 +109,6 @@ repo-name: "{repo-stream-part}.{repo-os-part}" - stream-parameter: stream: "hicn" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. scm: - gerrit-trigger-scm: @@ -144,8 +153,7 @@ - ../scripts/packagecloud_push.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn33-new" + - fdio-infra-publish - job-template: name: "cicn-libparc-verify-{stream}-{os}-{executor-arch}" @@ -167,10 +175,6 @@ branch: "{branch}" - os-parameter: os: "{os}" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. scm: - gerrit-trigger-scm: @@ -192,8 +196,7 @@ - include-raw-cicn-libparc-build.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn33-new" + - fdio-infra-publish - job-template: name: "cicn-viper-merge-{stream}-{os}-{executor-arch}" @@ -222,10 +225,6 @@ repo-name: "{repo-stream-part}.{repo-os-part}" - stream-parameter: stream: "hicn" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. scm: - gerrit-trigger-scm: @@ -273,8 +272,7 @@ - ../scripts/packagecloud_push.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn33-new" + - fdio-infra-publish - job-template: name: "cicn-viper-verify-{stream}-{os}-{executor-arch}" @@ -296,10 +294,6 @@ branch: "{branch}" - os-parameter: os: "{os}" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. scm: - gerrit-trigger-scm: @@ -321,8 +315,7 @@ - include-raw-cicn-viper-build.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn33-new" + - fdio-infra-publish - project: name: cicn-info diff --git a/jjb/csit/csit-perf.yaml b/jjb/csit/csit-perf.yaml index 057f12598..323988bf4 100644 --- a/jjb/csit/csit-perf.yaml +++ b/jjb/csit/csit-perf.yaml @@ -24,17 +24,21 @@ - 2n-skx - 2n-dnv - 2n-zn2 + - 2n-tx2: + executor: 'ubuntu1804arm-us' - 3n-skx - 3n-hsw - 3n-dnv - 3n-tsh: - executor: 'ubuntu1804arm-us' + executor: 'ubuntu1804arm-us' - 'csit-dpdk-perf-mrr-weekly-master{node-arch}': node-arch: - '' - -2n-clx - -2n-skx - -2n-zn2 + - -2n-tx2: + executor: 'ubuntu1804arm-us' - -3n-skx - -3n-tsh: executor: 'ubuntu1804arm-us' @@ -45,11 +49,13 @@ - 2n-skx - 2n-dnv - 2n-zn2 + - 2n-tx2: + executor: 'ubuntu1804arm-us' - 3n-skx - 3n-hsw - 3n-dnv - 3n-tsh: - executor: 'ubuntu1804arm-us' + executor: 'ubuntu1804arm-us' stream: - master: branch: 'master' @@ -63,25 +69,31 @@ - '2009_lts': branch: 'rls2009_lts' branch-id: 'oper-rls2009_lts' + - '2101': + branch: 'rls2101' + branch-id: 'oper-rls2101' - 'csit-vpp-perf-mrr-daily-master{node-arch}': node-arch: - '': - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,3,5' - -2n-clx: - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,3,5' - -2n-skx: - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,3,5' - -2n-dnv: - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,4' - -2n-zn2: - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,4' + - -2n-tx2: + executor: 'ubuntu1804arm-us' + periodicity: '0 2 * * 1,4' - -3n-skx: - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,4' - -3n-dnv: - periodicity: '0 2,14 * * *' + periodicity: '0 2 * * 1,4' - -3n-tsh: executor: 'ubuntu1804arm-us' - periodicity: '0 14 * * *' + periodicity: '0 2 * * 4' - 'csit-vpp-perf-mrr-weekly-{stream}-{node-arch}': node-arch: - 2n-clx @@ -108,6 +120,8 @@ - 2n-skx - 2n-dnv - 2n-zn2 + - 2n-tx2: + executor: 'ubuntu1804arm-us' - 3n-skx - 3n-hsw - 3n-dnv @@ -119,6 +133,8 @@ - 2n-skx - 2n-dnv - 2n-zn2 + - 2n-tx2: + executor: 'ubuntu1804arm-us' - 3n-skx - 3n-hsw - 3n-dnv @@ -130,6 +146,8 @@ - 2n-skx - 2n-dnv - 2n-zn2 + - 2n-tx2: + executor: 'ubuntu1804arm-us' - 3n-skx - 3n-hsw - 3n-dnv @@ -141,6 +159,8 @@ - 2n-skx - 2n-dnv - 2n-zn2 + - 2n-tx2: + executor: 'ubuntu1804arm-us' - 3n-skx - 3n-hsw - 3n-dnv @@ -166,10 +186,16 @@ - '2009_lts': branch: 'rls2009_lts' branch-id: 'oper-rls2009_lts' + - '2101': + branch: 'rls2101' + branch-id: 'oper-rls2101' stream_report: - '2009': branch: 'rls2009' branch-id: 'oper-rls2009' + - '2101': + branch: 'rls2101' + branch-id: 'oper-rls2101' build-artifact-num-to-keep: 10 diff --git a/jjb/csit/csit-tox.yaml b/jjb/csit/csit-tox.yaml index efb8bbdd4..3999144e8 100644 --- a/jjb/csit/csit-tox.yaml +++ b/jjb/csit/csit-tox.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -29,6 +29,8 @@ branch: 'rls2009' - '2009_lts': branch: 'rls2009_lts' + - '2101': + branch: 'rls2101' - job-template: name: 'csit-verify-tox-{stream}' @@ -68,9 +70,37 @@ build-timeout: '{build-timeout}' triggers: - - gerrit-trigger-patch-submitted: - name: '{project}' - branch: '{branch}' + # Tox trigger is special. + - gerrit: + server-name: 'Primary' + trigger-on: + - patchset-created-event: + exclude-drafts: 'true' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + # Not enabling on draft published event. + # Frequently, people publish for public access, not fr verify. + # Use recheck comment to trigger verify. + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + override-votes: true + gerrit-build-notbuilt-verified-value: 0 + gerrit-build-successful-verified-value: 1 + failed-message: 'Tox failed. No further verify jobs will be started.' + successful-message: 'tox_success' + skip-vote: + successful: true + failed: false + unstable: false + notbuilt: false builders: - shell: diff --git a/jjb/csit/csit-vpp-device.yaml b/jjb/csit/csit-vpp-device.yaml index 4f2fd7869..9a4b47da2 100644 --- a/jjb/csit/csit-vpp-device.yaml +++ b/jjb/csit/csit-vpp-device.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -32,6 +32,9 @@ - '2009_lts': branch: 'rls2009_lts' branch-id: 'oper-rls2009_lts' + - '2101': + branch: 'rls2101' + branch-id: 'oper-rls2101' - 'csit-vpp-device-{stream}-{os}-{node-arch}-weekly': stream: - master: @@ -40,6 +43,9 @@ - '2009_lts': branch: 'rls2009_lts' branch-id: 'oper-rls2009_lts' + - '2101': + branch: 'rls2101' + branch-id: 'oper-rls2101' project: 'csit' executor: 'vpp-csit-device' @@ -58,6 +64,9 @@ - '2009_lts': branch: 'rls2009_lts' branch-id: 'oper-rls2009_lts' + - '2101': + branch: 'rls2101' + branch-id: 'oper-rls2101' exclude: - stream: '2005' node-arch: '1n-tx2' @@ -141,7 +150,7 @@ build-timeout: '{build-timeout}' triggers: - - 'gerrit-trigger-csit-vpp-devicetest-full': + - gerrit-trigger-csit-vpp-devicetest-full: project: '{project}' branch: '{branch}' skip-vote: '{skip-vote}' diff --git a/jjb/csit/csit.yaml b/jjb/csit/csit.yaml index 0d5c4ac48..96be2ae28 100644 --- a/jjb/csit/csit.yaml +++ b/jjb/csit/csit.yaml @@ -36,6 +36,8 @@ branch: 'rls2009' - '2009_lts': branch: 'rls2009_lts' + - '2101': + branch: 'rls2101' - project: name: csit-view @@ -236,7 +238,7 @@ build-timeout: '{build-timeout}' triggers: - - timed: '0 2,14 * * *' + - timed: '0 14 * * 1,3,5' builders: - maven-target: diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 66a70ba48..03b99f5ec 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -25,6 +25,14 @@ description: "Architecture parameter" - parameter: + name: archive-artifacts-parameter + parameters: + - string: + name: ARCHIVE_ARTIFACTS + default: '{artifacts}' + description: "Artifacts to archive on the log server." + +- parameter: name: compiler-parameter parameters: - string: @@ -187,6 +195,14 @@ name: TOPO default: '{topo}' description: "Topology parameter" + +- parameter: + name: workspace-archive-artifacts-parameter + parameters: + - string: + name: WS_ARCHIVE_ARTIFACTS + default: '{artifacts}' + description: "Artifacts to archive in $WORKSPACE/archives." # [end] Keep 'parameter' entries ordered alphabetically, please. #### SCMS @@ -346,23 +362,20 @@ - trigger: name: gerrit-trigger-csit-vpp-devicetest-full - # Comes from CSIT, for automatic jobs running "device" tests with VPP builds. + # Comes from CSIT, for automatic jobs running device tests with VPP builds. # The job acts as an automatic verify job for CSIT contributions. triggers: - gerrit: server-name: 'Primary' trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event + # The main way the job is triggered automatically on patch upload. + # Gated by tox job, so we do not spend resources on WiP changes. - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' + comment-contains-value: 'tox_success' + # Manual trigger to allow testing WiP changes with selected tags. - comment-added-contains-event: comment-contains-value: 'csit-devicetest' + # A comment with "recheck" restarts tox job, so no condition here. projects: - project-compare-type: 'ANT' project-pattern: '{project}' @@ -635,6 +648,35 @@ - recipients - publisher: + name: fdio-infra-publish + # macro to finish up a build. + # + # Handles the following: + # - Shipping logs to Nexus or S3 logs repository + # - Cleanup workspace + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - shell: !include-raw: scripts/post_build_deploy_archives.sh + - shell: !include-raw: scripts/backup_upload_archives.sh + - fdio-infra-ship-logs + mark-unstable-if-failed: true + - workspace-cleanup: + exclude: + # Do not clean up *.jenkins-trigger files for jobs that use a + # properties file as input for triggering another build. + - "**/*.jenkins-trigger" + fail-build: false + +- publisher: name: fdio-infra-shiplogs # To archive things, the job will need to create an "archives" directory in # the workspace and this macro will handle copying the contents of the @@ -657,6 +699,7 @@ - UNSTABLE build-steps: - shell: !include-raw: include-raw-deploy-archives.sh + - shell: !include-raw: scripts/backup_upload_archives.sh - maven-target: maven-version: '{maven-version}' pom: '.archives/deploy-archives.xml' @@ -719,6 +762,34 @@ num-to-keep: '{build-num-to-keep}' - builder: + name: fdio-infra-ship-logs + builders: + # Ensure no pre-existing .netrc files are overriding logs config + - lf-provide-maven-settings-cleanup + - config-file-provider: + files: + - file-id: "jenkins-log-archives-settings" + variable: "SETTINGS_FILE" + - conditional-step: + condition-kind: regex-match + regex: "^.*logs-s3.*" + label: $S3_BUCKET + on-evaluation-failure: dont-run + steps: + - config-file-provider: + files: + - file-id: "jenkins-s3-log-ship" + target: $HOME/.aws/credentials + - lf-infra-create-netrc: + server-id: logs + - shell: !include-raw: + - ../global-jjb/shell/logs-deploy.sh + - shell: !include-raw: + - ../global-jjb/shell/logs-clear-credentials.sh + - description-setter: + regexp: "(^(Nexus|S3) build logs: .*)" + +- builder: name: packer-validate builders: - config-file-provider: diff --git a/jjb/hc2vpp/hc2vpp-csit.yaml b/jjb/hc2vpp/hc2vpp-csit.yaml index a1f84820e..a0a18b4f1 100644 --- a/jjb/hc2vpp/hc2vpp-csit.yaml +++ b/jjb/hc2vpp/hc2vpp-csit.yaml @@ -13,12 +13,6 @@ - master: branch: 'master' repo-stream-part: 'master' - - '1810': - branch: 'stable/1810' - repo-stream-part: 'stable.1810' - - '1901': - branch: 'stable/1901' - repo-stream-part: 'stable.1901' - '1904': branch: 'stable/1904' repo-stream-part: 'stable.1904' @@ -26,8 +20,6 @@ os: - ubuntu1804: repo-os-part: 'ubuntu.bionic.main' - - ubuntu1604: - repo-os-part: 'ubuntu.xenial.main' odl: 'neon' @@ -46,8 +38,6 @@ os: - ubuntu1804: repo-os-part: 'ubuntu.bionic.main' - - ubuntu1604: - repo-os-part: 'ubuntu.xenial.main' odl: 'neon' diff --git a/jjb/hc2vpp/hc2vpp.yaml b/jjb/hc2vpp/hc2vpp.yaml index f030cfe6c..d8aef5138 100644 --- a/jjb/hc2vpp/hc2vpp.yaml +++ b/jjb/hc2vpp/hc2vpp.yaml @@ -11,10 +11,6 @@ repo-os-part: 'ubuntu.bionic.main' skip-site: 'false' skip-jar-deploy: 'false' - - ubuntu1604: - repo-os-part: 'ubuntu.xenial.main' - skip-site: 'true' - skip-jar-deploy: 'true' - centos7: repo-os-part: 'centos7' skip-site: 'true' @@ -23,12 +19,6 @@ - master: branch: 'master' repo-stream-part: 'master' - - '1810': - branch: 'stable/1810' - repo-stream-part: 'stable.1810' - - '1901': - branch: 'stable/1901' - repo-stream-part: 'stable.1901' - '1904': branch: 'stable/1904' repo-stream-part: 'stable.1904' diff --git a/jjb/hicn/hicn.yaml b/jjb/hicn/hicn.yaml index 61bb68b3a..15f66acac 100644 --- a/jjb/hicn/hicn.yaml +++ b/jjb/hicn/hicn.yaml @@ -1,5 +1,41 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - project: + name: hicn-os-eol-x86_64 + jobs: + - "hicn-verify-{stream}-{os}-{executor-arch}" + - "hicn-merge-{stream}-{os}-{executor-arch}" + + project: "hicn" + os: + - centos7: + repo-os-part: "centos7" + + executor-arch: "x86_64" + + stream: + - "2009": + branch: "stable/2009" + repo-stream-part: "stable.2009" + repository-name: "hicn" + - "2005": + branch: "stable/2005" + repo-stream-part: "stable.2005" + repository-name: "hicn" + +- project: name: hicn-x86_64 jobs: - "hicn-verify-{stream}-{os}-{executor-arch}" @@ -11,24 +47,41 @@ repo-os-part: "ubuntu.bionic.main" - ubuntu2004: repo-os-part: "ubuntu.focal.main" - - centos7: - repo-os-part: "centos7" executor-arch: "x86_64" stream: - - master: + - "master": branch: "master" repo-stream-part: "master" repository-name: "hicn" + - "2009": + branch: "stable/2009" + repo-stream-part: "stable.2009" + repository-name: "hicn" - "2005": branch: "stable/2005" repo-stream-part: "stable.2005" repository-name: "hicn" - type: - - short - - long +- project: + name: hicn-os-in-master-x86_64 + jobs: + - "hicn-verify-{stream}-{os}-{executor-arch}" + - "hicn-merge-{stream}-{os}-{executor-arch}" + + project: "hicn" + os: + - centos8: + repo-os-part: "centos8" + + executor-arch: "x86_64" + + stream: + - "master": + branch: "master" + repo-stream-part: "master" + repository-name: "hicn" - project: name: hicn-aarch64 @@ -54,10 +107,6 @@ repo-stream-part: "stable.2005" repository-name: "hicn" - type: - - short - - long - - project: name: hicn-beta-x86_64 jobs: @@ -69,8 +118,8 @@ repo-os-part: "ubuntu.bionic.main" - ubuntu2004: repo-os-part: "ubuntu.focal.main" - - centos7: - repo-os-part: "centos7" + - centos8: + repo-os-part: "centos8" executor-arch: "x86_64" @@ -79,10 +128,6 @@ branch: "hicn-light-ng" repo-stream-part: "hicn-light-ng" - type: - - short - - long - - project: name: hicn-beta-aarch64 jobs: @@ -100,10 +145,6 @@ branch: "hicn-light-ng" repo-stream-part: "hicn-light-ng" - type: - - short - - long - - project: name: hicn-misc jobs: @@ -132,8 +173,8 @@ repo-os-part: "ubuntu.bionic.main" - ubuntu2004: repo-os-part: "ubuntu.focal.main" - - centos7: - repo-os-part: "centos7" + - centos8: + repo-os-part: "centos8" executor-arch: "x86_64" @@ -142,10 +183,6 @@ branch: "master" repo-stream-part: "master" - type: - - short - - long - - project: name: hicn-extras-aarch64 jobs: @@ -164,10 +201,6 @@ branch: "master" repo-stream-part: "master" - type: - - short - - long - - project: name: hicn-view views: @@ -197,10 +230,8 @@ os: "{os}" - repo-name-parameter: repo-name: "{repo-stream-part}.ubuntu.bionic.main" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -258,8 +289,7 @@ include-raw-hicn-checkstyle.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: true max-failed-builds: 1 @@ -289,10 +319,8 @@ repo-name: "{repo-stream-part}.{repo-os-part}" - stream-parameter: stream: "{stream}" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -314,8 +342,7 @@ - include-raw-hicn-build.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: false max-failed-builds: 2 @@ -348,10 +375,8 @@ stream: "{repository-name}" - repo-name-parameter: repo-name: "{repo-stream-part}.{repo-os-part}" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -398,8 +423,7 @@ - include-raw-hicn-maven-push.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: false max-failed-builds: 1 @@ -443,10 +467,8 @@ repo-name: "{repo-stream-part}.{repo-os-part}" - stream-parameter: stream: "{stream}" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -476,8 +498,7 @@ - include-raw-hicn-extras-build.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: false max-failed-builds: 2 @@ -510,10 +531,8 @@ stream: "hicn" - repo-name-parameter: repo-name: "{repo-stream-part}.{repo-os-part}" - - string: - name: ARCHIVE_ARTIFACTS - default: "{archive-artifacts}" - description: Artifacts to archive to the logs server. + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -566,8 +585,7 @@ - include-raw-hicn-maven-push.sh publishers: - - fdio-infra-shiplogs: - maven-version: "mvn36" + - fdio-infra-publish - naginator: rerun-unstable-builds: false max-failed-builds: 1 diff --git a/jjb/jvpp/include-raw-jvpp-build.sh b/jjb/jvpp/include-raw-jvpp-build.sh deleted file mode 100755 index 3ccc97e09..000000000 --- a/jjb/jvpp/include-raw-jvpp-build.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# basic build script example -set -xe -o pipefail - -OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') -OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') - -echo OS_ID: ${OS_ID} -echo OS_VERSION_ID: ${OS_VERSION_ID} - -# do nothing but print the current slave hostname -hostname -export CCACHE_DIR=/tmp/ccache -if [[ -d ${CCACHE_DIR} ]];then - echo ${CCACHE_DIR} exists - du -sk ${CCACHE_DIR} -else - echo ${CCACHE_DIR} does not exist. This must be a new slave. -fi - -echo "cat /etc/bootstrap.sha" -if [[ -f /etc/bootstrap.sha ]];then - cat /etc/bootstrap.sha -else - echo "Cannot find cat /etc/bootstrap.sha" -fi - -echo "cat /etc/bootstrap-functions.sha" -if [[ -f /etc/bootstrap-functions.sha ]];then - cat /etc/bootstrap-functions.sha -else - echo "Cannot find cat /etc/bootstrap-functions.sha" -fi - -echo "sha1sum of this script: ${0}" -sha1sum $0 - -echo "CC=${CC}" - -echo "Building using \"make package\"" -# ensure that we build from scratch -./clean.sh -[[ "x${DRYRUN}" == "xTrue" ]] || cd Requirements && make UNATTENDED=yes install-dep && cd .. -[[ "x${DRYRUN}" == "xTrue" ]] || pwd -if [[ "${OS_ID}" == "centos" ]]; then - [[ "x${DRYRUN}" == "xTrue" ]] || /opt/rh/devtoolset-7/enable - [[ "x${DRYRUN}" == "xTrue" ]] || cmake3 . -DCMAKE_PROGRAM_PATH:PATH="/opt/rh/devtoolset-7/root/bin" -else - [[ "x${DRYRUN}" == "xTrue" ]] || cmake . -fi - -[[ "x${DRYRUN}" == "xTrue" ]] || make package -# This will build deb or rpm JVPP package based on OS. Built packages are located in build-root/packages/ - -echo "*******************************************************************" -echo "* JVPP BUILD SUCCESSFULLY COMPLETED" -echo "*******************************************************************" diff --git a/jjb/jvpp/include-raw-jvpp-maven-push.sh b/jjb/jvpp/include-raw-jvpp-maven-push.sh deleted file mode 100755 index 9b0c517ad..000000000 --- a/jjb/jvpp/include-raw-jvpp-maven-push.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -xe -o pipefail -echo "*******************************************************************" -echo "* STARTING PUSH OF JVPP PACKAGES TO REPOS" -echo "* NOTHING THAT HAPPENS BELOW THIS POINT IS RELATED TO BUILD FAILURE" -echo "*******************************************************************" - -[[ "$MVN" ]] || MVN="/opt/apache/maven/bin/mvn" -GROUP_ID="io.fd.${PROJECT}" -BASEURL="${NEXUSPROXY}/content/repositories/fd.io." -BASEREPOID='fdio-' - -if [[ "${OS}" == "ubuntu1604" ]]; then - # Find the files - DEBS=$(find ./build-root/packages/ -type f -iname '*.deb') - - for i in ${DEBS} - do - push_deb "$i" - done -elif [[ "${OS}" == "ubuntu1804" ]]; then - # Find the files - JARS=$(find ./java -type f -iname '*.jar') - DEBS=$(find ./build-root/packages/ -type f -iname '*.deb') - for i in ${JARS} - do - push_jar "$i" - done - - for i in ${DEBS} - do - push_deb "$i" - done -elif [[ "${OS}" == "centos7" ]]; then - # Find the files - RPMS=$(find ./build-root/packages/ -type f -iname '*.rpm') - SRPMS=$(find ./build-root/packages/ -type f -iname '*.srpm') - SRCRPMS=$(find ./build-root/packages/ -type f -name '*.src.rpm') - for i in ${RPMS} ${SRPMS} ${SRCRPMS} - do - push_rpm "$i" - done -fi -# vim: ts=4 sw=4 sts=4 et ft=sh : diff --git a/jjb/jvpp/include-raw-jvpp-packetcloud-push.sh b/jjb/jvpp/include-raw-jvpp-packetcloud-push.sh deleted file mode 100755 index 2917606cc..000000000 --- a/jjb/jvpp/include-raw-jvpp-packetcloud-push.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# PCIO_CO is a Jenkins Global Environment variable - -set -x - -echo "STARTING JVPP PACKAGECLOUD PUSH" - -sleep 10 - -FACTER_OS=$(/usr/bin/facter operatingsystem) - -if [ -f ~/.packagecloud ]; then - case "$FACTER_OS" in - Ubuntu) - FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename) - DEBS=$(find ./build-root/packages/ -type f -iname '*.deb') - package_cloud push "${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/" ${DEBS} - ;; - CentOS) - FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease) - FACTER_ARCH=$(/usr/bin/facter architecture) - RPMS=$(find ./build-root/packages/ -type f -iregex '.*/.*\.\(s\)?rpm') - package_cloud push "${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/" ${RPMS} - ;; - esac -fi diff --git a/jjb/jvpp/jvpp.yaml b/jjb/jvpp/jvpp.yaml deleted file mode 100644 index 034f67f8b..000000000 --- a/jjb/jvpp/jvpp.yaml +++ /dev/null @@ -1,199 +0,0 @@ ---- -- project: - name: jvpp - jobs: - - 'jvpp-verify-{stream}-{os}' - - 'jvpp-merge-{stream}-{os}' - project: 'jvpp' - os: - - ubuntu1604: - repo-os-part: 'ubuntu.xenial.main' - skip-site: 'false' - skip-jar-deploy: 'true' - vpp-job-prefix: '' - - ubuntu1804: - repo-os-part: 'ubuntu.bionic.main' - skip-site: 'false' - skip-jar-deploy: 'false' - vpp-job-prefix: 'beta-' - - centos7: - repo-os-part: 'centos7' - skip-site: 'true' - skip-jar-deploy: 'true' - vpp-job-prefix: '' - stream: - - master: - branch: 'master' - repo-stream-part: 'master' - - '1904': - branch: 'stable/1904' - repo-stream-part: 'stable.1904' - -- project: - name: jvpp-view - views: - - project-view - project-name: jvpp - -- job-template: - name: 'jvpp-verify-{stream}-{os}' - - project-type: freestyle - node: '{os}-us' - concurrent: true - archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core' - - build-discarder: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - os-parameter: - os: '{os}' - - repo-name-parameter: - repo-name: '{repo-stream-part}.{repo-os-part}' - - stream-parameter: - stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - scm: - - gerrit-trigger-scm: - credentials-id: 'jenkins-gerrit-credentials' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - wrappers: - - fdio-infra-wrappers: - build-timeout: '{build-timeout}' - - triggers: - - gerrit-trigger-patch-submitted: - name: '{project}' - branch: '{branch}' - - builders: - - lf-update-java-alternatives: - java-version: openjdk11 - - shell: - !include-raw-escape: - - ../scripts/setup_jvpp_dev_env.sh - - include-raw-jvpp-build.sh - - publishers: - - lf-infra-publish - - naginator: - rerun-unstable-builds: false - max-failed-builds: 2 - fixed-delay: 90 - - email-notification: - email-prefix: '[jvpp]' - -- job-template: - name: 'jvpp-merge-{stream}-{os}' - - project-type: freestyle - node: '{os}-us' - - build-discarder: - daysToKeep: '{build-days-to-keep}' - numToKeep: 100 - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - os-parameter: - os: '{os}' - - maven-project-parameter: - maven: 'mvn36' - - maven-exec: - maven-version: 'mvn36' - - stream-parameter: - stream: '{stream}' - - repo-name-parameter: - repo-name: '{repo-stream-part}.{repo-os-part}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - scm: - - gerrit-trigger-scm: - credentials-id: 'jenkins-gerrit-credentials' - refspec: '' - choosing-strategy: 'default' - - wrappers: - - fdio-infra-wrappers: - build-timeout: '{build-timeout}' - - triggers: - - gerrit-trigger-patch-merged: - name: '{project}' - branch: '{branch}' - - reverse: - jobs: 'vpp-{vpp-job-prefix}merge-{stream}-{os}' - result: 'success' - - builders: - - lf-update-java-alternatives: - java-version: openjdk11 - - config-file-provider: - files: - - file-id: '.packagecloud' - target: '/root' - - config-file-provider: - files: - - file-id: 'packagecloud_api' - target: '/root' - - maven-target: - maven-version: 'mvn36' - goals: '--version' - settings: 'jvpp-settings' - settings-type: cfp - global-settings: 'global-settings' - global-settings-type: cfp - - shell: - !include-raw-escape: - - ../scripts/setup_jvpp_dev_env.sh - - include-raw-jvpp-build.sh - - provide-maven-settings: - settings-file: 'jvpp-settings' - global-settings-file: 'global-settings' - - shell: - !include-raw-escape: - - ./include-raw-jvpp-packetcloud-push.sh - - shell: - !include-raw-escape: - - ../scripts/maven_push_functions.sh - - include-raw-jvpp-maven-push.sh - - publishers: - - lf-infra-publish - - naginator: - rerun-unstable-builds: false - max-failed-builds: 1 - fixed-delay: 90 - - email-notification: - email-prefix: '[jvpp]' - -- project: - name: jvpp-info - project-name: jvpp - jobs: - - gerrit-info-yaml-verify - build-node: centos7-builder-4c-4g - project: jvpp - branch: master diff --git a/jjb/scripts/backup_upload_archives.sh b/jjb/scripts/backup_upload_archives.sh new file mode 100755 index 000000000..1a9c577e1 --- /dev/null +++ b/jjb/scripts/backup_upload_archives.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/backup_upload_archives.sh" + +PYTHON_SCRIPT="/w/workspace/test-logs/artifact.py" + +# This script uploads the artifacts to a backup upload location +if [ -f "$PYTHON_SCRIPT" ]; then + echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done" + exit 0 +fi + +# TEMPORARY: talk to consul-aware resolver rather than external ones +echo "nameserver 172.17.0.1" >/etc/resolv.conf + +# the Python code below needs boto3 installed +python3 -m pip install boto3 +mkdir -p $(dirname "$PYTHON_SCRIPT") + +cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT' +#!/usr/bin/python3 + +"""Storage utilities library.""" + +import argparse +import gzip +import os +from mimetypes import MimeTypes + +from boto3 import resource +from botocore.client import Config + +ENDPOINT_URL = u"http://storage.service.consul:9000" +AWS_ACCESS_KEY_ID = u"storage" +AWS_SECRET_ACCESS_KEY = u"Storage1234" +REGION_NAME = u"yul1" +COMPRESS_MIME = ( + u"text/html", + u"text/xml", + u"application/octet-stream" +) + + +def compress(src_fpath): + """Compress a single file. + + :param src_fpath: Input file path. + :type src_fpath: str + """ + with open(src_fpath, u"rb") as orig_file: + with gzip.open(src_fpath + ".gz", u"wb") as zipped_file: + zipped_file.writelines(orig_file) + + +def upload(storage, bucket, src_fpath, dst_fpath): + """Upload single file to destination bucket. + + :param storage: S3 storage resource. + :param bucket: S3 bucket name. + :param src_fpath: Input file path. + :param dst_fpath: Destination file path on remote storage. + :type storage: Object + :type bucket: str + :type src_fpath: str + :type dst_fpath: str + """ + mime_guess = MimeTypes().guess_type(src_fpath) + mime = mime_guess[0] + encoding = mime_guess[1] + if not mime: + mime = "application/octet-stream" + + if mime in COMPRESS_MIME and bucket in "logs" and encoding != "gzip": + compress(src_fpath) + src_fpath = src_fpath + ".gz" + dst_fpath = dst_fpath + ".gz" + + extra_args = dict() + extra_args['ContentType'] = mime + + storage.Bucket(bucket + ".fd.io").upload_file( + src_fpath, + dst_fpath, + ExtraArgs=extra_args + ) + print("https://" + bucket + ".nginx.service.consul/" + dst_fpath) + + +def upload_recursive(storage, bucket, src_fpath): + """Recursively uploads input folder to destination. + + Example: + - bucket: logs + - src_fpath: /home/user + - dst_fpath: logs.fd.io/home/user + + :param storage: S3 storage resource. + :param bucket: S3 bucket name. + :param src_fpath: Input folder path. + :type storage: Object + :type bucket: str + :type src_fpath: str + """ + for path, _, files in os.walk(src_fpath): + for file in files: + _path = path.replace(src_fpath, u"") + _dir = src_fpath[1:] if src_fpath[0] == "/" else src_fpath + _dst_fpath = os.path.normpath(_dir + "/" + _path + "/" + file) + _src_fpath = os.path.join(path, file) + upload(storage, bucket, _src_fpath, _dst_fpath) + + +def main(): + """Main function for storage manipulation.""" + + parser = argparse.ArgumentParser() + parser.add_argument( + u"-d", u"--dir", required=True, type=str, + help=u"Directory to upload to storage." + ) + parser.add_argument( + u"-b", u"--bucket", required=True, type=str, + help=u"Target bucket on storage." + ) + args = parser.parse_args() + + # Create main storage resource. + storage = resource( + u"s3", + endpoint_url=ENDPOINT_URL, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + config=Config( + signature_version=u"s3v4" + ), + region_name=REGION_NAME + ) + + upload_recursive( + storage=storage, + bucket=args.bucket, + src_fpath=args.dir + ) + + +if __name__ == u"__main__": + main() + +END_OF_PYTHON_SCRIPT + +WS_ARCHIVES_DIR="$WORKSPACE/archives" +ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER" +ARCHIVES_ZIP_FILE="$WORKSPACE/.archives/archives.zip" + +TMP_ARCHIVES_DIR="/tmp/archives" +mkdir -p $TMP_ARCHIVES_DIR +pushd $TMP_ARCHIVES_DIR + +if [ -f "$ARCHIVES_ZIP_FILE" ]; then + # FIXME: this branch to be removed upon full transition to lftools: + # we are include_raw from fdio-infra-shiplogs publisher, and + # it already packed everything into archives.zip and removed + # the original archives directory. So just unpack and use that. + echo "Detected existing archives.zip, uploading the contents of it" + unzip "$ARCHIVES_ZIP_FILE" +else + # the include_raw is done from fdio-infra-publish publisher + mkdir -p "$ARCHIVES_DIR" + if [ -e "$WS_ARCHIVES_DIR" ]; then + echo "Found $WS_ARCHIVES_DIR, uploading its contents" + cp -r "$WS_ARCHIVES_DIR" . + else + echo "No $WS_ARCHIVES_DIR found. Creating a dummy file." + echo "No archives found while doing backup upload" > "$ARCHIVES_DIR/no-archives-found.txt" + fi +fi + +echo "Contents of the archives dir:" +ls -alR $TMP_ARCHIVES_DIR +echo "Running uploader script $PYTHON_SCRIPT:" +python3 $PYTHON_SCRIPT -d . -b logs || echo "Failed to upload logs" +popd diff --git a/jjb/scripts/packagecloud_push.sh b/jjb/scripts/packagecloud_push.sh index af3bf48ff..a4ae9b166 100644 --- a/jjb/scripts/packagecloud_push.sh +++ b/jjb/scripts/packagecloud_push.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,6 +17,22 @@ echo "---> jjb/scripts/packagecloud_push.sh" set -euxo pipefail +line="*************************************************************************" + +# Nothing was built if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision ID +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping package push. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DRYRUN="${DRYRUN:-}" +if [ "${DRYRUN,,}" = "true" ] ; then + echo -e "\n$line\nSkipping package push because DRYRUN is '${DRYRUN,,}'.\n$line\n" + exit 0 +fi + echo "STARTING PACKAGECLOUD PUSH" sleep 10 @@ -58,8 +74,7 @@ if [ -f ~/.packagecloud ]; then fi ;; *) - echo "ERROR: Unsupported OS '$FACTER_OS'" - echo "PACKAGECLOUD PUSH FAILED!" + echo -e "\n$line\n* ERROR: Unsupported OS '$FACTER_OS'\n* PACKAGECLOUD PUSH FAILED!\n$line\n" exit 1 ;; esac @@ -83,4 +98,4 @@ else exit 1 fi -echo "PACKAGECLOUD PUSH COMPLETE" +echo -e "\n$line\n* PACKAGECLOUD PUSH COMPLETE\n$line\n" diff --git a/jjb/scripts/post_build_deploy_archives.sh b/jjb/scripts/post_build_deploy_archives.sh new file mode 100755 index 000000000..0a47903e8 --- /dev/null +++ b/jjb/scripts/post_build_deploy_archives.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> jjb/scripts/post_build_deploy_archives.sh" + +set +e # Do not affect the build result if some part of archiving fails. +WS_ARCHIVES_DIR="$WORKSPACE/archives" +BUILD_ENV_LOG="$WS_ARCHIVES_DIR/_build-enviroment-variables.log" + +# Generate gdb-command script to output vpp stack traceback from core files. +gdb_cmdfile="/tmp/gdb-commands" +cat >$gdb_cmdfile <<'__END__' +# Usage: +# gdb $BINFILE $CORE -ex 'source -v gdb-commands' -ex quit + +set pagination off +thread apply all bt + +define printstack + set $i=0 + while $i < 15 + frame $i + x/i $pc + info locals + info reg + set $i = $i + 1 + end +end +thread apply all printstack + +# info proc mappings + +__END__ + +STACKTRACE="" +# Returns stacktrace filename in STACKTRACE +generate_vpp_stacktrace_and_delete_core() { + local corefile="$1" + if grep -qe 'debug' <<< "$WORKSPACE" ; then + local binfile="$WORKSPACE/build-root/install-vpp_debug-native/vpp/bin/vpp" + else + local binfile="$WORKSPACE/build-root/install-vpp-native/vpp/bin/vpp" + fi + + echo "Generating stack trace from core file: $corefile" + STACKTRACE="${corefile}.stacktrace" + gdb "$binfile" $corefile -ex 'source -v /tmp/gdb-commands' -ex quit > $STACKTRACE + # remove the core to save space + echo "Removing core file: $corefile" + rm -f "$corefile" + # Dump stacktrace to console log + if [ -f $STACKTRACE ] ; then + echo -e "\n=====[ $STACKTRACE ]=====\n$(cat $STACKTRACE)\n=====[ $STACKTRACE ]=====\n" + else + echo "Stacktrace file not generated!" + STACKTRACE="" + fi +} + +# Delete existing archives dir to ensure current artifact upload +rm -rf "$WS_ARCHIVES_DIR" +mkdir -p "$WS_ARCHIVES_DIR" + +# Log the build environment variables +echo "Logging build environment variables in '$BUILD_ENV_LOG'..." +env > $BUILD_ENV_LOG + +echo "WS_ARCHIVE_ARTIFACTS = '$WS_ARCHIVE_ARTIFACTS'" +if [ -n "${WS_ARCHIVE_ARTIFACTS}" ]; then + pushd $WORKSPACE + shopt -s globstar # Enable globstar to copy archives + archive_artifacts=$(echo ${WS_ARCHIVE_ARTIFACTS}) + shopt -u globstar # Disable globstar + for file in $archive_artifacts; do + if [ -f "$file" ] ; then + fname="$(basename $file)" + # Decompress core.gz file + if grep -qe '^core.*\.gz$' <<<"$fname" ; then + echo "Uncompressing core file $file" + gunzip "$file" + file="${file::(-3)}" + fi + # Convert core file to stacktrace + if [ "${fname::4}" = "core" ] ; then + generate_vpp_stacktrace_and_delete_core $file + [ -z "$STACKTRACE" ] && continue + file=$STACKTRACE + fi + # Set destination filename + if [ "${file::26}" = "/tmp/vpp-failed-unittests/" ] ; then + destfile=$WS_ARCHIVES_DIR${file:25} + else + destfile=$WS_ARCHIVE_DIR$file + fi + echo "Archiving '$file' to '$destfile'" + destdir="$(dirname $destfile)" + mkdir -p $destdir + mv $file $destfile + else + echo "Not archiving '$file'" + fi + done + popd +fi + +# find and gzip any 'text' files +find $WS_ARCHIVES_DIR -type f -print0 \ + | xargs -0r file \ + | egrep -e ':.*text.*' \ + | cut -d: -f1 \ + | xargs -d'\n' -r gzip + +echo "Workspace archived artifacts:" +ls -alR $WS_ARCHIVES_DIR diff --git a/jjb/scripts/setup_executor_env.sh b/jjb/scripts/setup_executor_env.sh index 08b37873c..2bfbe8b32 100644 --- a/jjb/scripts/setup_executor_env.sh +++ b/jjb/scripts/setup_executor_env.sh @@ -49,6 +49,10 @@ elif [ "$OS_ID" == "centos" ] ; then fi echo "$long_line" +echo "Python3 package list:" +pip3 list 2>/dev/null | column -t || true + +echo "$long_line" echo "Executor Downloads cache '$downloads_cache':" ls -lh "$downloads_cache" || true echo "$long_line" diff --git a/jjb/scripts/setup_vpp_dpdk_dev_env.sh b/jjb/scripts/setup_vpp_dpdk_dev_env.sh index cd5bab447..2c55f166b 100644 --- a/jjb/scripts/setup_vpp_dpdk_dev_env.sh +++ b/jjb/scripts/setup_vpp_dpdk_dev_env.sh @@ -48,6 +48,9 @@ function setup { force_opts="$force_opts --allow-change-held-packages" sudo apt-get -y $force_opts install vpp-ext-deps || true fi + echo "Removing packagecloud.io repository references and running apt-get update" + sudo rm -f /etc/apt/sources.list.d/fdio_*.list + sudo apt-get update -qq || true elif [ "${OS_ID,,}" == "centos" ] ; then if [ "${STREAM}" != "master" ] ; then echo "stream '${STREAM}' is not master: replacing packagecloud repo list with stream specific list" diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh index 52cee0978..bfeaacea9 100644 --- a/jjb/scripts/vpp/build.sh +++ b/jjb/scripts/vpp/build.sh @@ -1,7 +1,6 @@ #!/bin/bash -# basic build script example -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,11 +15,30 @@ echo "---> jjb/scripts/vpp/build.sh" -set -xe -o pipefail +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping build. A newer patch has been merged.\n$line\n" + exit 0 +fi OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +IS_CSIT_VPP_JOB="${IS_CSIT_VPP_JOB:-}" +MAKE_PARALLEL_FLAGS="${MAKE_PARALLEL_FLAGS:-}" +MAKE_PARALLEL_JOBS="${MAKE_PARALLEL_JOBS:-}" +MAKE_TEST_OS="${MAKE_TEST_OS:-ubuntu-18.04}" +MAKE_TEST_MULTIWORKER_OS="${MAKE_TEST_MULTIWORKER_OS:-debian-10}" +VPPAPIGEN_TEST_OS="${VPPAPIGEN_TEST_OS:-${MAKE_TEST_OS}}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" echo "sha1sum of this script: ${0}" sha1sum $0 @@ -35,11 +53,11 @@ sha1sum $0 # echo $CCACHE_DIR does not exist. #fi -if [ "x${MAKE_PARALLEL_FLAGS}" != "x" ] +if [ -n "${MAKE_PARALLEL_FLAGS}" ] then echo "Building VPP. Number of cores for build set with" \ "MAKE_PARALLEL_FLAGS='${MAKE_PARALLEL_FLAGS}'." -elif [ "x${MAKE_PARALLEL_JOBS}" != "x" ] +elif [ -n "${MAKE_PARALLEL_JOBS}" ] then echo "Building VPP. Number of cores for build set with" \ "MAKE_PARALLEL_JOBS='${MAKE_PARALLEL_JOBS}'." @@ -48,15 +66,24 @@ else "using build default ($(grep -c ^processor /proc/cpuinfo))." fi -echo "CC=${CC}" -echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" - -# If we are not a CSIT job just building packages, then use make verify, -# else use make pkg-verify. -if [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] -then - if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] - then +make_build_test() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make UNATTENDED=yes pkg-verify ; then + BUILD_ERROR="FAILED 'make pkg-verify'" + return + fi + if [ "${IS_CSIT_VPP_JOB,,}" == "true" ]; then + # CSIT jobs don't need to run make test + return + fi + if [ -n "${MAKE_PARALLEL_JOBS}" ]; then export TEST_JOBS="${MAKE_PARALLEL_JOBS}" echo "Testing VPP with ${TEST_JOBS} cores." else @@ -64,13 +91,38 @@ then echo "Testing VPP with automatically calculated number of cores. " \ "See test logs for the exact number." fi - echo "Building using \"make verify\"" - [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes verify -else - echo "Building using \"make pkg-verify\"" - [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes pkg-verify -fi + if [ "${OS_ID}-${OS_VERSION_ID}" == "${VPPAPIGEN_TEST_OS}" ]; then + if ! src/tools/vppapigen/test_vppapigen.py; then + BUILD_ERROR="FAILED src/tools/vppapigen/test_vppapigen.py" + return + fi + fi + if [ "${OS_ID}-${OS_VERSION_ID}" == "${MAKE_TEST_OS}" ]; then + if ! make COMPRESS_FAILED_TEST_LOGS=yes RETRIES=3 test; then + BUILD_ERROR="FAILED 'make test'" + return + fi + else + echo "Skip running 'make test' on ${OS_ID}-${OS_VERSION_ID}" + fi + if [ "${OS_ID}-${OS_VERSION_ID}" == "${MAKE_TEST_MULTIWORKER_OS}" ] && git grep VPP_WORKER_CONFIG; then + if ! make VPP_WORKER_CONFIG="workers 2" COMPRESS_FAILED_TEST_LOGS=yes RETRIES=3 test; then + BUILD_ERROR="FAILED 'make test' with VPP_WORKER_CONFIG='workers 2'" + return + else + echo -e "\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} MULTIWORKER MAKE TEST SUCCESSFULLY COMPLETED\n" + fi + else + echo "Skip running 'make test' with VPP_WORKER_CONFIG='workers 2' on ${OS_ID}-${OS_VERSION_ID}" + fi +} -echo "*******************************************************************" -echo "* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED" -echo "*******************************************************************" +if [ "${DRYRUN,,}" != "true" ] ; then + make_build_test +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} BUILD $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/debug-build.sh b/jjb/scripts/vpp/debug-build.sh index a9887340d..dba984f21 100644 --- a/jjb/scripts/vpp/debug-build.sh +++ b/jjb/scripts/vpp/debug-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,11 +15,16 @@ echo "---> jjb/scripts/vpp/debug-build.sh" -set -xe -o pipefail +set -euxo pipefail +line="*************************************************************************" OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" echo "sha1sum of this script: ${0}" sha1sum $0 @@ -27,15 +32,34 @@ sha1sum $0 # run with ASAN on export VPP_EXTRA_CMAKE_ARGS='-DVPP_ENABLE_SANITIZE_ADDR=ON' +make_build_test_debug() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make UNATTENDED=yes build ; then + BUILD_ERROR="FAILED 'make build'" + return + fi + if ! make UNATTENDED=yes TEST_JOBS=auto test-debug ; then + BUILD_ERROR="FAILED 'make test-debug'" + return + fi +} + # clang is not working with ASAN right now - see change 27268 # also, it does not work with gcc-7, we need gcc-8 at least # on ubuntu 20.04 executor the gcc is gcc9 - -make UNATTENDED=yes install-dep -make UNATTENDED=yes install-ext-deps -make UNATTENDED=yes build -make UNATTENDED=yes TEST_JOBS=auto test-debug - -echo "*******************************************************************" -echo "* VPP debug/asan test BUILD on ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} SUCCESSFULLY COMPLETED" -echo "*******************************************************************" +if [ "${DRYRUN,,}" != "true" ] ; then + make_build_test_debug +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} DEBUG BUILD $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/docs.sh b/jjb/scripts/vpp/docs.sh index 0899b6615..866b32217 100644 --- a/jjb/scripts/vpp/docs.sh +++ b/jjb/scripts/vpp/docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,19 +15,29 @@ echo "---> jjb/scripts/vpp/docs.sh" -set -xe -o pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp -[ "$DOC_FILE" ] || DOC_FILE=vpp.docs.zip -[ "$DOC_DIR" ] || DOC_DIR=build-root/docs/html -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site/ -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -[ "$VERSION" ] || VERSION=$(./build-root/scripts/version rpm-version) +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping doxygen docs build. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DOCS_REPO_URL=${DOCS_REPO_URL:-"https://nexus.fd.io/content/sites/site"} +PROJECT_PATH=${PROJECT_PATH:-"io/fd/vpp"} +DOC_FILE=${DOC_FILE:-"vpp.docs.zip"} +DOC_DIR=${DOC_DIR:-"build-root/docs/html"} +SITE_DIR=${SITE_DIR:-"build-root/docs/deploy-site"} +RESOURCES_DIR=${RESOURCES_DIR:-"${SITE_DIR}/src/site/resources"} +MVN=${MVN:-"/opt/apache/maven/bin/mvn"} +VERSION=${VERSION:-"$(./build-root/scripts/version rpm-version)"} make doxygen -if [[ ${JOB_NAME} == *merge* ]]; then +if [[ ${JOB_NAME} == *merge* ]] ; then mkdir -p $(dirname ${RESOURCES_DIR}) mv -f ${DOC_DIR} ${RESOURCES_DIR} cd ${SITE_DIR} diff --git a/jjb/scripts/vpp/gcc-build.sh b/jjb/scripts/vpp/gcc-build.sh index 5cdbc2633..488e8d7d7 100644 --- a/jjb/scripts/vpp/gcc-build.sh +++ b/jjb/scripts/vpp/gcc-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,27 +15,59 @@ echo "---> jjb/scripts/vpp/gcc-build.sh" -set -xe -o pipefail +set -euxo pipefail +line="*************************************************************************" OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') OS_ARCH=$(uname -m) +DRYRUN="${DRYRUN:-}" +BUILD_RESULT="SUCCESSFULLY COMPLETED" +BUILD_ERROR="" +RETVAL="0" echo "sha1sum of this script: ${0}" sha1sum $0 export CC=gcc -make UNATTENDED=yes install-dep -make UNATTENDED=yes install-ext-deps -make UNATTENDED=yes build -# TODO: Add 'smoke test' env var to select smoke test cases -# then update this accordingly. For now pick a few basic suites... -MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash" -for suite in $MAKE_TEST_SUITES ; do - make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test - make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug -done +make_build_release_build_test_gcov_sanity() { + if ! make UNATTENDED=yes install-dep ; then + BUILD_ERROR="FAILED 'make install-dep'" + return + fi + if ! make UNATTENDED=yes install-ext-deps ; then + BUILD_ERROR="FAILED 'make install-ext-deps'" + return + fi + if ! make UNATTENDED=yes build-release ; then + BUILD_ERROR="FAILED 'make build'" + return + fi + if ! make UNATTENDED=yes build ; then + BUILD_ERROR="FAILED 'make build'" + return + fi + # TODO: Add 'smoke test' env var to select smoke test cases + # then update this accordingly. For now pick a few basic suites... + MAKE_TEST_SUITES="vlib vppinfra vpe_api vapi vom bihash" + for suite in $MAKE_TEST_SUITES ; do + if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test ; then + BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test'!" + return + fi + if ! make UNATTENDED=yes GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug ; then + BUILD_ERROR="FAILED 'make GCOV_TESTS=1 TEST_JOBS=auto TEST=$suite test-debug'!" + return + fi + done +} -echo "*******************************************************************" -echo "* VPP GCC on ${OS_ID^^}-${OS_VERSION}-${OS_ARCH^^} BUILD SUCCESSFULLY COMPLETED" -echo "*******************************************************************" +if [ "${DRYRUN,,}" != "true" ] ; then + make_build_release_build_test_gcov_sanity +fi +if [ -n "$BUILD_ERROR" ] ; then + BUILD_RESULT="$BUILD_ERROR" + RETVAL="1" +fi +echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^} GCC BUILD $BUILD_RESULT\n$line\n" +exit $RETVAL diff --git a/jjb/scripts/vpp/make-test-docs.sh b/jjb/scripts/vpp/make-test-docs.sh index d15cac380..d20a7b817 100644 --- a/jjb/scripts/vpp/make-test-docs.sh +++ b/jjb/scripts/vpp/make-test-docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,14 +15,24 @@ echo "---> jjb/scripts/vpp/make-test-docs.sh" -set -xe -o pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp -[ "$DOC_DIR" ] || DOC_DIR=build-root/build-test/doc/html -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/vpp_make_test -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -[ "$VERSION" ] || VERSION=$(./build-root/scripts/version rpm-version) +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping 'make test' doxygen docs build. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DOCS_REPO_URL=${DOCS_REPO_URL:-"https://nexus.fd.io/content/sites/site"} +PROJECT_PATH=${PROJECT_PATH:-"io/fd/vpp"} +DOC_DIR=${DOC_DIR:-"build-root/build-test/doc/html"} +SITE_DIR=${SITE_DIR:-"build-root/docs/deploy-site"} +RESOURCES_DIR=${RESOURCES_DIR:-"${SITE_DIR}/src/site/resources/vpp_make_test"} +MVN=${MVN:-"/opt/apache/maven/bin/mvn"} +VERSION=${VERSION:-"$(./build-root/scripts/version rpm-version)"} make test-doc diff --git a/jjb/scripts/vpp/sphinx-docs.sh b/jjb/scripts/vpp/sphinx-docs.sh index b9eb64bc7..d80b4cd38 100644 --- a/jjb/scripts/vpp/sphinx-docs.sh +++ b/jjb/scripts/vpp/sphinx-docs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,15 +15,25 @@ echo "---> jjb/scripts/vpp/sphinx-docs.sh" -set -xe -o pipefail -[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" -[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/vpp -[ "$DOC_FILE" ] || DOC_FILE=vpp.docs.zip -[ "$DOC_DIR" ] || DOC_DIR=./docs/_build/html -[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site -[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources -[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" -[ "$VERSION" ] || VERSION=$(./build-root/scripts/version rpm-version) +set -euxo pipefail + +line="*************************************************************************" +# Don't build anything if this is a merge job being run when +# the git HEAD id is not the same as the Gerrit New Revision id. +if [[ ${JOB_NAME} == *merge* ]] && [ -n "${GERRIT_NEWREV:-}" ] && + [ "$GERRIT_NEWREV" != "$GIT_COMMIT" ] ; then + echo -e "\n$line\nSkipping sphinx docs build. A newer patch has been merged.\n$line\n" + exit 0 +fi + +DOCS_REPO_URL=${DOCS_REPO_URL:-"https://nexus.fd.io/content/sites/site"} +PROJECT_PATH=${PROJECT_PATH:-"io/fd/vpp"} +DOC_FILE=${DOC_FILE:-"vpp.docs.zip"} +DOC_DIR=${DOC_DIR:-"./docs/_build/html"} +SITE_DIR=${SITE_DIR:-"build-root/docs/deploy-site"} +RESOURCES_DIR=${RESOURCES_DIR:-"${SITE_DIR}/src/site/resources"} +MVN=${MVN:-"/opt/apache/maven/bin/mvn"} +VERSION=${VERSION:-"$(./build-root/scripts/version rpm-version)"} make docs-venv CONFIRM=-y FORCE=--force-yes make docs diff --git a/jjb/vpp/docs.yaml b/jjb/vpp/docs.yaml index c9e2bb853..6965995a6 100644 --- a/jjb/vpp/docs.yaml +++ b/jjb/vpp/docs.yaml @@ -14,10 +14,6 @@ stream: - master: branch: 'master' - - '1908': - branch: 'stable/1908' - - '2005': - branch: 'stable/2005' - '2009': branch: 'stable/2009' - '2101': diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index 93f1157aa..c321f0458 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -1,3 +1,16 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- # PROJECT DEFINITIONS # @@ -5,16 +18,13 @@ # vpp-x86_64, vpp-aarch64, vpp-csit # # NOTES: -# - The 'branch-head-refspec' attribute is used for sandbox testing. +# - The 'branch-refspec' attribute is used for sandbox testing. # Use the get_gerrit_refspec() bash function in -# .../ci-management/docker/scripts/lib_common.sh +# .../ci-management/extras/bash/sandbox_test_functions.sh # to query gerrit for the refspec of HEAD in a branch. # TODO: see if there is a refspec that works without requiring # use of get_gerrit_refspec and hand entering the result # in the WebUI 'Build With Parameters' gerrit refspec field. -# - Use the following to import get_gerrit_refspec into a bash shell -# from the ci-management directory: -# $ eval "$(grep -A12 get_gerrit ./docker/scripts/lib_common.sh)" # [end] NOTES # VPP-CHECKSTYLE-X86_64 PROJECT @@ -26,25 +36,20 @@ stream: - master: branch: 'master' + branch-refspec: '' + os: 'ubuntu2004' repo-stream-part: 'master' - branch-head-refspec: '' - - '1908': - branch: 'stable/1908' - repo-stream-part: 'stable.1908' - branch-head-refspec: '' - - '2005': - branch: 'stable/2005' - repo-stream-part: 'stable.2005' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' + os: 'ubuntu1804' repo-stream-part: 'stable.2009' - branch-head-refspec: '' + os: 'ubuntu1804' - '2101': branch: 'stable/2101' + branch-refspec: '' + os: 'ubuntu1804' repo-stream-part: 'stable.2101' - branch-head-refspec: '' - os: 'ubuntu1804' executor-arch: 'x86_64' # [end] VPP-CHECKSTYLE-X86_64 PROJECT @@ -56,27 +61,20 @@ - 'vpp-merge-{stream}-{os}-{executor-arch}' project: 'vpp' make-parallel-jobs: '4' + archive-artifacts: '/tmp/vpp-failed-unittests/*/*' stream: - master: branch: 'master' + branch-refspec: '' repo-stream-part: 'master' - branch-head-refspec: '' - - '1908': - branch: 'stable/1908' - repo-stream-part: 'stable.1908' - branch-head-refspec: '' - - '2005': - branch: 'stable/2005' - repo-stream-part: 'stable.2005' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - branch-head-refspec: '' - '2101': branch: 'stable/2101' + branch-refspec: '' repo-stream-part: 'stable.2101' - branch-head-refspec: '' os: - ubuntu1804: repo-os-part: 'ubuntu.bionic.main' @@ -91,22 +89,17 @@ - 'vpp-merge-{stream}-{os}-{executor-arch}' project: 'vpp' make-parallel-jobs: '4' + archive-artifacts: '/tmp/vpp-failed-unittests/*/*' stream: - - '1908': - branch: 'stable/1908' - repo-stream-part: 'stable.1908' - branch-head-refspec: '' - - '2005': - branch: 'stable/2005' - repo-stream-part: 'stable.2005' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - branch-head-refspec: '' os: - centos7: repo-os-part: 'centos7' + - debian9: + repo-os-part: 'stretch' executor-arch: 'x86_64' # [end] VPP-OS-EOL-X86_64 PROJECT @@ -133,25 +126,23 @@ - 'vpp-merge-{stream}-{os}-{executor-arch}' project: 'vpp' make-parallel-jobs: '4' + archive-artifacts: '/tmp/vpp-failed-unittests/*/*' stream: - master: branch: 'master' + branch-refspec: '' repo-stream-part: 'master' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - branch-head-refspec: '' - '2101': branch: 'stable/2101' + branch-refspec: '' repo-stream-part: 'stable.2101' - branch-head-refspec: '' - os: os: - centos8: repo-os-part: 'centos8' - - debian9: - repo-os-part: 'stretch' - debian10: repo-os-part: 'buster' - ubuntu2004: @@ -161,7 +152,7 @@ # VPP-DEBUG-X86_64 PROJECT # -# Currently a Non-Voting job due to intermittent failures. +# Currently a periodic job due to intermittent failures. # Limiting branches to master and 2009, since other branches # may not get fixes for the intermittent failures. # @@ -171,15 +162,21 @@ - 'vpp-debug-verify-{stream}-{os}-{executor-arch}' project: 'vpp' make-parallel-jobs: '4' + archive-artifacts: '/tmp/vpp-failed-unittests/*/*' + comment-trigger-value: 'debug-verify' stream: - master: branch: 'master' + branch-refspec: '' + # Once per hour + periodicity: 'H * * * *' repo-stream-part: 'master' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' + # Once per day + periodicity: 'H H * * *' repo-stream-part: 'stable.2009' - branch-head-refspec: '' os: - ubuntu2004: repo-os-part: 'ubuntu.focal.main' @@ -191,8 +188,8 @@ # Beta jobs are experimental jobs for testing new OS versions on master. # # They are only manually triggered. Either using the Jenkins WebUI -# 'Build with Parameters' to run a single job or adding 'vpp-beta-verify' -# or 'vpp-beta-merge' as a comment to run all beta verify or merge jobs. +# 'Build with Parameters' to run a single job or adding 'beta-verify' +# or 'beta-merge' as a comment to run all beta verify or merge jobs. # # Once testing is complete, the os is moved to the current # 'vpp-os-in-XXXX-x86_64' project. If no OSes are being @@ -202,16 +199,17 @@ # name: vpp-beta-x86_64 # jobs: # - 'vpp-beta-verify-{stream}-{os}-{executor-arch}': -# comment-trigger-value: 'vpp-beta-verify' +# comment-trigger-value: 'beta-verify' # - 'vpp-beta-merge-{stream}-{os}-{executor-arch}': -# comment-trigger-value: 'vpp-beta-merge' +# comment-trigger-value: 'beta-merge' # project: 'vpp' # make-parallel-jobs: '4' +# archive-artifacts: '/tmp/vpp-failed-unittests/*/*' # stream: # - master: # branch: 'master' +# branch-refspec: '' # repo-stream-part: 'master' -# branch-head-refspec: '' # os: # - debian10: # repo-os-part: 'buster' @@ -231,16 +229,16 @@ stream: - master: branch: 'master' + branch-refspec: '' repo-stream-part: 'master' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - branch-head-refspec: '' - '2101': branch: 'stable/2101' + branch-refspec: '' repo-stream-part: 'stable.2101' - branch-head-refspec: '' # [end] VPP-GCC-X86_64 PROJECT # VPP-AARCH64 PROJECT @@ -251,31 +249,33 @@ - 'vpp-merge-{stream}-{os}-{executor-arch}' project: 'vpp' make-parallel-jobs: '16' + archive-artifacts: '/tmp/vpp-failed-unittests/*/*' stream: - master: branch: 'master' + branch-refspec: '' repo-stream-part: 'master' - branch-head-refspec: '' - - '1908': - branch: 'stable/1908' - repo-stream-part: 'stable.1908' - branch-head-refspec: '' - - '2005': - branch: 'stable/2005' - repo-stream-part: 'stable.2005' - branch-head-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - branch-head-refspec: '' - '2101': branch: 'stable/2101' + branch-refspec: '' repo-stream-part: 'stable.2101' - branch-head-refspec: '' os: + - centos8: + repo-os-part: 'centos8' - ubuntu1804: repo-os-part: 'ubuntu-arm.bionic.main' executor-arch: 'aarch64' + exclude: + # OS introduced after 2009 + - stream: '2009' + os: 'centos8' + # OS introduced after 2101 + - stream: '2101' + os: 'centos8' # [end] VPP-AARCH64 PROJECT # VPP-OS-IN-2009-AARCH64 PROJECT @@ -302,15 +302,16 @@ # # project: 'vpp' # make-parallel-jobs: '16' +# archive-artifacts: '/tmp/vpp-failed-unittests/*/*' # stream: # - master: # branch: 'master' +# branch-refspec: '' # repo-stream-part: 'master' -# branch-head-refspec: '' # - '2009': # branch: 'stable/2009' +# branch-refspec: '' # repo-stream-part: 'stable.2009' -# branch-head-refspec: '' # os: # - ubuntu2004: # repo-os-part: 'ubuntu.focal.main' @@ -322,8 +323,8 @@ # Beta jobs are experimental jobs for testing new OS versions on master. # # They are only manually triggered. Either using the Jenkins WebUI -# 'Build with Parameters' to run a single job or adding 'vpp-beta-verify' -# or 'vpp-beta-merge' as a comment to run all beta verify or merge jobs. +# 'Build with Parameters' to run a single job or adding 'beta-verify' +# or 'beta-merge' as a comment to run all beta verify or merge jobs. # # Once testing is complete, the os is moved to the current # 'vpp-os-in-XXXX-aarch64' project. If no OSes are being @@ -338,25 +339,35 @@ comment-trigger-value: 'beta-merge' project: 'vpp' make-parallel-jobs: '16' + archive-artifacts: '/tmp/vpp-failed-unittests/*/*' stream: - master: branch: 'master' + branch-refspec: '' repo-stream-part: 'master' - branch-head-refspec: '' os: - - centos8: - repo-os-part: 'centos8' - ubuntu2004: repo-os-part: 'ubuntu.focal.main' executor-arch: aarch64 # [end] VPP-BETA-AARCH64 PROJECT -# VPP-CSIT-PERPATCH PROJECT +# VPP-CSIT-VERIFY PROJECT - project: - name: vpp-csit-perpatch + name: vpp-csit-verify jobs: - - 'vpp-csit-verify-device-{stream}-{device-node-arch}': + - 'vpp-csit-verify-device-perpatch': make-parallel-jobs: '32' + exclude: + - device-node-arch: '1n-tx2' + - 'vpp-csit-verify-device-periodic': + make-parallel-jobs: '32' + periodicity: 'H * * * *' + skip-vote: 'true' + comment-trigger-value: 'devicetest-periodic' + exclude: + - stream: '2009' + - stream: '2101' + - device-node-arch: '1n-skx' - 'vpp-csit-verify-perf-{stream}-{node-arch}' project: 'vpp' os: ubuntu1804 @@ -366,24 +377,20 @@ stream: - master: branch: 'master' + branch-refspec: '' repo-stream-part: 'master' - - '1908': - branch: 'stable/1908' - repo-stream-part: 'stable.1908' - - '2005': - branch: 'stable/2005' - repo-stream-part: 'stable.2005' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - '2101': branch: 'stable/2101' + branch-refspec: '' repo-stream-part: 'stable.2101' device-node-arch: - 1n-skx - 1n-tx2: device-executor: 'vpp-csit-arm-ubuntu18' - skip-vote: 'true' node-arch: - 2n-clx - 2n-skx @@ -395,15 +402,11 @@ - 3n-tsh: executor: 'ubuntu1804arm-s' make-parallel-jobs: '16' - exclude: - - stream: '1908' - device-node-arch: '1n-tx2' - - stream: '2001' - device-node-arch: '1n-tx2' - - stream: '2005' - device-node-arch: '1n-tx2' + - 2n-tx2: + executor: 'ubuntu1804arm-s' + make-parallel-jobs: '16' -# [end] VPP-CSIT-PERPATCH PROJECT +# [end] VPP-CSIT-VERIFY PROJECT # VPP-CSIT-API-CRC PROJECT # @@ -417,17 +420,17 @@ executor: 'vpp-csit-device' skip-vote: 'false' stream: - - '1908': - branch: 'stable/1908' - repo-stream-part: 'stable.1908' + - master: + branch: 'master' + branch-refspec: '' - '2009': branch: 'stable/2009' + branch-refspec: '' repo-stream-part: 'stable.2009' - '2101': branch: 'stable/2101' + branch-refspec: '' repo-stream-part: 'stable.2101' - - master: - branch: 'master' os: ubuntu1804 # [end] VPP-CSIT-API-CRC PROJECT @@ -461,7 +464,6 @@ project-type: freestyle node: 'builder-{os}-prod-{executor-arch}' concurrent: true - archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core' build-discarder: daysToKeep: '{build-days-to-keep}' @@ -469,17 +471,20 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' - os-parameter: os: '{os}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.ubuntu.bionic.main' - - gerrit-refspec-parameter: - refspec: '{branch-head-refspec}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -565,8 +570,8 @@ - ../scripts/vpp/commitmsg.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish + # [end] VPP-CHECKSTYLE-VERIFY JOB TEMPLATE # VPP-VERIFY JOB TEMPLATE @@ -576,7 +581,6 @@ project-type: freestyle node: 'builder-{os}-prod-{executor-arch}' concurrent: true - archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core' build-discarder: daysToKeep: '{build-days-to-keep}' @@ -584,25 +588,24 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' - stream-parameter: stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - gerrit-refspec-parameter: - refspec: '{branch-head-refspec}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -634,8 +637,7 @@ - ../scripts/vpp/build.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-VERIFY JOB TEMPLATE # VPP-DEBUG-VERIFY JOB TEMPLATE @@ -645,7 +647,6 @@ project-type: freestyle node: 'builder-{os}-prod-{executor-arch}' concurrent: true - archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core' build-discarder: daysToKeep: '{build-days-to-keep}' @@ -653,21 +654,24 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: 'ubuntu1804' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.ubuntu.bionic.main' - stream-parameter: stream: '{stream}' - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - gerrit-refspec-parameter: - refspec: '{branch-head-refspec}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -680,9 +684,9 @@ build-timeout: '{build-timeout}' triggers: - # TODO: Ensure all supported jobs pass reliably before - # enabling voting. - - gerrit-trigger-checkstyle-nonvoting: + - timed: '{periodicity}' + - gerrit-trigger-manually-triggered: + comment-trigger-value: '{comment-trigger-value}' name: '{project}' branch: '{branch}' @@ -701,9 +705,8 @@ - ../scripts/vpp/debug-build.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' -# [end] VPP-DEBUG-VERIFY JOB TEMPLATE + - fdio-infra-publish +# [end] VPP-DEBUG-VERIFY-PERIODIC JOB TEMPLATE # VPP-MERGE JOB TEMPLATE - job-template: @@ -718,27 +721,22 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' - - maven-project-parameter: - maven: 'mvn33-new' - - maven-exec: - maven-version: 'mvn33-new' - - stream-parameter: - stream: '{stream}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' + - stream-parameter: + stream: '{stream}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -785,8 +783,7 @@ - ../scripts/packagecloud_push.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-MERGE JOB TEMPLATE # VPP-BETA-VERIFY JOB TEMPLATE @@ -796,7 +793,6 @@ project-type: freestyle node: 'builder-{os}-prod-{executor-arch}' concurrent: true - archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core' build-discarder: daysToKeep: '{build-days-to-keep}' @@ -804,25 +800,24 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' - stream-parameter: stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - gerrit-refspec-parameter: - refspec: '{branch-head-refspec}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -855,8 +850,7 @@ - ../scripts/vpp/build.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-BETA-VERIFY JOB TEMPLATE # VPP-BETA-MERGE JOB TEMPLATE @@ -872,32 +866,27 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' - - maven-project-parameter: - maven: 'mvn33-new' - - maven-exec: - maven-version: 'mvn33-new' - - stream-parameter: - stream: '{stream}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' + - stream-parameter: + stream: '{stream}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: credentials-id: 'jenkins-gerrit-credentials' - refspec: '' + refspec: '$GERRIT_REFSPEC' choosing-strategy: 'default' wrappers: @@ -934,16 +923,12 @@ - shell: !include-raw-escape: - ../scripts/vpp/build.sh - - provide-maven-settings: - settings-file: 'vpp-settings' - global-settings-file: 'global-settings' - shell: !include-raw-escape: - ../scripts/packagecloud_push.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-BETA-MERGE JOB TEMPLATE # VPP-GCC-VERIFY JOB TEMPLATE @@ -953,7 +938,6 @@ project-type: freestyle node: 'builder-{os}-prod-{executor-arch}' concurrent: true - archive-artifacts: '/tmp/vpp-failed-unittests/*/* **/core' build-discarder: daysToKeep: '{build-days-to-keep}' @@ -961,25 +945,24 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - gerrit-parameter: branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' - stream-parameter: stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' - - gerrit-refspec-parameter: - refspec: '{branch-head-refspec}' + - workspace-archive-artifacts-parameter: + artifacts: '{archive-artifacts}' scm: - gerrit-trigger-scm: @@ -1011,13 +994,13 @@ - ../scripts/vpp/gcc-build.sh publishers: - - fdio-infra-shiplogs: - maven-version: 'mvn36' + - fdio-infra-publish # [end] VPP-GCC-VERIFY JOB TEMPLATE -# VPP-CSIT-VERIFY-DEVICE JOB TEMPLATE +# VPP-CSIT-VERIFY-DEVICE-PERIODIC JOB TEMPLATE - job-template: name: 'vpp-csit-verify-device-{stream}-{device-node-arch}' + id: 'vpp-csit-verify-device-periodic' description: | <ul> <li>executor @@ -1038,27 +1021,112 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' + - archive-artifacts-parameter: + artifacts: '{archive-artifacts}' + - gerrit-csit-refspec-parameter + - gerrit-event-comment-text-parameter + - gerrit-event-type-parameter - gerrit-parameter: branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.ubuntu.bionic.main' - - gerrit-refspec-parameter - - gerrit-event-type-parameter - - gerrit-event-comment-text-parameter - - gerrit-csit-refspec-parameter - stream-parameter: stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. + + scm: + - gerrit-trigger-scm: + credentials-id: 'jenkins-gerrit-credentials' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'default' + + wrappers: + - fdio-infra-wrappers: + build-timeout: '90' + + triggers: + - timed: '{periodicity}' + - gerrit-trigger-manually-triggered: + comment-trigger-value: '{comment-trigger-value}' + name: '{project}' + branch: '{branch}' + + builders: + - shell: + !include-raw-escape: + - ../scripts/setup_executor_env.sh + - shell: + !include-raw-escape: + - ../scripts/setup_vpp_ubuntu_docker_test.sh + - shell: + !include-raw-escape: + - ../scripts/setup_vpp_dpdk_dev_env.sh + - shell: + !include-raw-escape: + - ../scripts/vpp/csit-device.sh + + publishers: + - robot-report: + output-path: 'archives' + + - fdio-infra-shiplogs: + maven-version: 'mvn36' +# [end] VPP-CSIT-VERIFY-DEVICE-PERIODIC JOB TEMPLATE + +# VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE +- job-template: + name: 'vpp-csit-verify-device-{stream}-{device-node-arch}' + id: 'vpp-csit-verify-device-perpatch' + description: | + <ul> + <li>executor + <ul> + <li>{device-executor} + </ul> + </ul> + + project-type: freestyle + node: '{device-executor}' + concurrent: true + archive-artifacts: '**/csit_current/**/*.*' + latest-only: false + + build-discarder: + daysToKeep: '{build-days-to-keep}' + numToKeep: 100 + artifactDaysToKeep: '{build-artifact-days-to-keep}' + artifactNumToKeep: '{build-artifact-num-to-keep}' + + # Please keep parameters in alphabetical order + parameters: + - archive-artifacts-parameter: + artifacts: '{archive-artifacts}' + - gerrit-csit-refspec-parameter + - gerrit-event-comment-text-parameter + - gerrit-event-type-parameter + - gerrit-parameter: + branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' - make-parallel-jobs-parameter: make-parallel-jobs: '{make-parallel-jobs}' + - os-parameter: + os: '{os}' + - project-parameter: + project: '{project}' + - repo-name-parameter: + repo-name: '{repo-stream-part}.ubuntu.bionic.main' + - stream-parameter: + stream: '{stream}' scm: - gerrit-trigger-scm: @@ -1096,7 +1164,7 @@ - fdio-infra-shiplogs: maven-version: 'mvn36' -# [end] VPP-CSIT-VERIFY-DEVICE JOB TEMPLATE +# [end] VPP-CSIT-VERIFY-DEVICE-PERPATCH JOB TEMPLATE # VPP-CSIT-VERIFY-PERF JOB TEMPLATE - job-template: @@ -1127,31 +1195,31 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' + - archive-artifacts-parameter: + artifacts: '{archive-artifacts}' + - csit-perf-trial-duration-parameter: + csit-perf-trial-duration: "10.0" + - csit-perf-trial-multiplicity-parameter: + csit-perf-trial-multiplicity: "5" + - gerrit-csit-refspec-parameter + - gerrit-event-comment-text-parameter + - gerrit-event-type-parameter + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' - gerrit-parameter: branch: '{branch}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' - os-parameter: os: '{os}' + - project-parameter: + project: '{project}' - repo-name-parameter: repo-name: '{repo-stream-part}.ubuntu.bionic.main' - - gerrit-refspec-parameter - - gerrit-event-type-parameter - - gerrit-event-comment-text-parameter - - gerrit-csit-refspec-parameter - - csit-perf-trial-duration-parameter: - csit-perf-trial-duration: "10.0" - - csit-perf-trial-multiplicity-parameter: - csit-perf-trial-multiplicity: "5" - stream-parameter: stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. - - make-parallel-jobs-parameter: - make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -1188,7 +1256,8 @@ - robot-report: output-path: 'csit_current/0/' - - lf-infra-publish + - fdio-infra-shiplogs: + maven-version: 'mvn36' # [end] VPP-CSIT-VERIFY-PERF JOB TEMPLATE # VPP-CSIT-VERIFY-API-CRC JOB TEMPLATE @@ -1198,7 +1267,6 @@ project-type: freestyle node: '{os}-us' concurrent: true - archive-artifacts: '' latest-only: false build-discarder: @@ -1207,23 +1275,23 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + # Please keep parameters in alphabetical order parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - gerrit-refspec-parameter - - gerrit-csit-refspec-parameter - - stream-parameter: - stream: '{stream}' - - string: - name: ARCHIVE_ARTIFACTS - default: '{archive-artifacts}' - description: Artifacts to archive to the logs server. + - archive-artifacts-parameter: + artifacts: '{archive-artifacts}' # Not sure whether not failing has any useful usage, # but it does not hurt to have some flexibility for future. - csit-fail-on-crc-mismatch-parameter: fail-on-crc-mismatch: 'True' + - gerrit-csit-refspec-parameter + - gerrit-parameter: + branch: '{branch}' + - gerrit-refspec-parameter: + refspec: '{branch-refspec}' + - project-parameter: + project: '{project}' + - stream-parameter: + stream: '{stream}' scm: - gerrit-trigger-scm: @@ -1236,8 +1304,6 @@ build-timeout: 300 triggers: - # TODO: Make other verify jobs trigger on success of this one, - # without breaking verification for VPP branches before stable/1908. - gerrit-trigger-checkstyle: name: '{project}' branch: '{branch}' |