From 9f2d4cec9b9db1ebbcf1c5bbbf7fa54170006208 Mon Sep 17 00:00:00 2001 From: Juraj Linkeš Date: Fri, 29 May 2020 15:16:10 +0200 Subject: Add MAKE_PARALLEL_JOBS parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VPP uses the MAKE_PARALLEL_JOBS variable to limit the number of cpus used for build. Make that varialbe configurable as job paramter instead of setting it in script for easier job management. Change-Id: Iaaafdb216c3af9244827b270c540a70b5844021f Signed-off-by: Juraj Linkeš --- jjb/global-defaults.yaml | 3 +++ jjb/global-macros.yaml | 8 ++++++++ jjb/scripts/vpp/arm-build.sh | 8 +++++++- jjb/scripts/vpp/build.sh | 5 +++++ jjb/scripts/vpp/parallel-vars.sh | 11 ----------- jjb/vpp/vpp.yaml | 33 ++++++++++++++++++++++++++++----- 6 files changed, 51 insertions(+), 17 deletions(-) delete mode 100644 jjb/scripts/vpp/parallel-vars.sh diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml index cba706d7..d77b53d7 100644 --- a/jjb/global-defaults.yaml +++ b/jjb/global-defaults.yaml @@ -24,6 +24,9 @@ # voting skip-vote: false + # build + make-parallel-jobs: 4 + # CSIT csit-perf-trial-duration: 1.0 csit-perf-trial-multiplicity: 1 diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 5247bfc6..8354f5ce 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -32,6 +32,14 @@ default: '{cc}' description: "CC parameter, can be gnu or clang" +- parameter: + name: make-parallel-jobs-parameter + parameters: + - string: + name: MAKE_PARALLEL_JOBS + default: '{make-parallel-jobs}' + description: "The number of cores to use for builds." + - parameter: name: csit-fail-on-crc-mismatch-parameter parameters: diff --git a/jjb/scripts/vpp/arm-build.sh b/jjb/scripts/vpp/arm-build.sh index 2abe6105..798e90e9 100644 --- a/jjb/scripts/vpp/arm-build.sh +++ b/jjb/scripts/vpp/arm-build.sh @@ -46,11 +46,17 @@ echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" # If and only if we are doing verify *after* make verify was made to work # and we are not a CSIT job just building packages, then use make verify, # else use the old build-root/vagrant/build.sh +if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] +then + echo "Building with MAKE_PARALLEL_JOBS=${MAKE_PARALLEL_JOBS}" + TEST_JOBS="TEST_JOBS=${MAKE_PARALLEL_JOBS}" +fi + if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \ [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] then echo "Building using \"make verify\"" - [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes verify SKIP_AARCH64=yes + [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes SKIP_AARCH64=yes ${TEST_JOBS} verify else echo "Building using \"make build-root/vagrant/build.sh\"" [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes install-dep diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh index 6fdaec13..9fd989fd 100644 --- a/jjb/scripts/vpp/build.sh +++ b/jjb/scripts/vpp/build.sh @@ -40,6 +40,11 @@ echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" # If and only if we are doing verify *after* make verify was made to work # and we are not a CSIT job just building packages, then use make verify, # else use make pkg-verify. +if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] +then + echo "Building with MAKE_PARALLEL_JOBS=${MAKE_PARALLEL_JOBS}" +fi + if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \ [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] then diff --git a/jjb/scripts/vpp/parallel-vars.sh b/jjb/scripts/vpp/parallel-vars.sh deleted file mode 100644 index 1a5de149..00000000 --- a/jjb/scripts/vpp/parallel-vars.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -CORES=4 -if [[ $(uname -m) == 'aarch64' ]] -then - CORES=16 -fi - -echo "Using MAKE_PARALLEL_FLAGS='-j $CORES' TEST_JOBS=$CORES for parallel build/test" - -export MAKE_PARALLEL_FLAGS="-j $CORES" -export TEST_JOBS=$CORES diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index 70c0ddaa..216ec400 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -95,7 +95,8 @@ - project: name: vpp-perpatch jobs: - - 'vpp-csit-verify-device-{stream}-{device-node-arch}' + - 'vpp-csit-verify-device-{stream}-{device-node-arch}': + make-parallel-jobs: '32' - 'vpp-csit-verify-perf-{stream}-{node-arch}' project: 'vpp' os: ubuntu1804 @@ -133,6 +134,7 @@ - 3n-dnv - 3n-tsh: executor: 'ubuntu1804arm-s' + make-parallel-jobs: '16' exclude: - stream: '1904' os: 'ubuntu1804' @@ -189,6 +191,7 @@ os: - ubuntu1804: repo-os-part: 'ubuntu-arm.bionic.main' + make-parallel-jobs: '16' # TODO: Merge with (e.g.) vpp-perpatch or similar, # when number of stable branches supported matches. @@ -257,6 +260,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -314,6 +319,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -388,6 +395,9 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' + - gerrit-refspec-parameter scm: - gerrit-trigger-scm: @@ -413,8 +423,6 @@ - ../scripts/setup_vpp_dpdk_dev_env.sh - shell: !include-raw-escape: - - ../scripts/vpp/parallel-vars.sh - # Do not split, the line below wants env vars from the line above. - ../scripts/vpp/arm-build.sh publishers: @@ -452,6 +460,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -485,8 +495,6 @@ - ../scripts/setup_vpp_dpdk_dev_env.sh - shell: !include-raw-escape: - - ../scripts/vpp/parallel-vars.sh - # Do not split, the line below wants env vars from the line above. - ../scripts/vpp/arm-build-no-test.sh - provide-maven-settings: @@ -629,6 +637,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -690,6 +700,9 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' + - gerrit-refspec-parameter scm: - gerrit-trigger-scm: @@ -746,6 +759,8 @@ repo-name: '{repo-stream-part}.ubuntu.bionic.main' - stream-parameter: stream: '{stream}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -907,6 +922,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -986,6 +1003,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -1076,6 +1095,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -1162,6 +1183,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: -- cgit 1.2.3-korg