diff options
author | Juraj Linkeš <juraj.linkes@pantheon.tech> | 2020-05-29 15:16:10 +0200 |
---|---|---|
committer | Juraj Linkeš <juraj.linkes@pantheon.tech> | 2020-06-01 08:18:28 +0200 |
commit | 9f2d4cec9b9db1ebbcf1c5bbbf7fa54170006208 (patch) | |
tree | fdaee0174d62da029875b1aefe10bb18ce320bce | |
parent | 0e4d21ba4f01911130e2ed54bd1faff8baf8b5b0 (diff) |
Add MAKE_PARALLEL_JOBS parameter
VPP uses the MAKE_PARALLEL_JOBS variable to limit the number of cpus
used for build. Make that varialbe configurable as job paramter instead
of setting it in script for easier job management.
Change-Id: Iaaafdb216c3af9244827b270c540a70b5844021f
Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
-rw-r--r-- | jjb/global-defaults.yaml | 3 | ||||
-rw-r--r-- | jjb/global-macros.yaml | 8 | ||||
-rw-r--r-- | jjb/scripts/vpp/arm-build.sh | 8 | ||||
-rw-r--r-- | jjb/scripts/vpp/build.sh | 5 | ||||
-rw-r--r-- | jjb/scripts/vpp/parallel-vars.sh | 11 | ||||
-rw-r--r-- | jjb/vpp/vpp.yaml | 33 |
6 files changed, 51 insertions, 17 deletions
diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml index cba706d7..d77b53d7 100644 --- a/jjb/global-defaults.yaml +++ b/jjb/global-defaults.yaml @@ -24,6 +24,9 @@ # voting skip-vote: false + # build + make-parallel-jobs: 4 + # CSIT csit-perf-trial-duration: 1.0 csit-perf-trial-multiplicity: 1 diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 5247bfc6..8354f5ce 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -33,6 +33,14 @@ description: "CC parameter, can be gnu or clang" - parameter: + name: make-parallel-jobs-parameter + parameters: + - string: + name: MAKE_PARALLEL_JOBS + default: '{make-parallel-jobs}' + description: "The number of cores to use for builds." + +- parameter: name: csit-fail-on-crc-mismatch-parameter parameters: - string: diff --git a/jjb/scripts/vpp/arm-build.sh b/jjb/scripts/vpp/arm-build.sh index 2abe6105..798e90e9 100644 --- a/jjb/scripts/vpp/arm-build.sh +++ b/jjb/scripts/vpp/arm-build.sh @@ -46,11 +46,17 @@ echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" # If and only if we are doing verify *after* make verify was made to work # and we are not a CSIT job just building packages, then use make verify, # else use the old build-root/vagrant/build.sh +if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] +then + echo "Building with MAKE_PARALLEL_JOBS=${MAKE_PARALLEL_JOBS}" + TEST_JOBS="TEST_JOBS=${MAKE_PARALLEL_JOBS}" +fi + if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \ [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] then echo "Building using \"make verify\"" - [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes verify SKIP_AARCH64=yes + [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes SKIP_AARCH64=yes ${TEST_JOBS} verify else echo "Building using \"make build-root/vagrant/build.sh\"" [ "x${DRYRUN}" == "xTrue" ] || make UNATTENDED=yes install-dep diff --git a/jjb/scripts/vpp/build.sh b/jjb/scripts/vpp/build.sh index 6fdaec13..9fd989fd 100644 --- a/jjb/scripts/vpp/build.sh +++ b/jjb/scripts/vpp/build.sh @@ -40,6 +40,11 @@ echo "IS_CSIT_VPP_JOB=${IS_CSIT_VPP_JOB}" # If and only if we are doing verify *after* make verify was made to work # and we are not a CSIT job just building packages, then use make verify, # else use make pkg-verify. +if [ "x${MAKE_PARALLEL_JOBS}" != "x" ] +then + echo "Building with MAKE_PARALLEL_JOBS=${MAKE_PARALLEL_JOBS}" +fi + if (git log --oneline | grep 37682e1 > /dev/null 2>&1) && \ [ "x${IS_CSIT_VPP_JOB}" != "xTrue" ] then diff --git a/jjb/scripts/vpp/parallel-vars.sh b/jjb/scripts/vpp/parallel-vars.sh deleted file mode 100644 index 1a5de149..00000000 --- a/jjb/scripts/vpp/parallel-vars.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -CORES=4 -if [[ $(uname -m) == 'aarch64' ]] -then - CORES=16 -fi - -echo "Using MAKE_PARALLEL_FLAGS='-j $CORES' TEST_JOBS=$CORES for parallel build/test" - -export MAKE_PARALLEL_FLAGS="-j $CORES" -export TEST_JOBS=$CORES diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index 70c0ddaa..216ec400 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -95,7 +95,8 @@ - project: name: vpp-perpatch jobs: - - 'vpp-csit-verify-device-{stream}-{device-node-arch}' + - 'vpp-csit-verify-device-{stream}-{device-node-arch}': + make-parallel-jobs: '32' - 'vpp-csit-verify-perf-{stream}-{node-arch}' project: 'vpp' os: ubuntu1804 @@ -133,6 +134,7 @@ - 3n-dnv - 3n-tsh: executor: 'ubuntu1804arm-s' + make-parallel-jobs: '16' exclude: - stream: '1904' os: 'ubuntu1804' @@ -189,6 +191,7 @@ os: - ubuntu1804: repo-os-part: 'ubuntu-arm.bionic.main' + make-parallel-jobs: '16' # TODO: Merge with (e.g.) vpp-perpatch or similar, # when number of stable branches supported matches. @@ -257,6 +260,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -314,6 +319,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -388,6 +395,9 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' + - gerrit-refspec-parameter scm: - gerrit-trigger-scm: @@ -413,8 +423,6 @@ - ../scripts/setup_vpp_dpdk_dev_env.sh - shell: !include-raw-escape: - - ../scripts/vpp/parallel-vars.sh - # Do not split, the line below wants env vars from the line above. - ../scripts/vpp/arm-build.sh publishers: @@ -452,6 +460,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -485,8 +495,6 @@ - ../scripts/setup_vpp_dpdk_dev_env.sh - shell: !include-raw-escape: - - ../scripts/vpp/parallel-vars.sh - # Do not split, the line below wants env vars from the line above. - ../scripts/vpp/arm-build-no-test.sh - provide-maven-settings: @@ -629,6 +637,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -690,6 +700,9 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' + - gerrit-refspec-parameter scm: - gerrit-trigger-scm: @@ -746,6 +759,8 @@ repo-name: '{repo-stream-part}.ubuntu.bionic.main' - stream-parameter: stream: '{stream}' + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -907,6 +922,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -986,6 +1003,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -1076,6 +1095,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: @@ -1162,6 +1183,8 @@ name: ARCHIVE_ARTIFACTS default: '{archive-artifacts}' description: Artifacts to archive to the logs server. + - make-parallel-jobs-parameter: + make-parallel-jobs: '{make-parallel-jobs}' scm: - gerrit-trigger-scm: |