diff options
| author |   Vanessa Rene Valderrama <vvalderrama@linuxfoundation.org> | 2016-12-20 05:36:48 -0600 |
|---|---|---|
| committer | Vanessa Rene Valderrama <vvalderrama@linuxfoundation.org> | 2016-12-21 09:37:15 -0600 |
| commit | 2f98b4c1f809ff9d256a09cc0cdf4669633c1dee (patch) | |
| tree | 1288b9e28503192049a6e5071062d8f9211700b2 /jjb | |
| parent | 6079577f7cd9a64d761a80581c8abe40e79fcd6a (diff) | |
Global log shipping macro
Ship all console and environment logs as well as anything in
$WORKSPACE/archives to https://logs.fd.io.
All text files will be pre-compressed for quicker serving from the log
hosting which is configured to properly serve the files so they will
expand properly in the browser.
Change-Id: Ica8be1ca9af82dfc64d063a5d3fa30be3585253c
Signed-off-by: Vanessa Rene Valderrama <vvalderrama@linuxfoundation.org>
Diffstat (limited to 'jjb')
| -rw-r--r-- | jjb/global-macros.yaml | 31 | ||||
| -rw-r--r-- | jjb/include-raw-deploy-archives.sh | 106 |
2 files changed, 137 insertions, 0 deletions
diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index 8601fa8d8..f3171a9a9 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -282,6 +282,37 @@ output-path: '{output-path}' other-files: '' +- publisher: + name: fdio-infra-shiplogs + # To archive things, the job will need to create an "archives" directory in + # the workspace and this macro will handle copying the contents of the + # archives directory. + # + # Uses the build parameter ARCHIVE_ARTIFACTS if not empty to find files to + # archive. You can pass globstar patterns for example "**/*.xml **/*.log" as + # the archive pattern. This is a space separated list of files to archive. + # + # Also ensure that the workspace is cleaned up at the end of the build. + publishers: + - postbuildscript: + builders: + - shell: !include-raw: include-raw-deploy-archives.sh + - maven-target: + maven-version: '{maven-version}' + pom: '.archives/deploy-archives.xml' + goals: 'clean deploy -V -B -q -Dmaven.repo.local=/tmp/r' + settings: 'jenkins-log-archives-settings' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + - description-setter: + regexp: '^Build logs: .*' + script-only-if-succeeded: false + script-only-if-failed: false + mark-unstable-if-failed: true + - workspace-cleanup: + fail-build: false + ###### BUILDERS - builder: name: ci-management-check-unicode diff --git a/jjb/include-raw-deploy-archives.sh b/jjb/include-raw-deploy-archives.sh new file mode 100644 index 000000000..0072d4687 --- /dev/null +++ b/jjb/include-raw-deploy-archives.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +set +e # Do not affect the build result if some part of archiving fails. + +ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER" +[ "$LOGS_SERVER" ] || LOGS_SERVER="https://logs.fd.io" +[ "$LOGS_REPO_URL" ] || LOGS_REPO_URL="https://nexus.fd.io/content/sites/logs/" + +echo "Build logs: <a href=\"$LOGS_SERVER/$SILO/$ARCHIVES_DIR\">$LOGS_SERVER/$SILO/$ARCHIVES_DIR</a>" + +mkdir .archives +cd .archives/ + +cat > deploy-archives.xml <<EOF +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>logs</groupId> + <artifactId>logs</artifactId> + <version>1.0.0</version> + <packaging>pom</packaging> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8.2</version> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + <plugin> + <groupId>org.sonatype.plugins</groupId> + <artifactId>maven-upload-plugin</artifactId> + <version>0.0.1</version> + <executions> + <execution> + <id>publish-site</id> + <phase>deploy</phase> + <goals> + <goal>upload-file</goal> + </goals> + <configuration> + <serverId>fdioproject-log-archives</serverId> + <repositoryUrl>$LOGS_REPO_URL/content-compressed</repositoryUrl> + <file>archives.zip</file> + <repositoryPath>$SILO</repositoryPath> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> +EOF + +mkdir -p $ARCHIVES_DIR +mkdir -p $WORKSPACE/archives +if [ ! -z "${{ARCHIVE_ARTIFACTS}}" ]; then + pushd $WORKSPACE + shopt -s globstar # Enable globstar to copy archives + archive_artifacts=$(echo ${{ARCHIVE_ARTIFACTS}}) + for f in $archive_artifacts; do + echo "Archiving $f" + mkdir -p $WORKSPACE/archives/$(dirname $f) + mv $f $WORKSPACE/archives/$f + done + shopt -u globstar # Disable globstar once archives are copied + popd +fi + + +# Ignore logging if archives doesn't exist +mv $WORKSPACE/archives/ $ARCHIVES_DIR > /dev/null 2>&1 +touch $ARCHIVES_DIR/_build-details.txt +echo "build-url: ${{BUILD_URL}}" >> $ARCHIVES_DIR/_build-details.txt +env > $ARCHIVES_DIR/_build-enviroment-variables.txt + +# capture system info +touch $ARCHIVES_DIR/_sys-info.txt +{{ + echo -e "uname -a:\n `uname -a` \n" + echo -e "df -h:\n `df -h` \n" + echo -e "free -m:\n `free -m` \n" + echo -e "nproc:\n `nproc` \n" + echo -e "lscpu:\n `lscpu` \n" + echo -e "ip addr:\n `/sbin/ip addr` \n" +}} 2>&1 | tee -a $ARCHIVES_DIR/_sys-info.txt + +# Magic string used to trim console logs at the appropriate level during wget +echo "-----END_OF_BUILD-----" +wget -O $ARCHIVES_DIR/console.log ${{BUILD_URL}}consoleText +wget -O $ARCHIVES_DIR/console-timestamp.log ${{BUILD_URL}}/timestamps?time=HH:mm:ss\&appendLog +sed -i '/^-----END_OF_BUILD-----$/,$d' $ARCHIVES_DIR/console.log +sed -i '/^.*-----END_OF_BUILD-----$/,$d' $ARCHIVES_DIR/console-timestamp.log + +gzip $ARCHIVES_DIR/*.txt $ARCHIVES_DIR/*.log +# find and gzip any 'text' files +find $ARCHIVES_DIR -type f -print0 \ + | xargs -0r file \ + | egrep -e ':.*text.*' \ + | cut -d: -f1 \ + | xargs -d'\n' -r gzip + +zip -r archives.zip $JENKINS_HOSTNAME/ +du -sh archives.zip |