diff options
| -rw-r--r-- | jjb/csit/csit-vpp-pylint.yaml | 2 | ||||
| -rw-r--r-- | jjb/csit/csit.yaml | 148 | ||||
| -rw-r--r-- | jjb/csit/include-raw-csit-docs.sh | 59 | ||||
| -rw-r--r-- | jjb/deb_dpdk/deb_dpdk.yaml | 15 | ||||
| -rw-r--r-- | jjb/global-macros.yaml | 4 | ||||
| -rw-r--r-- | jjb/hc2vpp/csit-hc2vpp.yaml | 12 | ||||
| -rw-r--r-- | jjb/honeycomb/honeycomb.yaml | 24 | ||||
| -rw-r--r-- | jjb/rpm_dpdk/rpm_dpdk.yaml | 6 | ||||
| -rw-r--r-- | jjb/vpp/vpp.yaml | 28 | ||||
| -rw-r--r-- | packer/README.markdown | 37 | ||||
| -rw-r--r-- | packer/provision/basebuild.sh | 190 | ||||
| -rw-r--r-- | packer/provision/baseline.sh | 168 | ||||
| -rw-r--r-- | packer/provision/null_data.sh | 4 | ||||
| -rw-r--r-- | packer/provision/rh-user_data.sh | 4 | ||||
| -rw-r--r-- | packer/provision/system_reseal.sh | 38 | ||||
| -rw-r--r-- | packer/templates/basebuild.json | 52 | ||||
| -rw-r--r-- | packer/vars/.gitignore | 2 | ||||
| -rw-r--r-- | packer/vars/centos.json | 7 | ||||
| -rw-r--r-- | packer/vars/cloud-env.json.example | 6 | ||||
| -rw-r--r-- | packer/vars/ubuntu-14.04.json | 7 | ||||
| -rw-r--r-- | packer/vars/ubuntu-16.04.json | 7 |
21 files changed, 782 insertions, 38 deletions
diff --git a/jjb/csit/csit-vpp-pylint.yaml b/jjb/csit/csit-vpp-pylint.yaml index 0d3068945..0915643e1 100644 --- a/jjb/csit/csit-vpp-pylint.yaml +++ b/jjb/csit/csit-vpp-pylint.yaml @@ -18,7 +18,7 @@ name: 'csit-validate-pylint-{stream}' project-type: freestyle - node: verify + node: ubuntu1604-basebuild-4c-4g concurrent: true logrotate: diff --git a/jjb/csit/csit.yaml b/jjb/csit/csit.yaml index cb2a9e181..18f806cfb 100644 --- a/jjb/csit/csit.yaml +++ b/jjb/csit/csit.yaml @@ -8,6 +8,8 @@ - 'csit-vpp-verify-{stream_timed}-weekly' - 'csit-vpp-verify-{stream_timed}-semiweekly' - 'csit-vpp-verify-perf-master-nightly-all' + - 'csit-docs-merge-{stream}' + - 'csit-docs-verify-{stream}' project: 'csit' branch: 'master' @@ -31,11 +33,12 @@ branch: 'rls1609' branch-id: 'oper-rls1609' + - job-template: name: 'csit-vpp-functional-{stream}-virl' project-type: freestyle - node: verify-ubuntu1604 + node: ubuntu1604-basebuild-4c-4g concurrent: true logrotate: @@ -77,11 +80,12 @@ - robot-report: output-path: '' + - job-template: name: 'csit-vpp-perf-{stream}-{type}' project-type: freestyle - node: verify-ubuntu1604 + node: ubuntu1604-basebuild-4c-4g concurrent: false logrotate: @@ -260,12 +264,13 @@ xpath-type: 'nodeset' xpath: '//*[@framesize="78" and contains(@tags,"PDR") and @workerthreads="4"]' + - job-template: name: 'csit-vpp-verify-{stream_timed}-weekly' description: 'Purpose of this job is to validate CSIT code of corresponding branch against verified vpp build from corresponding branch and create corresponding operational branch in case of success.' project-type: freestyle - node: verify-ubuntu1604 + node: ubuntu1604-basebuild-4c-4g concurrent: false logrotate: @@ -324,7 +329,7 @@ description: 'Purpose of this job is to validate latest vpp build from corresponding branch stored in nexus against verified CSIT codebase of corresponding branch.' project-type: freestyle - node: verify-ubuntu1604 + node: ubuntu1604-basebuild-4c-4g concurrent: false logrotate: @@ -366,12 +371,13 @@ - robot-report: output-path: 'csit/archive' + - job-template: name: 'csit-vpp-verify-perf-master-nightly-all' description: 'Purpose of this job is to validate latest vpp build from corresponding branch stored in nexus against verified CSIT codebase of corresponding branch.' project-type: freestyle - node: verify-ubuntu1604 + node: ubuntu1604-basebuild-4c-4g concurrent: false logrotate: @@ -542,6 +548,138 @@ xpath-type: 'nodeset' xpath: '//*[@framesize="78" and contains(@tags,"PDR") and @workerthreads="4"]' + +- job-template: + name: 'csit-docs-verify-{stream}' + + project-type: freestyle + node: 'ubuntu1604-basebuild-4c-4g' + concurrent: true + + logrotate: + daysToKeep: '{build-days-to-keep}' + numToKeep: '{build-num-to-keep}' + artifactDaysToKeep: '{build-artifact-days-to-keep}' + artifactNumToKeep: '{build-artifact-num-to-keep}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - maven-exec: + maven-version: 'mvn33-new' + + scm: + - gerrit-trigger-scm: + credentials-id: 'jenkins-gerrit-credentials' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - fdio-infra-wrappers: + build-timeout: '{build-timeout}' + - fdio-infra-wrappers-non-activity-timeout: + build-timeout: 360 + + triggers: + - gerrit-trigger-patch-submitted: + name: '{project}' + branch: '{branch}' + + builders: + - maven-target: + maven-version: 'mvn33-new' + goals: '--version' + settings: 'csit-settings' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + - provide-maven-settings: + settings-file: 'csit-settings' + global-settings-file: 'global-settings' + - shell: + !include-raw-escape: include-raw-csit-docs.sh + + publishers: + - archive: + artifacts: 'resources/tools/doc_gen/csit.docs.tar.gz' + allow-empty: 'true' + fingerprint: false + only-if-success: true + default-excludes: false + + +- job-template: + name: 'csit-docs-merge-{stream}' + + project-type: freestyle + node: 'ubuntu1604-basebuild-4c-4g' + concurrent: true + + logrotate: + daysToKeep: '{build-days-to-keep}' + numToKeep: '{build-num-to-keep}' + artifactDaysToKeep: '{build-artifact-days-to-keep}' + artifactNumToKeep: '{build-artifact-num-to-keep}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - maven-exec: + maven-version: 'mvn33-new' + + scm: + - gerrit-trigger-scm: + credentials-id: 'jenkins-gerrit-credentials' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - fdio-infra-wrappers: + build-timeout: '{build-timeout}' + - fdio-infra-wrappers-non-activity-timeout: + build-timeout: 360 + + triggers: + - reverse: + jobs: 'csit-vpp-verify-master-weekly' + result: 'success' + - gerrit: + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'run-docs' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + + builders: + - maven-target: + maven-version: 'mvn33-new' + goals: '--version' + settings: 'csit-settings' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + - provide-maven-settings: + settings-file: 'csit-settings' + global-settings-file: 'global-settings' + - shell: + !include-raw-escape: include-raw-csit-docs.sh + + publishers: + - archive: + artifacts: 'resources/tools/doc_gen/csit.docs.tar.gz' + allow-empty: 'true' + fingerprint: false + only-if-success: true + default-excludes: false + + - parameter: name: test-type-parameter parameters: diff --git a/jjb/csit/include-raw-csit-docs.sh b/jjb/csit/include-raw-csit-docs.sh new file mode 100644 index 000000000..789af1598 --- /dev/null +++ b/jjb/csit/include-raw-csit-docs.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -xe -o pipefail + +[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site" +[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit +[ "$DOC_FILE" ] || DOC_FILE=csit.docs.tar.gz +[ "$DOC_DIR" ] || DOC_DIR=resources/tools/doc_gen +[ "$SITE_DIR" ] || SITE_DIR=${DOC_DIR}/_build +[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn" + +cd ${DOC_DIR} + +chmod +x ./run_doc.sh +./run_doc.sh + +retval=$? +if [ ${retval} -ne "0" ]; then + echo "Documentation generation failed!" +exit ${retval} +fi + +if [[ ${JOB_NAME} == *merge* ]]; then + + cd ${SITE_DIR} + + cat > pom.xml << EOF + <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>io.fd.csit</groupId> + <artifactId>docs</artifactId> + <version>1.0.0</version> + <packaging>pom</packaging> + <properties> + <generateReports>false</generateReports> + </properties> + <build> + <extensions> + <extension> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-webdav-jackrabbit</artifactId> + <version>2.9</version> + </extension> + </extensions> + </build> + <distributionManagement> + <site> + <id>fdio-site</id> + <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${STREAM}</url> + </site> + </distributionManagement> + </project> +EOF + + ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C + + cd - + +fi diff --git a/jjb/deb_dpdk/deb_dpdk.yaml b/jjb/deb_dpdk/deb_dpdk.yaml index 81c1d9917..08040e2c1 100644 --- a/jjb/deb_dpdk/deb_dpdk.yaml +++ b/jjb/deb_dpdk/deb_dpdk.yaml @@ -23,7 +23,7 @@ name: 'deb_dpdk-verify-{stream}-{os}' project-type: freestyle - node: 'verify-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -66,7 +66,7 @@ name: 'deb_dpdk-verify-image-{stream}-{os}' project-type: freestyle - node: 'verify-image-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -122,7 +122,7 @@ name: 'deb_dpdk-merge-{stream}-{os}' project-type: freestyle - node: 'merge-{os}' + node: '{os}-basebuild-4c-4g' logrotate: daysToKeep: '{build-days-to-keep}' @@ -137,6 +137,8 @@ branch: '{branch}' - os-parameter: os: '{os}' + - maven-exec: + maven-version: 'mvn33-new' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' @@ -156,6 +158,13 @@ branch: '{branch}' builders: + - maven-target: + maven-version: 'mvn33-new' + goals: '--version' + settings: 'deb_dpdk-settings' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp - config-file-provider: files: - file-id: 'deb_dpdk-settings' diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml index ba0d1a596..8601fa8d8 100644 --- a/jjb/global-macros.yaml +++ b/jjb/global-macros.yaml @@ -141,6 +141,8 @@ - 'jenkins-gerrit-credentials' - jclouds: single-use: True + - openstack: + single-use: True - wrapper: # This wrapper is required for all jobs as it configures the wrappers @@ -158,6 +160,8 @@ - 'jenkins-gerrit-credentials' - jclouds: single-use: True + - openstack: + single-use: True ###### TRIGGERS - trigger: diff --git a/jjb/hc2vpp/csit-hc2vpp.yaml b/jjb/hc2vpp/csit-hc2vpp.yaml index f76d9b13e..94e469948 100644 --- a/jjb/hc2vpp/csit-hc2vpp.yaml +++ b/jjb/hc2vpp/csit-hc2vpp.yaml @@ -21,7 +21,7 @@ name: 'hc2vpp-csit-integration-{stream}-{os}' project-type: freestyle - node: verify-{os} + node: 'ubuntu1604-basebuild-4c-4g' concurrent: true logrotate: @@ -76,7 +76,7 @@ name: 'hc2vpp-csit-verify-{stream}-{os}' project-type: maven - node: verify-{os} + node: 'ubuntu1604-basebuild-4c-4g' concurrent: true logrotate: @@ -93,7 +93,9 @@ - os-parameter: os: '{os}' - maven-project-parameter: - maven: 'mvn33' + maven: 'mvn33-new' + - maven-exec: + maven-version: 'mvn33-new' scm: - gerrit-trigger-scm: @@ -111,7 +113,7 @@ server-name: 'Primary' trigger-on: - comment-added-contains-event: - comment-contains-value: 'verify-csit$' + comment-contains-value: 'verify-csit' projects: - project-compare-type: 'ANT' project-pattern: '{project}' @@ -131,7 +133,7 @@ !include-raw-escape: include-raw-hc2vpp-csit-verify-prebuild.sh maven: - maven-name: 'mvn33' + maven-name: 'mvn33-new' root-pom: 'pom.xml' goals: 'clean install -V -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r -Dmaven.site.skip={skip-site} site -Dmaven.site.deploy.skip={skip-site} -DskipTests' maven-opts: '-Xmx2048m -Dmaven.compile.fork=true' diff --git a/jjb/honeycomb/honeycomb.yaml b/jjb/honeycomb/honeycomb.yaml index 6135f7592..c3ff0c5ea 100644 --- a/jjb/honeycomb/honeycomb.yaml +++ b/jjb/honeycomb/honeycomb.yaml @@ -20,7 +20,7 @@ - job-template: name: 'honeycomb-verify-{stream}-{os}' project-type: maven - node: verify-{os} + node: '{os}-basebuild-4c-4g' concurrent: true jdk: openjdk8-{os} @@ -57,7 +57,7 @@ - shell: | if [ "${{OS}}" == "ubuntu1404" ];then sudo update-java-alternatives -s /usr/lib/jvm/java-1.8.0-openjdk-amd64||echo 'ignoring failure';fi maven: - maven-name: 'mvn33' + maven-name: 'mvn33-new' root-pom: 'pom.xml' goals: 'clean install -V -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r -Dmerge site' maven-opts: '-Xmx2048m -Dmaven.compile.fork=true' @@ -73,7 +73,7 @@ - job-template: name: 'honeycomb-merge-{stream}-{os}' project-type: maven - node: merge-{os} + node: '{os}-basebuild-4c-4g' jdk: openjdk8-{os} logrotate: @@ -88,7 +88,9 @@ - os-parameter: os: '{os}' - maven-project-parameter: - maven: 'mvn33' + maven: 'mvn33-new' + - maven-exec: + maven-version: 'mvn33-new' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' @@ -111,7 +113,7 @@ branch: '{branch}' maven: - maven-name: 'mvn33' + maven-name: 'mvn33-new' root-pom: 'pom.xml' goals: 'clean install -V -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r -Dmerge -Dsonar -Dmaven.site.skip={skip-site} site -Dmaven.site.deploy.skip={skip-site} site:deploy' maven-opts: '-Xmx2048m -Dmaven.compile.fork=true' @@ -122,7 +124,7 @@ postbuilders: - maven-target: - maven-version: 'mvn33' + maven-version: 'mvn33-new' goals: '--version' settings: 'honeycomb-settings' settings-type: cfp @@ -150,7 +152,7 @@ - job-template: name: 'honeycomb-benchmark-master' project-type: maven - node: verify-ubuntu1604 + node: ubuntu1604-basebuild-4c-4g jdk: openjdk8-ubuntu1604 logrotate: @@ -165,7 +167,9 @@ - os-parameter: os: 'ubuntu1604' - maven-project-parameter: - maven: 'mvn33' + maven: 'mvn33-new' + - maven-exec: + maven-version: 'mvn33-new' - repo-name-parameter: repo-name: 'master.ubuntu.xenial.main' @@ -185,7 +189,7 @@ - timed: '@daily' maven: - maven-name: 'mvn33' + maven-name: 'mvn33-new' root-pom: 'pom.xml' goals: 'clean install -V -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r -Dmerge -Pbenchmark' maven-opts: '-Xmx2048m -Dmaven.compile.fork=true' @@ -209,4 +213,4 @@ display-table: true format: 'csv' inclusion-flag: 'include-by-string' - exclude: 'Score' + exclude: 'Score, Score Error (99,9%)' diff --git a/jjb/rpm_dpdk/rpm_dpdk.yaml b/jjb/rpm_dpdk/rpm_dpdk.yaml index 3adc37819..63c1a74c7 100644 --- a/jjb/rpm_dpdk/rpm_dpdk.yaml +++ b/jjb/rpm_dpdk/rpm_dpdk.yaml @@ -18,7 +18,7 @@ name: 'rpm_dpdk-verify-{stream}-{os}' project-type: freestyle - node: 'verify-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -61,7 +61,7 @@ name: 'rpm_dpdk-verify-image-{stream}-{os}' project-type: freestyle - node: 'verify-image-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -117,7 +117,7 @@ name: 'rpm_dpdk-merge-{stream}-{os}' project-type: freestyle - node: 'merge-{os}' + node: '{os}-basebuild-4c-4g' logrotate: daysToKeep: '{build-days-to-keep}' diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml index a47f8b1f5..730ab847d 100644 --- a/jjb/vpp/vpp.yaml +++ b/jjb/vpp/vpp.yaml @@ -68,7 +68,7 @@ name: 'vpp-verify-{stream}-{os}' project-type: freestyle - node: 'verify-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -117,7 +117,7 @@ name: 'vpp-verify-image-{stream}-{os}' project-type: freestyle - node: 'verify-image-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -172,7 +172,7 @@ name: 'vpp-merge-{stream}-{os}' project-type: freestyle - node: 'merge-{os}' + node: '{os}-basebuild-4c-4g' logrotate: daysToKeep: '{build-days-to-keep}' @@ -188,7 +188,9 @@ - os-parameter: os: '{os}' - maven-project-parameter: - maven: 'mvn33' + maven: 'mvn33-new' + - maven-exec: + maven-version: 'mvn33-new' - repo-name-parameter: repo-name: '{repo-stream-part}.{repo-os-part}' @@ -213,7 +215,7 @@ - shell: !include-raw-escape: include-raw-vpp-build.sh - maven-target: - maven-version: 'mvn33' + maven-version: 'mvn33-new' goals: '--version' settings: 'vpp-settings' settings-type: cfp @@ -238,7 +240,7 @@ name: 'vpp-csit-verify-virl-{stream}' project-type: freestyle - node: 'verify-{os}' + node: '{os}-basebuild-4c-4g' concurrent: true logrotate: @@ -292,7 +294,7 @@ name: 'vpp-csit-verify-hw-perf-{stream}-{type}' project-type: freestyle - node: 'verify-{os}' + node: '{os}-basebuild-4c-4g' concurrent: false logrotate: @@ -356,7 +358,7 @@ name: 'vpp-docs-verify-{stream}' project-type: freestyle - node: 'verify-ubuntu1604' + node: ubuntu1604-basebuild-4c-4g concurrent: true logrotate: @@ -372,6 +374,8 @@ branch: '{branch}' - os-parameter: os: 'ubuntu1604' + - maven-exec: + maven-version: 'mvn33-new' scm: - gerrit-trigger-scm: @@ -390,7 +394,7 @@ builders: - maven-target: - maven-version: 'mvn33' + maven-version: 'mvn33-new' goals: '--version' settings: 'vpp-settings' settings-type: cfp @@ -413,7 +417,7 @@ name: 'vpp-docs-merge-{stream}' project-type: freestyle - node: 'verify-ubuntu1604' + node: ubuntu1604-basebuild-4c-4g concurrent: true logrotate: @@ -429,6 +433,8 @@ branch: '{branch}' - os-parameter: os: 'ubuntu1604' + - maven-exec: + maven-version: 'mvn33-new' scm: - gerrit-trigger-scm: @@ -463,7 +469,7 @@ builders: - maven-target: - maven-version: 'mvn33' + maven-version: 'mvn33-new' goals: '--version' settings: 'vpp-settings' settings-type: cfp diff --git a/packer/README.markdown b/packer/README.markdown new file mode 100644 index 000000000..6c50f4ad7 --- /dev/null +++ b/packer/README.markdown @@ -0,0 +1,37 @@ +# FD.io Project CI Packer + +[Packer][1] is a tool for automatically creating VM and container images, +configuring them and post-processing them into standard output formats. + +We build FD.io's CI images via Packer. + +## Building + +You'll need to [install Packer][2], of course. + +FD.io's Packer configuration is divided into build-specific variables, +output-specific templates and a set of shared provisioning scripts. To do a +specific build, combine the template for the desired output artifact type with +a variable file. To build a new basebuild instance the following would be done: + +``` +packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/basebuild.json +``` + +**NOTE:** vars/cloud-env.json is a gitignored file as it contains private +information. There is a vars/cloud-env.json.example file that may be used as a +base for creating the one needed. + +This would build a bootable image in the FD.io CI cloud environment. + +From a high level, the builds: + +* Boot a specified base image in the cloud +* Run a set of shell scripts, listed in the template's shell provisioner + section, to do any configuration required by the builder. +* Execute a shutdown of the running instance +* Execute a 'nova image-create' operation against the shutdown instance. +* Perform a 'nova delete' operation against the shutdown instance. + +[1]: https://www.packer.io/ +[2]: https://www.packer.io/intro/getting-started/setup.html diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh new file mode 100644 index 000000000..f43f3857d --- /dev/null +++ b/packer/provision/basebuild.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + + # RH Install build tools + echo "---> Installing RH build tools $(date +'%Y%m%dT%H%M%S')" + RH_TOOLS_PKGS="@development redhat-lsb glibc-static java-1.8.0-openjdk-devel yum-utils openssl-devel apr-devel indent" + yum install -y ${RH_TOOLS_PKGS} + + # Memory leakage checks + yum install -y valgrind + + # RH Install Python dependencies + ###REMOVED mysql-devel + echo "---> Installing RH Python dependencies $(date +'%Y%m%dT%H%M%S')" + RH_PYTHON_PKGS="python-devel python-virtualenv python-setuptools python-pip openssl-devel" + yum install -y ${RH_PYTHON_PKGS} + + # RH Install Documentation packages + ###Removed python-pyparsing + echo "---> Installing RH documentation packages $(date +'%Y%m%dT%H%M%S')" + RH_DOC_PKGS="doxygen graphviz python-jinja2 asciidoc dblatex source-highlight python-sphinx" + yum install -y install ${RH_DOC_PKGS} + + # RH Install GCC packages + echo "---> Installing RH GCC packages $(date +'%Y%m%dT%H%M%S')" + RH_GCC_PKGS="cpp gcc c++ cmake" + yum install -y ${RH_GCC_PKGS} + + # RH Install components to build Ganglia modules + echo "---> Installing RH components $(date +'%Y%m%dT%H%M%S')" + RH_GANGLIA_MODS="libconfuse-devel ganglia-devel mock" + yum install -y --enablerepo=epel ${RH_GANGLIA_MODS} + + # RH Install module for VPP project + echo uio_pci_generic >> /etc/modules + + # RH Install VPP packages to shorten build times + echo "---> Installing VPP dependencies $(date +'%Y%m%dT%H%M%S')" + RH_VPP_PKGS="curl autoconf automake bison ccache dkms git libtool libconfuse-dev git-review cscope" + yum install -y ${RH_VPP_PKGS} + + yum groupinstall "Development Tools" + # RH Install TLDK dependencies + RH_TLKD_PKGS="libpcap-devel libcap-devel" + yum install -y ${RH_TLKD_PKGS} + + # RH Install debuginfo packages + #echo "---> Installing debug packages $(date +'%Y%m%dT%H%M%S')" + #RH_DEBUG_PKGS="glibc openssl-libs zlib" + #debuginfo-install -y ${RH_DEBUG_PKGS} + + # # RH Packer builds happen from the centos flavor images + # PACKERDIR=$(mktemp -d) + # # disable double quote checking + # # shellcheck disable=SC2086 + # cd $PACKERDIR + # wget https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip + # unzip packer_0.10.1_linux_amd64.zip -d /usr/local/bin/ + # # rename packer to avoid conflicts with cracklib + # mv /usr/local/bin/packer /usr/local/bin/packer.io + + # # cleanup from the installation + # # disable double quote checking + # # shellcheck disable=SC2086 + # rm -rf $PACKERDIR + # # cleanup from previous install process + # if [ -d /tmp/packer ] + # then + # rm -rf /tmp/packer + # fi +} + +ubuntu_systems() { + + # DEB cloud packages + echo "---> Installing cloud packages $(date +'%Y%m%dT%H%M%S')" + CLOUD_PKGS="cloud-initramfs-dyn-netconf cloud-initramfs-growroot cloud-initramfs-rescuevol" + apt install -y ${CLOUD_PKGS} + + # DEB Install Python dependencies + echo "---> Installing Python dependencies $(date +'%Y%m%dT%H%M%S')" + PYTHON_PKGS="python-dev python-virtualenv python-setuptools python-pip libssl-dev libmysqlclient-dev python2.7" + apt install -y ${PYTHON_PKGS} + + # Memory leakage checks + apt install -y valgrind + + # DEB Install Documentation packages + echo "---> Installing documentation packages $(date +'%Y%m%dT%H%M%S')" + DOC_PKGS="doxygen graphviz python-pyparsing python-jinja2 asciidoc dblatex source-highlight" + apt install -y ${DOC_PKGS} + + # DEB Install the correct version of toolchain packages + echo "---> Installing latest toolchain packages from PPA $(date +'%Y%m%dT%H%M%S')" + + # DEB Install PPA packages + echo "---> Adding '$1' PPA $(date +'%Y%m%dT%H%M%S')" + dpkg -l software-properties-common > /dev/null 2>&1 || software-properties-common + + listfile=$(perl -e "print(q{$1} =~ m{^ppa:(.+)/ppa})")-ppa-${CODENAME}.list + if [ ! -f /etc/apt/sources.list.d/${listfile} ] + then + do_retry sudo apt-add-repository -y $1 + fi + + #Retry to prevent timeout failure + echo "---> Updating package index $(date +'%Y%m%dT%H%M%S')" + do_retry sudo apt-get update + echo "<--- Updating package index $(date +'%Y%m%dT%H%M%S')" + echo "<--- Adding '$1' PPA $(date +'%Y%m%dT%H%M%S')" + + # DEB Install GCC packages + echo "---> Installing GCC-5 packages $(date +'%Y%m%dT%H%M%S')" + GCC_PKGS="cpp gcc g++ cmake" + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update + apt install -y ${GCC_PKGS} + + # DEB Install VPP packages to shorten build times + echo "---> Installing VPP DEB_DEPENDS packages $(date +'%Y%m%dT%H%M%S')" + VPP_PKGS="curl build-essential autoconf automake bison libssl-dev ccache debhelper dkms git libtool libganglia1-dev libapr1-dev dh-systemd libconfuse-dev git-review exuberant-ctags cscope indent" + apt install -y ${VPP_PKGS} + + # DEB Install latest kernel and uio + echo "---> Installing kernel image and header packages $(date +'%Y%m%dT%H%M%S')" + DEB_PKGS="linux-image-extra-virtual linux-headers-virtual linux-headers-`uname -r`" + apt install -y ${DEB_PKGS} + + #Configuring thirdparty Nexus repo + echo "deb [trusted=yes] https://nexus.fd.io/content/repositories/thirdparty ./" > /etc/apt/sources.list.d/FD.io.thirdparty.list + apt-get update + + # DEB Install deb_dpdk packages to shorten build times + ###REMOVED sphinx-rtd-theme + echo "---> Installing deb_dpdk packages $(date +'%Y%m%dT%H%M%S')" + DEB_DPDK_PKGS="google-mock lsb-release dpkg-dev debian-xcontrol devscripts pristine-tar dh-python python-sphinx libpcap0.8-dev libstdc++5 python-scapy inkscape libxen-dev libxenstore3.0 python-sphinx-rtd-theme" + apt install -y ${DEB_DPDK_PKGS} + + sudo apt install -y libcap-dev libpcap-dev + + TEXLIVE_PKGS="texlive-fonts-recommended tex-common texlive-base texlive-binaries texlive-pictures texlive-latex-recommended preview-latex-style texlive-latex-extra" + apt install -y ${TEXLIVE_PKGS} + echo "<--- Installing deb_dpdk packages $(date +'%Y%m%dT%H%M%S')" + + # DEB Manipulation tools, edits debugger, and LSB + echo "---> Installing tools packages $(date +'%Y%m%dT%H%M%S')" + TOOL_PKGS="iproute2 ethtool vlan bridge-utils vim gdb lsb-release" + apt install -y ${TOOL_PKGS} + + # DEB Clean up packages for a smaller image + apt-get update + + # DEB Updating CA certificates + echo "---> Forcing CA certificate update $(date +'%Y%m%dT%H%M%S')" + sudo update-ca-certificates -f > /dev/null 2>&1 + echo "<--- Forcing CA certificate update $(date +'%Y%m%dT%H%M%S')" +} + +all_systems() { + + echo 'Configure keep alive to prevent timeout during testing' + local SSH_CFG=/etc/ssh/ssh_config + echo "TCPKeepAlive true" | sudo tee -a ${SSH_CFG} >/dev/null 2>&1 + echo "ServerAliveCountMax 30" | sudo tee -a ${SSH_CFG} >/dev/null 2>&1 + echo "ServerAliveInterval 10" | sudo tee -a ${SSH_CFG} >/dev/null 2>&1 + +} + +echo "---> Detecting OS" +ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/baseline.sh b/packer/provision/baseline.sh new file mode 100644 index 000000000..36783554d --- /dev/null +++ b/packer/provision/baseline.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + # Handle the occurance where SELINUX is actually disabled + SELINUX=$(grep -E '^SELINUX=(disabled|permissive|enforcing)$' /etc/selinux/config) + MODE=$(echo "$SELINUX" | cut -f 2 -d '=') + case "$MODE" in + permissive) + echo "************************************" + echo "** SYSTEM ENTERING ENFORCING MODE **" + echo "************************************" + # make sure that the filesystem is properly labelled. + # it could be not fully labeled correctly if it was just switched + # from disabled, the autorelabel misses some things + # skip relabelling on /dev as it will generally throw errors + restorecon -R -e /dev / + + # enable enforcing mode from the very start + setenforce enforcing + + # configure system for enforcing mode on next boot + sed -i 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config + ;; + disabled) + sed -i 's/SELINUX=disabled/SELINUX=permissive/' /etc/selinux/config + touch /.autorelabel + + echo "*******************************************" + echo "** SYSTEM REQUIRES A RESTART FOR SELINUX **" + echo "*******************************************" + ;; + enforcing) + echo "*********************************" + echo "** SYSTEM IS IN ENFORCING MODE **" + echo "*********************************" + ;; + esac + + echo "---> Updating operating system" + yum clean all -q + yum install -y -q deltarpm + yum update -y -q + + # add in components we need or want on systems + echo "---> Installing base packages" + yum install -y -q @base https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + # separate group installs from package installs since a non-existing + # group with dnf based systems (F21+) will fail the install if such + # a group does not exist + yum install -y -q unzip xz puppet git perl-XML-XPath wget make + + # All of our systems require Java (because of Jenkins) + # Install all versions of the OpenJDK devel but force 1.7.0 to be the + # default + + echo "---> Configuring OpenJDK" + yum install -y -q 'java-*-openjdk-devel' + + FACTER_OS=$(/usr/bin/facter operatingsystem) + FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease) + case "$FACTER_OS" in + Fedora) + if [ "$FACTER_OSVER" -ge "21" ] + then + echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist" + else + alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java + alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 + fi + ;; + *) + alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java + alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 + ;; + esac +} + +ubuntu_systems() { + # Ignore SELinux since slamming that onto Ubuntu leads to + # frustration + + export DEBIAN_FRONTEND=noninteractive + cat <<EOF >> /etc/apt/apt.conf +APT { + Get { + Assume-Yes "true"; + allow-change-held-packages "true"; + allow-downgrades "true"; + allow-remove-essential "true"; + }; +}; + +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +}; + +EOF + + echo "---> Updating operating system" + apt-get update -qq > /dev/null + apt-get upgrade -qq > /dev/null + + # add in stuff we know we need + echo "---> Installing base packages" + apt-get install -qq unzip xz-utils puppet git libxml-xpath-perl make wget > /dev/null + + # install Java 7 + echo "---> Configuring OpenJDK" + apt-get install -qq openjdk-7-jdk > /dev/null + + # make jdk8 available + add-apt-repository -y ppa:openjdk-r/ppa > /dev/null + apt-get update -qq > /dev/null + # We need to force openjdk-8-jdk to install + apt-get install -qq openjdk-8-jdk > /dev/null + + # make sure that we still default to openjdk 7 + update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java + update-alternatives --set javac /usr/lib/jvm/java-7-openjdk-amd64/bin/javac +} + +all_systems() { + # Allow jenkins access to update-alternatives command to switch java version + cat <<EOF >/etc/sudoers.d/89-jenkins-user-defaults +Defaults:jenkins !requiretty +jenkins ALL = NOPASSWD: /usr/bin/update-alternatives +EOF + + # Do any Distro specific installations here + echo "Checking distribution" + FACTER_OS=$(/usr/bin/facter operatingsystem) + case "$FACTER_OS" in + *) + echo "---> $FACTER_OS found" + echo "No extra steps for $FACTER_OS" + ;; + esac +} + +echo "---> Attempting to detect OS" +# upstream cloud images use the distro name as the initial user +ORIGIN=$(if [ -e /etc/redhat-release ] + then + echo redhat + else + echo ubuntu + fi) +#ORIGIN=$(logname) + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/null_data.sh b/packer/provision/null_data.sh new file mode 100644 index 000000000..3fa6a3c9f --- /dev/null +++ b/packer/provision/null_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# vi: ts=4 sw=4 sts=4 et : + +# Nothing to do for Ubuntu specific provisioning diff --git a/packer/provision/rh-user_data.sh b/packer/provision/rh-user_data.sh new file mode 100644 index 000000000..6bddb2449 --- /dev/null +++ b/packer/provision/rh-user_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# vi: ts=4 sw=4 sts=4 et : + +/bin/sed -i 's/ requiretty/ !requiretty/' /etc/sudoers; diff --git a/packer/provision/system_reseal.sh b/packer/provision/system_reseal.sh new file mode 100644 index 000000000..f8bc7dc5f --- /dev/null +++ b/packer/provision/system_reseal.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# vim: sw=2 ts=2 sts=2 et : + +rm -rf /etc/Pegasus/*.cnf /etc/Pegasus/*.crt /etc/Pegasus/*.csr \ + /etc/Pegasus/*.pem /etc/Pegasus/*.srl /root/anaconda-ks.cfg \ + /root/anaconda-post.log /root/initial-setup-ks.cfg /root/install.log \ + /root/install.log.syslog /var/cache/fontconfig/* /var/cache/gdm/* \ + /var/cache/man/* /var/lib/AccountService/users/* /var/lib/fprint/* \ + /var/lib/logrotate.status /var/log/*.log* /var/log/BackupPC/LOG \ + /var/log/ConsoleKit/* /var/log/anaconda.syslog /var/log/anaconda/* \ + /var/log/apache2/*_log /var/log/apache2/*_log-* /var/log/apt/* \ + /var/log/aptitude* /var/log/audit/* /var/log/btmp* /var/log/ceph/*.log \ + /var/log/chrony/*.log /var/log/cron* /var/log/cups/*_log /var/log/debug* \ + /var/log/dmesg* /var/log/exim4/* /var/log/faillog* /var/log/gdm/* \ + /var/log/glusterfs/*glusterd.vol.log /var/log/glusterfs/glusterfs.log \ + /var/log/httpd/*log /var/log/installer/* /var/log/jetty/jetty-console.log \ + /var/log/journal/* /var/log/lastlog* /var/log/libvirt/libvirtd.log \ + /var/log/libvirt/lxc/*.log /var/log/libvirt/qemu/*.log \ + /var/log/libvirt/uml/*.log /var/log/lightdm/* /var/log/mail/* \ + /var/log/maillog* /var/log/messages* /var/log/ntp /var/log/ntpstats/* \ + /var/log/ppp/connect-errors /var/log/rhsm/* /var/log/sa/* /var/log/secure* \ + /var/log/setroubleshoot/*.log /var/log/spooler* /var/log/squid/*.log \ + /var/log/syslog* /var/log/tallylog* /var/log/tuned/tuned.log /var/log/wtmp* \ + /var/named/data/named.run + +rm -rf ~/.viminfo /etc/ssh/ssh*key* /root/.ssh/* + +# kill any cloud-init related bits +rm -rf /var/lib/cloud/* + +# clean-up any manual packer uploads +rm -rf /tmp/packer + +# Force a system sync and sleep to get around any SSD issues +echo "Forcing sync and sleep for 10sec" +sync +sleep 10 diff --git a/packer/templates/basebuild.json b/packer/templates/basebuild.json new file mode 100644 index 000000000..426f290af --- /dev/null +++ b/packer/templates/basebuild.json @@ -0,0 +1,52 @@ +{ + "variables": { + "stack_tenant": null, + "stack_user": null, + "stack_pass": null, + "stack_network": null, + + "base_image": null, + "cloud_user": null, + "distro": null, + "cloud_user_data": null + }, + "builders": [ + { + "type": "openstack", + "identity_endpoint": "https://auth.vexxhost.net/v2.0/", + "tenant_name": "{{user `stack_tenant`}}", + "username": "{{user `stack_user`}}", + "password": "{{user `stack_pass`}}", + "region": "ca-ymq-1", + "ssh_username": "{{user `cloud_user`}}", + "image_name": "{{user `distro`}} - basebuild - {{isotime \"20060102-1504\"}}", + "source_image_name": "{{user `base_image`}}", + "flavor": "v1-standard-1", + "availability_zone": "ca-ymq-2", + "networks": [ + "{{user `stack_network`}}" + ], + "user_data_file": "{{user `cloud_user_data`}}" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": ["mkdir -p /tmp/packer"] + }, + { + "type": "file", + "source": "provision/basebuild/", + "destination": "/tmp/packer" + }, + { + "type": "shell", + "scripts": [ + "provision/baseline.sh", + "provision/basebuild.sh", + "provision/system_reseal.sh" + ], + "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + } + ] +} diff --git a/packer/vars/.gitignore b/packer/vars/.gitignore new file mode 100644 index 000000000..af920284d --- /dev/null +++ b/packer/vars/.gitignore @@ -0,0 +1,2 @@ +# exclude private cloud-env settings +cloud-env.json diff --git a/packer/vars/centos.json b/packer/vars/centos.json new file mode 100644 index 000000000..74d573099 --- /dev/null +++ b/packer/vars/centos.json @@ -0,0 +1,7 @@ +{ + "base_image": "CentOS 7 (20160518T212648) - LF upload", + "cloud_user": "centos", + + "distro": "CentOS 7", + "cloud_user_data": "provision/rh-user_data.sh" +} diff --git a/packer/vars/cloud-env.json.example b/packer/vars/cloud-env.json.example new file mode 100644 index 000000000..3a2ea9d2b --- /dev/null +++ b/packer/vars/cloud-env.json.example @@ -0,0 +1,6 @@ +{ + "stack_tenant": "TENANTID", + "stack_user": "USERID", + "stack_pass": "PASSWD", + "stack_network": "NETID", +} diff --git a/packer/vars/ubuntu-14.04.json b/packer/vars/ubuntu-14.04.json new file mode 100644 index 000000000..6c58edb78 --- /dev/null +++ b/packer/vars/ubuntu-14.04.json @@ -0,0 +1,7 @@ +{ + "base_image": "Ubuntu 14.04 (20160518T212648) - LF upload", + "cloud_user": "ubuntu", + + "distro": "Ubuntu 14.04", + "cloud_user_data": "provision/null_data.sh" +} diff --git a/packer/vars/ubuntu-16.04.json b/packer/vars/ubuntu-16.04.json new file mode 100644 index 000000000..ceced39ff --- /dev/null +++ b/packer/vars/ubuntu-16.04.json @@ -0,0 +1,7 @@ +{ + "base_image": "Ubuntu 16.04 (20160518T212648) - LF upload", + "cloud_user": "ubuntu", + + "distro": "Ubuntu 16.04", + "cloud_user_data": "provision/null_data.sh" +} |