aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools/disk-image-builder/ubuntu/build.sh
diff options
context:
space:
mode:
authorCarsten Koester <ckoester@cisco.com>2016-05-25 05:58:29 -0400
committerDave Wallace <dwallacelf@gmail.com>2016-06-02 00:18:28 +0000
commit9d378548cc86028c55baf17260d32d0055603b78 (patch)
tree1cc67da7195459abcaca5f12f0f4b74bc399b60e /resources/tools/disk-image-builder/ubuntu/build.sh
parent036f36181c177e4c77f22cdbdcb87fb2f94df3bf (diff)
CSIT-111: Add Packer based framework to auto-generate disk images
Change-Id: Ia18ea34a6288aec0f12f176f4cc264d0f215d1fd Signed-off-by: Carsten Koester <ckoester@cisco.com>
Diffstat (limited to 'resources/tools/disk-image-builder/ubuntu/build.sh')
-rwxr-xr-xresources/tools/disk-image-builder/ubuntu/build.sh276
1 files changed, 276 insertions, 0 deletions
diff --git a/resources/tools/disk-image-builder/ubuntu/build.sh b/resources/tools/disk-image-builder/ubuntu/build.sh
new file mode 100755
index 0000000000..c94f445801
--- /dev/null
+++ b/resources/tools/disk-image-builder/ubuntu/build.sh
@@ -0,0 +1,276 @@
+#!/bin/bash -e
+
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Purpose of this script is to build a VirtualBox or QCOW2 disk image
+# for use with CSIT testing.
+#
+# As input parameters, the script takes:
+#
+# - A base Linux distribution (currently, only "ubuntu-14.04.4" is
+# supported),
+# - A release timestamp in the format "YYYY-MM-DD" eg. "2016-05-21".
+# This timestamp MUST reference to a list of packages (APT, PIP)
+# that was previously generated by the "listmaker" VM and script.
+# - A path to the nested VM image.
+#
+# The bullk of the work is done by packer,
+# while this script does some of the pre- and post-processing.
+# Steps executed are:
+#
+# 1.) Determine if building a QCOW or VirtualBox image. Currently,
+# we build a QCOW image if VIRL_* environment variables are
+# present (and we therefore assume we are building for VIRL),
+# or VirtualBox otherwise.
+#
+# 2.) Download packer, if not already installed.
+#
+# 3.) Download APT and PIP packages as required for the image.
+# We're downloading them here, rather than letting the VM
+# download them off the public internet, for two reasons:
+# a.) This allows us to keep and cache packets between runs
+# and download them only once,
+# b.) This means only the build host needs a working proxy
+# configuration and we do not need to worry about setting
+# a proxy inside the VM.
+#
+# 4.) Link the nested VM image into the main VM's temp directory
+#
+# 5.) Create Version and Changelog files
+#
+# 6.) Run Packer. Packer, in turn, will:
+# 6.1.) Download the Operating System ISO image,
+# 6.2.) Start a VM using the selected hypervisor (VirtualBox or Qemu),
+# 6.3.) Boot the VM using the ISO image and send initial keystrokes
+# to initiate installation using a Preseed or Kickstart file,
+# 6.4.) Drive the installation using until the point the VM is reachable
+# over SSH,
+# 6.5.) Copy the temporary directory populated in steps 3-5 above to the VM,
+# 6.6.) Run a script on the VM that performs post-installation:
+# 6.6.1.) Install selected .deb packages
+# 6.6.2.) Install PIP packages as per requirements.txt file
+# 6.6.3.) Install nested VM image and VERSION/CHANGELOG files
+# 6.7.) Run a script on VM that creates a Vagrant user (VirtualBox only)
+# 6.8.) Run a script on VM that performs clean-up:
+# 6.8.1.) Remove any temporary files created,
+# 6.8.2.) Remove SSH host-keys
+# 6.8.3.) Remove root user password
+# 6.8.4.) Shut down the VM
+# 6.9.) [TODO]: Upload the image to VIRL (QCOW only), -or-
+# Convert the image into a Vagrant box (VirtualBox only), and
+# [TODO/FIX]: Upload the Vagrant box to Atlas (VirtualBox only)
+#
+# 7.) Clean up
+
+###
+### 0. Set constants and verify parameters.
+###
+cd $(dirname $0)
+BUILD_DIR="$(pwd)/build"
+PACKER_DIR="${BUILD_DIR}/packer"
+
+APT_CACHE_DIR="${BUILD_DIR}/cache/apt"
+PIP_CACHE_DIR="${BUILD_DIR}/cache/pip"
+
+PACKER_TEMPLATE="ubuntu-14.04.4.json"
+LISTS_DIR="$(dirname $0)/lists"
+
+function syntax {
+ echo 'Syntax: $0 <Operating System> <Release> <Nested VM image>'
+ echo
+ echo '<Operating System>: Base distro, eg. ubuntu-14.04.4'
+ echo '<Release>: Release timestamp, eg. 2016-05-21'
+ echo '<Nested VM image>: Path to nested VM image'
+
+ exit 1
+}
+
+## Parse command line options
+
+OS=$1
+RELDATE=$2
+NESTED_IMG=$3
+
+if [ "$3" = "" ]
+then
+ syntax
+fi
+
+## Identify version by looking at topmost version statement in CHANGELOG
+
+VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | \
+ sed -e 's/.*\[\(.*\)\].*/\1/')
+if [ "${VERSION}" = "" ]
+then
+ echo "Unable to determine build version from CHANGELOG file. Make sure"
+ echo "that there is an entry for the most recent version in CHANGELOG,"
+ echo "and that the entry is formated like"
+ echo
+ echo "## [1.0] - 2016-05-20"
+ exit 1
+fi
+RELEASE="csit-${OS}_${RELDATE}_${VERSION}"
+
+OUTPUT_DIR="${BUILD_DIR}/output/${RELEASE}"
+LIST="${LISTS_DIR}/${OS}_${RELDATE}_${VERSION}"
+
+if [ ! -d "${LIST}" ]
+then
+ echo "${LIST} not found"
+ syntax
+ exit 1
+fi
+if [ ! -f $NESTED_IMG ]
+then
+ echo "Nested image $NESTED_IMG not found"
+ syntax
+ exit 1
+fi
+
+ATLAS_RELDATE=${RELDATE//-}
+ATLAS_VERSION="${ATLAS_RELDATE}.${VERSION}"
+
+# Find an MD5 checksum utility
+
+MD5UTIL=$(which md5sum) || MD5UTIL=$(which md5)
+if [ $? -ne 0 ]
+then
+ echo "No MD5 utility found."
+ echo "Please make sure you either have \"md5sum\" or \"md5\" installed."
+ exit 1
+fi
+
+###
+### 1. Determine build target.
+###
+if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
+then
+ OUTPUT_PROVIDER="virtualbox"
+else
+ OUTPUT_PROVIDER="qemu"
+fi
+
+echo "Building version $VERSION for ${OUTPUT_PROVIDER}"
+echo "Release ${RELEASE}"
+echo "Using Nested VM image: ${NESTED_IMG}"
+echo
+
+
+###
+### 2. Download and extract packer, if not already installed
+###
+packer_os=$(uname -s)
+if [ "$packer_os" = "Darwin" ]
+then
+ packer_url="https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_darwin_amd64.zip"
+elif [ "$packer_os" = "Linux" ]
+then
+ packer_url="https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip"
+fi
+
+mkdir -p $BUILD_DIR
+wget -P ${PACKER_DIR} -N ${packer_url}
+
+unzip -n ${PACKER_DIR}/packer*zip -d ${PACKER_DIR}
+
+###
+### 3. Download APT and PIP packages, and cache them
+### Verify checksum of downloaded APT packages.
+### Link required packages into a temp directory for the VM image.
+###
+
+## APT
+
+rm -fr ${OUTPUT_DIR}
+mkdir -p ${OUTPUT_DIR}/temp/deb
+mkdir -p ${APT_CACHE_DIR}
+
+APT_FILE="${LIST}/apt-packages.txt"
+while read url name size checksum
+do
+ # Download if not already present
+ if [ ! -f ${APT_CACHE_DIR}/$name ]
+ then
+ wget -P ${APT_CACHE_DIR} -O ${APT_CACHE_DIR}/$name ${url//\'}
+ fi
+
+ # Verify checksum (always -- regardless of whether the package was
+ # just downloaded, or already tehere
+ actual_md5sum=$(${MD5UTIL} < ${APT_CACHE_DIR}/$name)
+ if [ ! "${actual_md5sum:0:32}" = "${checksum:7}" ]
+ then
+ echo File $name checksum failure."
+ echo "Got ${actual_md5sum:0:31} expected ${checksum:7}.
+ rm -f ${APT_CACHE_DIR}/$name
+ exit 1
+ fi
+
+ # Link package into VM temp directory
+ ln ${APT_CACHE_DIR}/$name ${OUTPUT_DIR}/temp/deb/${name}
+done < $APT_FILE
+
+## PIP
+
+mkdir -p ${PIP_CACHE_DIR}
+
+# Let PIP do the work of downloading and verifying packages
+pip install --download ${PIP_CACHE_DIR} -r ${LIST}/pip-requirements.txt
+
+# Link packages and requirements file into VM's temp directory
+mkdir -p ${OUTPUT_DIR}/temp/pip
+ln ${PIP_CACHE_DIR}/* ${OUTPUT_DIR}/temp/pip/
+ln ${LIST}/pip-requirements.txt ${OUTPUT_DIR}/temp/requirements.txt
+
+###
+### 4. Link the nested VM image
+###
+rm -fr ${OUTPUT_DIR}/temp/nested-vm
+mkdir ${OUTPUT_DIR}/temp/nested-vm
+ln $NESTED_IMG ${OUTPUT_DIR}/temp/nested-vm/
+
+###
+### 5. Create Version and Changelog files
+###
+
+echo ${RELEASE} > ${OUTPUT_DIR}/temp/VERSION
+ln CHANGELOG ${OUTPUT_DIR}/temp/CHANGELOG
+
+###
+### 6. Run packer
+###
+export PACKER_LOG=1
+export PACKER_LOG_PATH=${OUTPUT_DIR}/packer.log
+${PACKER_DIR}/packer build -var "release=${RELEASE}" \
+ -only ${RELEASE}-${OUTPUT_PROVIDER} \
+ -var "output_dir=${OUTPUT_DIR}/packer" \
+ -var "temp_dir=${OUTPUT_DIR}/temp" \
+ -var "atlas_version=${ATLAS_VERSION}" \
+ -force \
+ -machine-readable ${PACKER_TEMPLATE}
+
+# TODO: Need to figure out "packer push" later. Currently keeps failing.
+# ${PACKER_DIR}/packer push -name ckoester/test123 -var "release=${RELEASE}" \
+# -var "output_dir=${OUTPUT_DIR}/packer" \
+# -var "temp_dir=${OUTPUT_DIR}/temp" \
+# ${PACKER_TEMPLATE}
+
+###
+### 7. Clean up
+###
+rm -fr ${OUTPUT_DIR}/temp
+
+echo "Done."
+echo "Artifacts:"
+echo
+ls -lR ${OUTPUT_DIR}