aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools/disk-image-builder/ubuntu/build.sh
blob: 2b953f1c3adaf0d796cacd05dde105c8269aae28 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
#!/bin/bash -e

# Copyright (c) 2016 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Purpose of this script is to build a VirtualBox or QCOW2 disk image
# for use with CSIT testing.
#
# As input parameters, the script takes:
#
# - A base Linux distribution (currently, only "ubuntu-14.04.4" is
#   supported),
# - A release timestamp in the format "YYYY-MM-DD" eg. "2016-05-21".
#   This timestamp MUST reference to a list of packages (APT, PIP)
#   that was previously generated by the "listmaker" VM and script.
# - A path to the nested VM image.
#
# The bullk of the work is done by packer,
# while this script does some of the pre- and post-processing.
# Steps executed are:
#
# 1.) Determine if building a QCOW or VirtualBox image. Currently,
#     we build a QCOW image if VIRL_* environment variables are
#     present (and we therefore assume we are building for VIRL),
#     or VirtualBox otherwise.
#
# 2.) Download packer, if not already installed.
#
# 3.) Download APT and PIP packages as required for the image.
#     We're downloading them here, rather than letting the VM
#     download them off the public internet, for two reasons:
#     a.) This allows us to keep and cache packets between runs
#         and download them only once,
#     b.) This means only the build host needs a working proxy
#         configuration and we do not need to worry about setting
#         a proxy inside the VM.
#
# 4.) Link the nested VM image into the main VM's temp directory
#
# 5.) Create Version and Changelog files
#
# 6.) Run Packer. Packer, in turn, will:
#     6.1.) Download the Operating System ISO image,
#     6.2.) Start a VM using the selected hypervisor (VirtualBox or Qemu),
#     6.3.) Boot the VM using the ISO image and send initial keystrokes
#           to initiate installation using a Preseed or Kickstart file,
#     6.4.) Drive the installation using until the point the VM is reachable
#           over SSH,
#     6.5.) Copy the temporary directory populated in steps 3-5 above to the VM,
#     6.6.) Run a script on the VM that performs post-installation:
#           6.6.1.) Install selected .deb packages
#           6.6.2.) Install PIP packages as per requirements.txt file
#           6.6.3.) Install nested VM image and VERSION/CHANGELOG files
#     6.7.) Run a script on VM that creates a Vagrant user (VirtualBox only)
#     6.8.) Run a script on VM that performs clean-up:
#           6.8.1.) Remove any temporary files created,
#           6.8.2.) Remove SSH host-keys
#           6.8.3.) Remove root user password
#           6.8.4.) Shut down the VM
#     6.9.) [TODO]: Upload the image to VIRL (QCOW only), -or-
#           Convert the image into a Vagrant box (VirtualBox only), and
#           [TODO/FIX]: Upload the Vagrant box to Atlas (VirtualBox only)
#
# 7.) Clean up

###
### 0. Set constants and verify parameters.
###
cd $(dirname $0)
BUILD_DIR="$(pwd)/build"
PACKER_DIR="${BUILD_DIR}/packer"

APT_CACHE_DIR="${BUILD_DIR}/cache/apt"
PIP_CACHE_DIR="${BUILD_DIR}/cache/pip"

PACKER_TEMPLATE="ubuntu-14.04.4.json"
LISTS_DIR="$(dirname $0)/lists"

function syntax {
  echo 'Syntax: $0 <Operating System> <Release> <Nested VM image>'
  echo
  echo '<Operating System>: Base distro, eg. ubuntu-14.04.4'
  echo '<Release>:          Release timestamp, eg. 2016-05-21'
  echo '<Nested VM image>:  Path to nested VM image'

  exit 1
}

## Parse command line options

OS=$1
RELDATE=$2
NESTED_IMG=$3

if [ "$3" = "" ]
then
  syntax
fi

## Identify version by looking at topmost version statement in CHANGELOG

VERSION=$(cat $(dirname $0)/CHANGELOG  | grep '^## ' | head -1 | \
  sed -e 's/.*\[\(.*\)\].*/\1/')
if [ "${VERSION}" = "" ]
then
  echo "Unable to determine build version from CHANGELOG file. Make sure"
  echo "that there is an entry for the most recent version in CHANGELOG,"
  echo "and that the entry is formated like"
  echo
  echo "## [1.0] - 2016-05-20"
  exit 1
fi
RELEASE="csit-${OS}_${RELDATE}_${VERSION}"

OUTPUT_DIR="${BUILD_DIR}/output/${RELEASE}"
LIST="${LISTS_DIR}/${OS}_${RELDATE}_${VERSION}"

if [ ! -d "${LIST}" ]
then
  echo "${LIST} not found"
  syntax
  exit 1
fi
if [ ! -f $NESTED_IMG ]
then
  echo "Nested image $NESTED_IMG not found"
  syntax
  exit 1
fi

ATLAS_RELDATE=${RELDATE//-}
ATLAS_VERSION="${ATLAS_RELDATE}.${VERSION}"

# Find an MD5 checksum utility

MD5UTIL=$(which md5sum) || MD5UTIL=$(which md5)
if [ $? -ne 0 ]
then
  echo "No MD5 utility found."
  echo "Please make sure you either have \"md5sum\" or \"md5\" installed."
  exit 1
fi

###
### 1. Determine build target.
###
if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
then
  OUTPUT_PROVIDER="virtualbox"
else
  OUTPUT_PROVIDER="qemu"
fi

echo "Building version $VERSION for ${OUTPUT_PROVIDER}"
echo "Release ${RELEASE}"
echo "Using Nested VM image: ${NESTED_IMG}"
echo


###
### 2. Download and extract packer, if not already installed
###
packer_os=$(uname -s)
if [ "$packer_os" = "Darwin" ]
then
  packer_url="https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_darwin_amd64.zip"
elif [ "$packer_os" = "Linux" ]
then
  packer_url="https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip"
fi

mkdir -p $BUILD_DIR
wget -P ${PACKER_DIR} -N ${packer_url}

unzip -n ${PACKER_DIR}/packer*zip -d ${PACKER_DIR}

###
### 3. Download APT and PIP packages, and cache them
###    Verify checksum of downloaded APT packages.
###    Link required packages into a temp directory for the VM image.
###

## APT

# This seems to be where Ubuntu keeps superseded versions of packages
LAUNCHPAD_URL='https://launchpad.net/ubuntu/+archive/primary/+files/'

rm -fr ${OUTPUT_DIR}
mkdir -p ${OUTPUT_DIR}/temp/deb
mkdir -p ${APT_CACHE_DIR}

APT_FILE="${LIST}/apt-packages.txt"
while read url name size checksum
do
  # Download if not already present
  if [ ! -f ${APT_CACHE_DIR}/$name ]
  then
    if ! wget -P ${APT_CACHE_DIR} -O ${APT_CACHE_DIR}/$name ${url//\'}
    then
      echo "WARNING: Unable to get package using previously recorded URL, may"
      echo "         be superseded. Trying launchpad instead."
      url_basename=$(basename $url | tr -d "'")
      wget -P ${APT_CACHE_DIR} -O ${APT_CACHE_DIR}/$name "${LAUNCHPAD_URL}${url_basename}"
    fi
  fi

  # Verify checksum (always -- regardless of whether the package was
  # just downloaded, or was already there
  actual_md5sum=$(${MD5UTIL} < ${APT_CACHE_DIR}/$name)
  if [ ! "${actual_md5sum:0:32}" = "${checksum:7}" ]
  then
    echo File $name checksum failure."
    echo "Got ${actual_md5sum:0:31} expected ${checksum:7}.
    rm -f ${APT_CACHE_DIR}/$name
    exit 1
  fi

  # Link package into VM temp directory
  ln ${APT_CACHE_DIR}/$name ${OUTPUT_DIR}/temp/deb/${name}
done < $APT_FILE

## PIP

mkdir -p ${PIP_CACHE_DIR}

# Let PIP do the work of downloading and verifying packages
pip install --download ${PIP_CACHE_DIR} -r ${LIST}/pip-requirements.txt

# Link packages and requirements file into VM's temp directory
mkdir -p ${OUTPUT_DIR}/temp/pip
ln ${PIP_CACHE_DIR}/* ${OUTPUT_DIR}/temp/pip/
ln ${LIST}/pip-requirements.txt ${OUTPUT_DIR}/temp/requirements.txt

###
### 4. Link the nested VM image
###
rm -fr ${OUTPUT_DIR}/temp/nested-vm
mkdir ${OUTPUT_DIR}/temp/nested-vm
ln $NESTED_IMG ${OUTPUT_DIR}/temp/nested-vm/

###
### 5. Create Version and Changelog files
###

echo ${RELEASE} > ${OUTPUT_DIR}/temp/VERSION
ln CHANGELOG ${OUTPUT_DIR}/temp/CHANGELOG

###
### 6. Run packer
###
export PACKER_LOG=1
export PACKER_LOG_PATH=${OUTPUT_DIR}/packer.log
${PACKER_DIR}/packer build -var "release=${RELEASE}" \
 -only ${RELEASE}-${OUTPUT_PROVIDER} \
 -var "output_dir=${OUTPUT_DIR}/packer" \
 -var "temp_dir=${OUTPUT_DIR}/temp" \
 -var "atlas_version=${ATLAS_VERSION}" \
 -force \
 -machine-readable ${PACKER_TEMPLATE}

# TODO: Need to figure out "packer push" later. Currently keeps failing.
# ${PACKER_DIR}/packer push -name ckoester/test123 -var "release=${RELEASE}" \
#   -var "output_dir=${OUTPUT_DIR}/packer" \
#   -var "temp_dir=${OUTPUT_DIR}/temp" \
#   ${PACKER_TEMPLATE}

###
### 7. Clean up
###
rm -fr ${OUTPUT_DIR}/temp

echo "Done."
echo "Artifacts:"
echo
ls -lR ${OUTPUT_DIR}