From 2aa973ce201fdeb5d385708ae243ab14d3bcefb7 Mon Sep 17 00:00:00 2001 From: "C.J. Collier" Date: Tue, 17 May 2016 12:03:02 -0700 Subject: Document the process of re-spinning an image * Document the process of respinning jcloud images in README.txt * Create /scripts/respin-jcloud-images.sh * Create /vagrant/lib/respin-functions.sh * Automate the process of fetching latest base images * Automate the process of creating new "LF upload" snapshots * Automate process of creating jcloud image - Boot fresh VM - run bootstrap.sh - run system_reseal.sh - snapshot re-sealed system * Unable to automate updating jcloud image regex for staging nodes * Correct shasum to sha1sum * Add cloud-initramfs-* to PACKAGES list * Install cloud-init during system reseal Change-Id: Ie70d5f1fdee0e9f06810da6f37bfc710e3e06b8e Signed-off-by: C.J. Collier --- vagrant/lib/bootstrap-functions.sh | 81 +++++------ vagrant/lib/respin-functions.sh | 284 +++++++++++++++++++++++++++++++++++++ vagrant/lib/system_reseal.sh | 11 +- 3 files changed, 329 insertions(+), 47 deletions(-) create mode 100644 vagrant/lib/respin-functions.sh (limited to 'vagrant/lib') diff --git a/vagrant/lib/bootstrap-functions.sh b/vagrant/lib/bootstrap-functions.sh index d8c273d65..7805f67cf 100644 --- a/vagrant/lib/bootstrap-functions.sh +++ b/vagrant/lib/bootstrap-functions.sh @@ -8,16 +8,6 @@ do_cleanup() { perl -i -ne 'print unless /^127.0.1.1.*# temporary$/' /etc/hosts } -deb_enable_serial_console() { - # enable grub and login on serial console - - echo <> /etc/default/grub -GRUB_TERMINAL=serial -GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --word=8 --parity=no --stop=1" -EOF - update-grub -} - deb_probe_modules() { for mod in "$@" do @@ -48,15 +38,17 @@ Dpkg::Options { "--force-confold"; }; +quiet "2"; + EOF } deb_sync_minor() { echo '---> Updating OS' # Standard update + upgrade dance - apt-get -qq update - apt-get -qq upgrade - apt-get -qq dist-upgrade + apt-get update + apt-get upgrade + apt-get dist-upgrade } deb_correct_shell() { @@ -67,42 +59,41 @@ deb_correct_shell() { deb_flush() { echo '---> Flushing extra packages and package cache' - apt-get -qq autoremove - apt-get -qq clean + apt-get autoremove + apt-get clean } deb_add_ppa() { echo "---> Adding '$1' PPA" - apt-get -qq install software-properties-common - apt-add-repository -y $1 - apt-get -qq update + apt-get install software-properties-common + ATTEMPT=0 + while [ ${ATTEMPT} -le 4 ] + do + FAIL=0 + apt-add-repository -y $1 || FAIL=1 + if [ ${FAIL} -eq 0 ] + then + break + fi + ATTEMPT=$(expr $ATTEMPT + 1) + done + apt-get update } deb_install_pkgs() { + apt-get install lsb-release LSB_PATH=$(which lsb_release) - if [ $? == 0 ] - then - VERSION=$(lsb_release -r | awk '{print $2}') - DIST=$(lsb_release -i | awk '{print $3}') - CODENAME=$(lsb_release -c | awk '{print $2}') - else - ISSUE_TXT=$(head -1 /etc/issue) - DIST=$(echo "${ISSUE_TXT}" | awk '{print $1}') - if [ "$DIST" = "Ubuntu" ] - then - VERSION=$(echo "${ISSUE_TXT}" | awk '{print $2}' | sed -e 's/^(\d+\.\d+)(\.\d+)?$/\1/') - elif [ "$DIST" = "Debian" ] - then - VERSION=$(echo "${ISSUE_TXT}" | awk '{print $3}') - else - echo "Unrecognized distribution: ${DIST}" - fi - fi + VERSION=$(lsb_release -r | awk '{print $2}') + DIST=$(lsb_release -i | awk '{print $3}') + CODENAME=$(lsb_release -c | awk '{print $2}') echo "---> Detected [${DIST} v${VERSION} (${CODENAME})]" - PACKAGES="" # initialize PACKAGES + # initialize PACKAGES + PACKAGES="cloud-initramfs-dyn-netconf cloud-initramfs-growroot + cloud-initramfs-rescuevol" + if [ "$VERSION" = '14.04' ] then # openjdk-8-jdk is not available in 14.04 repos by default @@ -117,7 +108,7 @@ deb_install_pkgs() { # plymouth-label and plymouth-themes are required to get rid of # initrd warnings / errors on 16.04 - apt-get -qq install plymouth-themes plymouth-label + apt-get install plymouth-themes plymouth-label fi # Build tools - should match vpp/Makefile DEB_DEPENDS variable @@ -141,11 +132,11 @@ deb_install_pkgs() { echo '---> Installing packages' # disable double quoting check # shellcheck disable=SC2086 - apt-get -qq install ${PACKAGES} + apt-get install ${PACKAGES} # Specify documentation packages DOC_PACKAGES="doxygen graphviz" - apt-get -qq install ${DOC_PACKAGES} + apt-get install ${DOC_PACKAGES} } deb_enable_hugepages() { @@ -205,14 +196,14 @@ rh_install_pkgs() { yum install -q -y install ${DOC_PACKAGES} # Install python development - OUTPUT=$(yum search python34-devel 2>&1) - if [ "$OUTPUT" == "No matches" ] + OUTPUT=$(yum search python34-devel 2>&1 | grep 'No matches') + if [ -z "$OUTPUT" ] then - echo '---> Installing python-devel' - yum install -q -y python-devel - else echo '---> Installing python34-devel' yum install -q -y python34-devel + else + echo '---> Installing python-devel' + yum install -q -y python-devel fi echo '---> Configuring EPEL' diff --git a/vagrant/lib/respin-functions.sh b/vagrant/lib/respin-functions.sh new file mode 100644 index 000000000..e4fdb2a31 --- /dev/null +++ b/vagrant/lib/respin-functions.sh @@ -0,0 +1,284 @@ +#!/bin/bash + +# Copyright 2016 The Linux Foundation + +PVE_ROOT=/usr/src/git/lf/git.lf.org/cjcollier/python-virtual +CPPROJECT=${CPPROJECT:-fdio} +PVENAME="${CPPROJECT}-openstack" +PVE_PATH="${PVE_ROOT}/${PVENAME}" +PVERC=${PVE_PATH}/bin/activate +SERVER_NAME=${SERVER_NAME:-${USER}-vagrant} + +if [ ! -d ${PVE_PATH} ] +then + mkdir -p $(dirname $PVE_PATH) + if [ -f /etc/debian_version ] + then + sudo apt-get -y -qq install python-virtualenv libpython-dev + elif [ -f /etc/redhat-release ] + then + sudo yum -y install python-virtualenv + fi + + python-virtualenv ${PVE_PATH} + + echo "Please visit https://secure.vexxhost.com/console/#/account/credentials and place all $OS_* variables at the end of ${PVERC}" +fi + +RH_ARCH64=x86_64 +RH_ARCH32=i686 +DEB_ARCH64=amd64 +DEB_ARCH32=i386 +LV_IMG_DIR=/var/lib/libvirt/images/ +SRC_TIMESTAMP="" +DST_TIMESTAMP="" + +function new_dst_timestamp () +{ + if [ -z "${DST_TIMESTAMP}" ] + then + DST_TIMESTAMP=$(date +'%F T %T' | sed -e 's/[-: ]//g') + fi + + echo ${DST_TIMESTAMP} + return 0 +} + +function new_src_timestamp () +{ + if [ -z "${SRC_TIMESTAMP}" ] + then + SRC_TIMESTAMP=$(date +'%F T %T' | sed -e 's/[-: ]//g') + fi + + echo ${SRC_TIMESTAMP} + return 0 +} + +function latest_src_timestamp () +{ + if [ -z "${SRC_TIMESTAMP}" ] + then + SRC_TIMESTAMP=$(glance image-list | perl -n -e 'if( /\((\S+)\) - LF upload/ ){ print "$1\n" }' | sort | tail -1) + fi + + echo ${SRC_TIMESTAMP} + return 0 +} + +# +# usage: +# glance_image_create ${IMG_NAME} ${IMG_PATH} +# +# example: +# glance_image_create "CentOS 7 (20160517T143002) - LF upload" /var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2c +# +function glance_image_create () +{ + glance image-create --disk-format qcow2 --container-format bare --progress \ + --name "${1}" --file "${2}" +} + +function setup_rh () +{ + SRC_TIMESTAMP=$(new_src_timestamp) + DIST=$1 + VERSION=$2 + ARCH=$3 + ARCH=${ARCH:-${RH_ARCH64}} + IMG_FNAME="${DIST}-${VERSION}-${ARCH}-GenericCloud.qcow2c" + IMG_PATH="${LV_IMG_DIR}/${IMG_FNAME}" + IMG_NAME="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload" +} + + +# +# usage: +# create_rh_image ${DIST} ${VERSION} ${ARCH} +# +# example: +# create_rh_image CentOS 7 x86_64 +# +function create_rh_image () +{ + setup_rh "$@" + + if [ ! -f ${IMG_PATH} ]; then download_rh_image "$@"; fi + + glance_image_create "${IMG_NAME}" "${IMG_PATH}" +} + +function download_rh_image () +{ + setup_rh "$@" + echo "--> Fetching image file for ${DIST} ${VERSION}" + wget -cP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}" +} + + +declare -A deb_codename_map +deb_codename_map=(['3.0']=woody \ + ['3.1']=sarge \ + ['4']=etch \ + ['5']=lenny \ + ['6']=squeeze \ + ['7']=wheezy \ + ['8']=jessie \ + ['9']=stretch \ + ['10']=buster \ + ) +declare -A ubuntu_codename_map +ubuntu_codename_map=(['6.06']=dapper \ + ['8.04']=hardy \ + ['10.04']=lucid \ + ['12.04']=precise \ + ['14.04']=trusty \ + ['16.04']=xenial \ + ) +DEB_CURRENT_VER='8.4.0' +DEB_CURRENT_CODENAME='jessie' + +DEB_TESTING_VER='9.0.0' +DEB_TESTING_CODENAME='stretch' + +DEB_UNSTABLE_VER='10.0.0' +DEB_UNSTABLE_CODENAME='buster' + +function setup_deb () +{ + SRC_TIMESTAMP=$(new_src_timestamp) + DIST=$1 + VERSION=$2 + ARCH=$3 + ARCH=${ARCH:-${DEB_ARCH64}} + + declare -A V + VVAL=$(echo ${VERSION} | perl -ne 'm/(?:(\d+)(?:\.(\d+))?)(?:\.(\d+))?/; $min=$2 // 0; $mic = $3 // 0; print qq{([maj]=$1 [min]=$min [mic]=$mic)}') + eval "V=${VVAL}" + + LCDIST=$(echo ${DIST} | perl -ne 'print lc') + + MAJOR_VERSION="${V['maj']}" + MINOR_VERSION="${MAJOR_VERSION}.${V['min']}" + MICRO_VERSION="${MINOR_VERSION}.${V['mic']}" + + CODENAME="" + + if [ "Debian" == "${DIST}" ] + then + CODENAME="${deb_codename_map[$MINOR_VERSION]}" + CODENAME=${CODENAME:-${deb_codename_map[$MAJOR_VERSION]}} + if [ -z "$CODENAME" ] + then + echo "--> no codename for ${DIST} v${MICRO_VERSION}" + return -2 + fi + + URL_PFX="http://cdimage.debian.org/cdimage/openstack/" + + if [ "${DEB_CURRENT_CODENAME}" == "${CODENAME}" ] + then + OSTACK_SUBDIR='current' + QCOW_VER=${MICRO_VERSION} + elif [ "${DEB_TESTING_CODENAME}" == "${CODENAME}" ] + then + OSTACK_SUBDIR='testing' + QCOW_VER='testing' + else + echo "--> Not certain where to find images for ${DIST} v${MICRO_VERSION}" + fi + + IMG_FNAME="${LCDIST}-${QCOW_VER}-openstack-${ARCH}.qcow2" + URL="http://cdimage.debian.org/cdimage/openstack/${OSTACK_SUBDIR}/${IMG_FNAME}" + + elif [ "Ubuntu" == "${DIST}" ] + then + CODENAME="${ubuntu_codename_map[$MINOR_VERSION]}" + if [ -z "$CODENAME" ] + then + echo "--> no codename for ${DIST} v${MICRO_VERSION}" + return -2 + fi + + IMG_FNAME="${CODENAME}-server-cloudimg-${ARCH}-disk1.img" + URL="https://cloud-images.ubuntu.com/${CODENAME}/current/${IMG_FNAME}" + else + echo "--> unrecognized distribution: ${DIST}" + return -1 + fi + + export IMG_PATH="${LV_IMG_DIR}/${IMG_FNAME}" + export IMG_NAME="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload" + +} +# +# usage: +# download_deb_image ${DIST} ${VERSION} ${ARCH} +# +# example: +# download_deb_image Ubuntu 14.04 amd64 +# +function download_deb_image () +{ + setup_deb "$@" + + if [ -z "$URL" ]; then echo "Cannot fetch qcow2 image for ${DIST} v${MICRO_VERSION}"; return -3; fi + echo "--> Fetching image file for ${DIST} ${VERSION}" + wget -cP ${LV_IMG_DIR} "${URL}" +} + +# Used to upload +# +# usage: +# create_deb_image ${DIST} ${VERSION} ${ARCH} +# +# example: +# create_deb_image Ubuntu 14.04 amd64 +# +function create_deb_image () +{ + setup_deb "$@" + + if [ ! -f ${IMG_PATH} ]; then download_deb_image "$@"; fi + + glance_image_create "${IMG_NAME}" "${IMG_PATH}" +} + +function respin_deb_image () +{ + SRC_TIMESTAMP=$(latest_src_timestamp) + DST_TIMESTAMP=$(new_dst_timestamp) + setup_deb "$@" + export IMAGE="${IMG_NAME}" + echo "--> creating instance of image '${IMAGE}' as server name '${SERVER_NAME}'" + vagrant up + if [ "Ubuntu" == "${DIST}" ] + then + DST_IMAGE="${DIST} ${VERSION} LTS - basebuild - ${DST_TIMESTAMP}" + elif [ "Debian" == "${DIST}" ] + then + DST_IMAGE="${DIST} ${VERSION} - basebuild - ${DST_TIMESTAMP}" + else + echo "unrecognized disribution: ${DIST}" + exit -4 + fi + echo "--> Taking snapshot of image '${IMG_NAME}' with name '${DST_IMAGE}'" + nova image-create --poll "${SERVER_NAME}" "${DST_IMAGE}" + echo "--> Bringing down vagrant instance" + vagrant destroy +} + +function respin_rh_image () +{ + SRC_TIMESTAMP=$(latest_src_timestamp) + DST_TIMESTAMP=$(new_dst_timestamp) + setup_rh "$@" + IMAGE="${IMG_NAME}" + echo "--> creating instance of image '${IMG_NAME}' as server name '${SERVER_NAME}'" + vagrant up + DST_IMAGE="${DIST} ${VERSION} - basebuild - ${DST_TIMESTAMP}" + echo "--> Taking snapshot of image '${IMG_NAME}' with name '${DST_IMAGE}'" + nova image-create --poll "${SERVER_NAME}" "${DST_IMAGE}" + echo "--> Bringing down vagrant instance" + vagrant destroy +} diff --git a/vagrant/lib/system_reseal.sh b/vagrant/lib/system_reseal.sh index 6c7aa4d52..02d008498 100644 --- a/vagrant/lib/system_reseal.sh +++ b/vagrant/lib/system_reseal.sh @@ -48,6 +48,15 @@ rm -rf /var/lib/cloud/* # cleanup /vagrant rm -rf /vagrant +if [ -f /etc/debian_version ] +then + echo "********************************************" + echo "* NOW INSTALLING cloud-init PACKAGE *" + echo "********************************************" + + apt-get -y -qq install cloud-init > /dev/null 2>&1 +fi + # Force a system sync and sleep to get around any SSD issues echo "Forcing sync and sleep for 10sec" sync @@ -56,5 +65,3 @@ sleep 10 echo "********************************************" echo "* PLEASE SNAPSHOT IMAGE AT THIS TIME *" echo "********************************************" -init 1 -sleep 10 -- cgit 1.2.3-korg