diff options
| -rw-r--r-- | scripts/init-respin-env.sh | 146 | ||||
| -rwxr-xr-x | scripts/respin-jcloud-images.sh | 48 | ||||
| -rw-r--r-- | scripts/setup-vagrant.sh | 43 | ||||
| -rw-r--r-- | vagrant/lib/Respin.pm | 30 | ||||
| -rw-r--r-- | vagrant/lib/respin-functions.sh | 89 |
5 files changed, 253 insertions, 103 deletions
diff --git a/scripts/init-respin-env.sh b/scripts/init-respin-env.sh new file mode 100644 index 000000000..6ecfe8363 --- /dev/null +++ b/scripts/init-respin-env.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# Copyright 2016 The Linux Foundation + +set -e + +vagrant_version=1.8.1 +ruby_version=2.1.5 +ruby_patch=https://gist.github.com/mislav/055441129184a1512bb5.txt +rbenv_git=https://github.com/rbenv/rbenv.git + +PVENAME="${CPPROJECT}-openstack" +PVE_PATH="${PVE_ROOT}/${PVENAME}" +PVERC=${PVE_PATH}/bin/activate +PVE_BINDIR=$(dirname $PVERC) +PVE_ROOT="${HOME}/src/python-virtual" + +LOCAL_LIB="${HOME}/src/local-lib" +LL_LIBDIR="${LOCAL_LIB}/lib" + +RH_ARCH64=x86_64 +RH_ARCH32=i686 +DEB_ARCH64=amd64 +DEB_ARCH32=i386 +LV_IMG_DIR=/var/lib/libvirt/images/ +SRC_TIMESTAMP="" +DST_TIMESTAMP="" + +init_virtualenv +init_rbenv +init_ruby +select_ruby ${ruby_version} +init_virtualenv +init_vagrant +install_vagrant_plugins +import_vagrant_box +init_local_lib +init_javascript + +function init_virtualenv () +{ + test -d ${PVE_BINDIR} && return 0 + + if [ -f /etc/debian_version ] + then + sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev + elif [ -f /etc/redhat-release ] + then + sudo yum -y install python-virtualenv + fi + + mkdir -p ${PVE_PATH} + virtualenv ${PVE_PATH} + + echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}" + echo "Press enter when finished" + read +} + +function init_local_lib () +{ + test -d ${LL_LIBDIR} && return 0 + + echo "local lib init incomplete" +} + +function init_javascript () +{ + which js && which jq && return 0 + + if [ -f /etc/debian_version ] + then + sudo apt-get -y -qq install nodejs jq + elif [ -f /etc/redhat-release ] + then + sudo yum -y install nodejs jq + fi +} + +function init_vagrant () +{ + which vagrant && return 0 + + vagrant_pkg_name=vagrant_${vagrant_version}_x86_64.deb + vagrant_pkg=https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_pkg_name} + + wget -c ${vagrant_pkg} + sudo dpkg -i ${vagrant_pkg_name} +} + +function init_rbenv () +{ + which rbenv && return 0 + + # clone rbenv + test -d ~/.rbenv/.git || git clone ${rbenv_git} ~/.rbenv + + # clone ruby-build + mkdir -p ~/.rbenv/plugins + test -d ~/.rbenv/plugins/ruby-build/.git || git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build + + # build ruby-build + cd ~/.rbenv && src/configure && make -C src + + # Add rbenv to bashrc + grep HOME/.rbenv/bin ~/.bash_profile || echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile + + # Add rbenv to current environment + export PATH="$HOME/.rbenv/bin:$PATH" +} + +function init_ruby () +{ + # Install ruby build deps + sudo apt-get build-dep ruby + #sudo apt-get -y install \ + # autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev \ + # zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev + + # Build ruby + curl -fsSL ${ruby_patch} | rbenv install --patch ${ruby_version} +} + +function select_ruby () +{ + # Select ruby ${ruby_version} from rbenv + rbenv local ${1} + rbenv global ${1} +} + +function install_vagrant_plugins () +{ + for plugin in vagrant-openstack-provider vagrant-cachier vagrant-mutate + do + vagrant plugin install ${plugin} + done +} + +function import_vagrant_box () +{ + # Add dummy box + vagrant box add dummy https://github.com/huit/vagrant-openstack/blob/master/dummy.box + + cp ${CI_MGMT}/vagrant/examples/box/dummy/Vagrantfile ~/.vagrant.d/boxes/dummy/0/openstack/ +} + diff --git a/scripts/respin-jcloud-images.sh b/scripts/respin-jcloud-images.sh index 64a5c92f3..2af9e4c1b 100755 --- a/scripts/respin-jcloud-images.sh +++ b/scripts/respin-jcloud-images.sh @@ -1,25 +1,43 @@ #!/bin/bash - +# Copyright 2016 The Linux Foundation set -e +SUPPORTED_PLATFORMS=( + 'Ubuntu 14.04 amd64' + 'Ubuntu 16.04 amd64' + 'CentOS 7 x86_64' +) + CI_MGMT=$(realpath $(dirname $(realpath $0))/..) -# Copyright 2016 The Linux Foundation <cjcollier@linuxfoundation.org> source ${CI_MGMT}/vagrant/lib/respin-functions.sh +source ${PVERC} + +echo nova: $(which nova) + +export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")} + +for PLATFORM in "${SUPPORTED_PLATFORMS[@]}" +do + read -ra DVA <<< "${PLATFORM}" + DIST="${DVA[0]}" + VERSION="${DVA[1]}" + ARCH="${DVA[2]}" + DTYPE=$(dist_type ${DIST}) -source ${PVE_PATH}/bin/activate + AGE_JSON=$(latest_src_age ${DIST} ${VERSION} ${ARCH}); -# Acquire bootstrap images -#download_deb_image 'Ubuntu' '14.04' 'amd64' -#download_deb_image 'Ubuntu' '16.04' 'amd64' -#download_rh_image 'CentOS' '7' 'x86_64' + # only fetch new base image if our latest one is more than two weeks old + if [ $(echo ${AGE_JSON} | jq .week) -ge "3" ] + then + # Acquire bootstrap images + download_${DTYPE}_image "${DIST}" "${VERSION}" "${ARCH}" -# Push images to openstack via glance -#create_deb_image 'Ubuntu' '14.04' 'amd64' -#create_deb_image 'Ubuntu' '16.04' 'amd64' -#create_rh_image 'CentOS' '7' 'x86_64' + # Push images to openstack via glance + create_${DTYPE}_image "${DIST}" "${VERSION}" "${ARCH}" + fi -# Respin images -respin_deb_image 'Ubuntu' '14.04' 'amd64' -respin_deb_image 'Ubuntu' '16.04' 'amd64' -respin_rh_image 'CentOS' '7' 'x86_64' + # Respin images + cd ${CI_MGMT}/vagrant/basebuild + respin_${DTYPE}_image "${DIST}" "${VERSION}" "${ARCH}" +done diff --git a/scripts/setup-vagrant.sh b/scripts/setup-vagrant.sh deleted file mode 100644 index b7145d49a..000000000 --- a/scripts/setup-vagrant.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -e - -# fetch and install vagrant package -vagrant_version=1.8.1 -vagrant_pkg_name=vagrant_${vagrant_version}_x86_64.deb -vagrant_pkg=https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_pkg_name} -wget -c $vagrant_pkg -sudo dpkg -i $vagrant_pkg_name - -# clone rbenv -test -d ~/.rbenv/.git || git clone https://github.com/rbenv/rbenv.git ~/.rbenv - -# clone ruby-build -mkdir -p ~/.rbenv/plugins -test -d ~/.rbenv/plugins/ruby-build/.git || git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build - -# build ruby-build -cd ~/.rbenv && src/configure && make -C src - -# Add rbenv to bashrc -echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile - -# Add rbenv to current environment -export PATH="$HOME/.rbenv/bin:$PATH" - -# Install ruby build deps -sudo apt-get build-dep ruby2.3 -#sudo apt-get -y install \ -# autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev \ -# zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev - -# Build ruby 2.1.5 -curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.1.5 - -# Select ruby 2.1.5 from rbenv -rbenv local 2.1.5 -rbenv global 2.1.5 - -# Add dummy box -vagrant box add dummy https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box -cp ${CIADM_DIR}/vagrant/examples/box/dummy/Vagrantfile ~/.vagrant.d/boxes/dummy/0/openstack/ diff --git a/vagrant/lib/Respin.pm b/vagrant/lib/Respin.pm new file mode 100644 index 000000000..e72c601a1 --- /dev/null +++ b/vagrant/lib/Respin.pm @@ -0,0 +1,30 @@ +package Respin; + +use strict; +use warnings; +use DateTime; +use DateTime::Format::Duration; +use DateTime::Duration; +use JSON::XS; + +my $iso8601_rx = qr{^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})$}; + +my $json = JSON::XS->new->utf8; + +my $dur_fmt = DateTime::Format::Duration->new( + normalize => 1, + pattern => + q{{"week":"%V","day":"%u","hour":"%k","minute":"%M","second":"%S"}} +); + +sub latest_src_age { + my ( $now, $src ) = @_; + + my ( %now, %src ); + @now{qw(year month day hour minute second)} = ( $now =~ $iso8601_rx ); + @src{qw(year month day hour minute second)} = ( $src =~ $iso8601_rx ); + + print $dur_fmt->format_duration_from_deltas( + DateTime->new(%now)->subtract_datetime_absolute( DateTime->new(%src) ) + ->deltas ); +} diff --git a/vagrant/lib/respin-functions.sh b/vagrant/lib/respin-functions.sh index 9a9f9809c..7aa742c32 100644 --- a/vagrant/lib/respin-functions.sh +++ b/vagrant/lib/respin-functions.sh @@ -1,53 +1,37 @@ #!/bin/bash -# Copyright 2016 The Linux Foundation <cjcollier@linuxfoundation.org> - -PVE_ROOT="${HOME}/src/python-virtual" -CPPROJECT=${CPPROJECT:-fdio} -PVENAME="${CPPROJECT}-openstack" -PVE_PATH="${PVE_ROOT}/${PVENAME}" -PVERC=${PVE_PATH}/bin/activate -SERVER_NAME=${SERVER_NAME:-${USER}-vagrant} - -STACK_PROVIDER=vexxhost -STACK_PORTAL=secure.${STACK_PROVIDER}.com -STACK_ID_SERVER=auth.${STACK_PROVIDER}.net - -export OPENSTACK_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/" -export OPENSTACK_FLAVOR='v1-standard-4' -export STACK_REGION_NAME='ca-ymq-1' -export AVAILABILITY_ZONE='ca-ymq-2' -export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")} - -if [ ! -d ${PVE_PATH} ] -then - mkdir -p $(dirname $PVE_PATH) - if [ -f /etc/debian_version ] - then - sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev - elif [ -f /etc/redhat-release ] - then - sudo yum -y install python-virtualenv - fi +# Copyright 2016 The Linux Foundation - python-virtualenv ${PVE_PATH} +source ${CI_MGMT}/vagrant/lib/vagrant-functions.sh - echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}" -fi -RH_ARCH64=x86_64 -RH_ARCH32=i686 -DEB_ARCH64=amd64 -DEB_ARCH32=i386 -LV_IMG_DIR=/var/lib/libvirt/images/ -SRC_TIMESTAMP="" -DST_TIMESTAMP="" +source ${PVERC} +pip install -q --upgrade pip setuptools python-{cinder,glance,keystone,neutron,nova,openstack}client + +# +# usage: +# AGE_JSON=$(latest_src_age ${DIST} ${VERSION} ${ARCH}) +# +function latest_src_age () +{ + SRC_TS=$(latest_src_timestamp "$@") + NOW_TS=$(new_timestamp) + + perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )' + + return 0 +} + +function new_timestamp () +{ + date +'%F T %T' | sed -e 's/[-: ]//g' +} function new_dst_timestamp () { if [ -z "${DST_TIMESTAMP}" ] then - DST_TIMESTAMP=$(date +'%F T %T' | sed -e 's/[-: ]//g') + DST_TIMESTAMP=$(new_timestamp) fi echo ${DST_TIMESTAMP} @@ -101,7 +85,6 @@ function setup_rh () IMG_NAME="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload" } - # # usage: # create_rh_image ${DIST} ${VERSION} ${ARCH} @@ -122,7 +105,7 @@ function download_rh_image () { setup_rh "$@" echo "--> Fetching image file for ${DIST} ${VERSION}" - wget -cP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}" + wget -qcP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}" } @@ -234,7 +217,7 @@ function download_deb_image () if [ -z "$URL" ]; then echo "Cannot fetch qcow2 image for ${DIST} v${MICRO_VERSION}"; return -3; fi echo "--> Fetching image file for ${DIST} ${VERSION}" - wget -cP ${LV_IMG_DIR} "${URL}" + wget -qcP ${LV_IMG_DIR} "${URL}" } # Used to upload @@ -251,6 +234,7 @@ function create_deb_image () if [ ! -f ${IMG_PATH} ]; then download_deb_image "$@"; fi + echo "--> Pushing image ${IMG_NAME}" glance_image_create "${IMG_NAME}" "${IMG_PATH}" } @@ -261,7 +245,7 @@ function respin_deb_image () setup_deb "$@" export IMAGE="${IMG_NAME}" echo "--> creating instance of image '${IMAGE}' as server name '${SERVER_NAME}'" - vagrant up + vagrant up --provider=openstack if [ "Ubuntu" == "${DIST}" ] then DST_IMAGE="${DIST} ${VERSION} LTS - basebuild - ${DST_TIMESTAMP}" @@ -285,10 +269,25 @@ function respin_rh_image () setup_rh "$@" IMAGE="${IMG_NAME}" echo "--> creating instance of image '${IMG_NAME}' as server name '${SERVER_NAME}'" - vagrant up + vagrant up --provider=openstack DST_IMAGE="${DIST} ${VERSION} - basebuild - ${DST_TIMESTAMP}" echo "--> Taking snapshot of image '${IMG_NAME}' with name '${DST_IMAGE}'" nova image-create --poll "${SERVER_NAME}" "${DST_IMAGE}" echo "--> Bringing down vagrant instance" vagrant destroy } + +function dist_type () +{ + case "${1}" in + CentOS | RHEL | SuSE) + echo "rh" ;; + Debian | Ubuntu | Kali | ProxMox | VyOS) + echo "deb" ;; + *) + echo "Unrecognized distribution: ${1}" + exit 2 ;; + esac + +} + |