From afa1309db5813b191f5852bcab05f1f0c1d38047 Mon Sep 17 00:00:00 2001 From: "C.J. Collier" Date: Thu, 30 Jun 2016 06:56:43 -0700 Subject: Improve respin scripts In order to create a new OpenStack basebuild Images, a multi-phase process must be executed. The first phase is to configure the system for use with dynamic language environments. Due to security precautions, We do not currently have the ability to automate the process of respinning a new Image. We have a goal of fully automating the process so that a new Image will be generated on every commit. Until the process is self-hosted, we must manually fetch credentials from the OpenStack provider's web UI in order to make use of the python openstack bindings and to complete the basebuild re-spin process using Vagrant. The more generic portions of the Image are imported once every two to three weeks from upstream distributors. This duration is computed thus: $(perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )') | jq .week Once fetched, these qcow2 Images are pushed to OpenStack using glance and labelled as: "$(lsb_release -i) $(lsb_release -r) (YYYYMMDDTHHMMSS) - LF upload" The generated basebuild Images are pushed to OpenStack with glance labelled as: "$(lsb_release -i) $(lsb_release -r) - basebuild - YYYYMMDDTHHMMSS" ==== This patch makes it possible to perform the above multi-phase processes by executing a single script: bash scripts/respin-jcloud-images.sh This script sources a function library from the following file: vagrant/lib/respin-functions.sh Defined in this library are (among others): # Fetch latest image from upstream distributor download_deb_image( $dist, $version, $arch ) download_rh_image( $dist, $version, $arch ) # Fetch latest image from upstream distributor create_deb_image( $dist, $version, $arch ) create_rh_image( $dist, $version, $arch ) # Run Vagrant provisioning on upstream image respin_deb_image( $dist, $version, $arch ) respin_rh_image( $dist, $version, $arch ) The following script initializes vagrant and the following dynamic language environments: * python-virtualenv * pip * setuptools * python-{cinder,glance,keystone,neutron,nova,openstack}client * rbenv * Vagrant * vagrant-openstack-provider * nodejs * jq * local::lib * JSON::XS * DateTime * DateTime::Format::Duration * DateTime::Duration scripts/init-respin-env.sh The following module compares ISO 8601 dates (as included in Image labels) with the current time and prints to STDOUT the count of weeks, days, hours, minutes and seconds in JSON: vagrant/lib/Respin.pm Change-Id: I434cf2882e5e337ae4b55a4a7acb774a62b528b7 Signed-off-by: C.J. Collier --- vagrant/lib/Respin.pm | 30 ++++++++++++++ vagrant/lib/respin-functions.sh | 89 ++++++++++++++++++++--------------------- 2 files changed, 74 insertions(+), 45 deletions(-) create mode 100644 vagrant/lib/Respin.pm (limited to 'vagrant/lib') diff --git a/vagrant/lib/Respin.pm b/vagrant/lib/Respin.pm new file mode 100644 index 000000000..e72c601a1 --- /dev/null +++ b/vagrant/lib/Respin.pm @@ -0,0 +1,30 @@ +package Respin; + +use strict; +use warnings; +use DateTime; +use DateTime::Format::Duration; +use DateTime::Duration; +use JSON::XS; + +my $iso8601_rx = qr{^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})$}; + +my $json = JSON::XS->new->utf8; + +my $dur_fmt = DateTime::Format::Duration->new( + normalize => 1, + pattern => + q{{"week":"%V","day":"%u","hour":"%k","minute":"%M","second":"%S"}} +); + +sub latest_src_age { + my ( $now, $src ) = @_; + + my ( %now, %src ); + @now{qw(year month day hour minute second)} = ( $now =~ $iso8601_rx ); + @src{qw(year month day hour minute second)} = ( $src =~ $iso8601_rx ); + + print $dur_fmt->format_duration_from_deltas( + DateTime->new(%now)->subtract_datetime_absolute( DateTime->new(%src) ) + ->deltas ); +} diff --git a/vagrant/lib/respin-functions.sh b/vagrant/lib/respin-functions.sh index 9a9f9809c..7aa742c32 100644 --- a/vagrant/lib/respin-functions.sh +++ b/vagrant/lib/respin-functions.sh @@ -1,53 +1,37 @@ #!/bin/bash -# Copyright 2016 The Linux Foundation - -PVE_ROOT="${HOME}/src/python-virtual" -CPPROJECT=${CPPROJECT:-fdio} -PVENAME="${CPPROJECT}-openstack" -PVE_PATH="${PVE_ROOT}/${PVENAME}" -PVERC=${PVE_PATH}/bin/activate -SERVER_NAME=${SERVER_NAME:-${USER}-vagrant} - -STACK_PROVIDER=vexxhost -STACK_PORTAL=secure.${STACK_PROVIDER}.com -STACK_ID_SERVER=auth.${STACK_PROVIDER}.net - -export OPENSTACK_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/" -export OPENSTACK_FLAVOR='v1-standard-4' -export STACK_REGION_NAME='ca-ymq-1' -export AVAILABILITY_ZONE='ca-ymq-2' -export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")} - -if [ ! -d ${PVE_PATH} ] -then - mkdir -p $(dirname $PVE_PATH) - if [ -f /etc/debian_version ] - then - sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev - elif [ -f /etc/redhat-release ] - then - sudo yum -y install python-virtualenv - fi +# Copyright 2016 The Linux Foundation - python-virtualenv ${PVE_PATH} +source ${CI_MGMT}/vagrant/lib/vagrant-functions.sh - echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}" -fi -RH_ARCH64=x86_64 -RH_ARCH32=i686 -DEB_ARCH64=amd64 -DEB_ARCH32=i386 -LV_IMG_DIR=/var/lib/libvirt/images/ -SRC_TIMESTAMP="" -DST_TIMESTAMP="" +source ${PVERC} +pip install -q --upgrade pip setuptools python-{cinder,glance,keystone,neutron,nova,openstack}client + +# +# usage: +# AGE_JSON=$(latest_src_age ${DIST} ${VERSION} ${ARCH}) +# +function latest_src_age () +{ + SRC_TS=$(latest_src_timestamp "$@") + NOW_TS=$(new_timestamp) + + perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )' + + return 0 +} + +function new_timestamp () +{ + date +'%F T %T' | sed -e 's/[-: ]//g' +} function new_dst_timestamp () { if [ -z "${DST_TIMESTAMP}" ] then - DST_TIMESTAMP=$(date +'%F T %T' | sed -e 's/[-: ]//g') + DST_TIMESTAMP=$(new_timestamp) fi echo ${DST_TIMESTAMP} @@ -101,7 +85,6 @@ function setup_rh () IMG_NAME="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload" } - # # usage: # create_rh_image ${DIST} ${VERSION} ${ARCH} @@ -122,7 +105,7 @@ function download_rh_image () { setup_rh "$@" echo "--> Fetching image file for ${DIST} ${VERSION}" - wget -cP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}" + wget -qcP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}" } @@ -234,7 +217,7 @@ function download_deb_image () if [ -z "$URL" ]; then echo "Cannot fetch qcow2 image for ${DIST} v${MICRO_VERSION}"; return -3; fi echo "--> Fetching image file for ${DIST} ${VERSION}" - wget -cP ${LV_IMG_DIR} "${URL}" + wget -qcP ${LV_IMG_DIR} "${URL}" } # Used to upload @@ -251,6 +234,7 @@ function create_deb_image () if [ ! -f ${IMG_PATH} ]; then download_deb_image "$@"; fi + echo "--> Pushing image ${IMG_NAME}" glance_image_create "${IMG_NAME}" "${IMG_PATH}" } @@ -261,7 +245,7 @@ function respin_deb_image () setup_deb "$@" export IMAGE="${IMG_NAME}" echo "--> creating instance of image '${IMAGE}' as server name '${SERVER_NAME}'" - vagrant up + vagrant up --provider=openstack if [ "Ubuntu" == "${DIST}" ] then DST_IMAGE="${DIST} ${VERSION} LTS - basebuild - ${DST_TIMESTAMP}" @@ -285,10 +269,25 @@ function respin_rh_image () setup_rh "$@" IMAGE="${IMG_NAME}" echo "--> creating instance of image '${IMG_NAME}' as server name '${SERVER_NAME}'" - vagrant up + vagrant up --provider=openstack DST_IMAGE="${DIST} ${VERSION} - basebuild - ${DST_TIMESTAMP}" echo "--> Taking snapshot of image '${IMG_NAME}' with name '${DST_IMAGE}'" nova image-create --poll "${SERVER_NAME}" "${DST_IMAGE}" echo "--> Bringing down vagrant instance" vagrant destroy } + +function dist_type () +{ + case "${1}" in + CentOS | RHEL | SuSE) + echo "rh" ;; + Debian | Ubuntu | Kali | ProxMox | VyOS) + echo "deb" ;; + *) + echo "Unrecognized distribution: ${1}" + exit 2 ;; + esac + +} + -- cgit 1.2.3-korg