Improve respin scripts

In order to create a new OpenStack basebuild Images, a multi-phase
process must be executed.

The first phase is to configure the system for use with dynamic
language environments.

Due to security precautions, We do not currently have the ability to
automate the process of respinning a new Image.  We have a goal of
fully automating the process so that a new Image will be generated on
every commit.  Until the process is self-hosted, we must manually
fetch credentials from the OpenStack provider's web UI in order to
make use of the python openstack bindings and to complete the
basebuild re-spin process using Vagrant.

The more generic portions of the Image are imported once every two to
three weeks from upstream distributors.  This duration is computed
thus:

  $(perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )') | jq .week

Once fetched, these qcow2 Images are pushed to OpenStack using glance
and labelled as:

  "$(lsb_release -i) $(lsb_release -r) (YYYYMMDDTHHMMSS) - LF upload"

The generated basebuild Images are pushed to OpenStack with glance
labelled as:

  "$(lsb_release -i) $(lsb_release -r) - basebuild - YYYYMMDDTHHMMSS"

====

This patch makes it possible to perform the above multi-phase
processes by executing a single script:

  bash scripts/respin-jcloud-images.sh

This script sources a function library from the following file:

  vagrant/lib/respin-functions.sh

Defined in this library are (among others):

  # Fetch latest image from upstream distributor
  download_deb_image( $dist, $version, $arch )
  download_rh_image( $dist, $version, $arch )

  # Fetch latest image from upstream distributor
  create_deb_image( $dist, $version, $arch )
  create_rh_image( $dist, $version, $arch )

  # Run Vagrant provisioning on upstream image
  respin_deb_image( $dist, $version, $arch )
  respin_rh_image( $dist, $version, $arch )

The following script initializes vagrant and the following dynamic language environments:

  * python-virtualenv
  * pip
  * setuptools
  * python-{cinder,glance,keystone,neutron,nova,openstack}client

  * rbenv
  * Vagrant
  * vagrant-openstack-provider

  * nodejs
  * jq

  * local::lib
  * JSON::XS
  * DateTime
  * DateTime::Format::Duration
  * DateTime::Duration

  scripts/init-respin-env.sh

The following module compares ISO 8601 dates (as included in Image labels) with
the current time and prints to STDOUT the count of weeks, days, hours,
minutes and seconds in JSON:

  vagrant/lib/Respin.pm

Change-Id: I434cf2882e5e337ae4b55a4a7acb774a62b528b7
Signed-off-by: C.J. Collier <cjcollier@linuxfoundation.org>

5 files changed, 253 insertions, 103 deletions

diff --git a/scripts/init-respin-env.sh b/scripts/init-respin-env.sh
new file mode 100644
index 000000000..6ecfe8363
--- /dev/null
+++ b/scripts/init-respin-env.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+
+# Copyright 2016 The Linux Foundation
+
+set -e
+
+vagrant_version=1.8.1
+ruby_version=2.1.5
+ruby_patch=https://gist.github.com/mislav/055441129184a1512bb5.txt
+rbenv_git=https://github.com/rbenv/rbenv.git
+
+PVENAME="${CPPROJECT}-openstack"
+PVE_PATH="${PVE_ROOT}/${PVENAME}"
+PVERC=${PVE_PATH}/bin/activate
+PVE_BINDIR=$(dirname $PVERC)
+PVE_ROOT="${HOME}/src/python-virtual"
+
+LOCAL_LIB="${HOME}/src/local-lib"
+LL_LIBDIR="${LOCAL_LIB}/lib"
+
+RH_ARCH64=x86_64
+RH_ARCH32=i686
+DEB_ARCH64=amd64
+DEB_ARCH32=i386
+LV_IMG_DIR=/var/lib/libvirt/images/
+SRC_TIMESTAMP=""
+DST_TIMESTAMP=""
+
+init_virtualenv
+init_rbenv
+init_ruby
+select_ruby ${ruby_version}
+init_virtualenv
+init_vagrant
+install_vagrant_plugins
+import_vagrant_box
+init_local_lib
+init_javascript
+
+function init_virtualenv ()
+{
+    test -d ${PVE_BINDIR} && return 0
+
+    if [ -f /etc/debian_version ]
+    then
+        sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev
+    elif [ -f /etc/redhat-release ]
+    then
+        sudo yum -y install python-virtualenv
+    fi
+
+    mkdir -p ${PVE_PATH}
+    virtualenv ${PVE_PATH}
+
+    echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}"
+    echo "Press enter when finished"
+    read
+}
+
+function init_local_lib ()
+{
+    test -d ${LL_LIBDIR} && return 0
+
+    echo "local lib init incomplete"
+}
+
+function init_javascript ()
+{
+    which js && which jq && return 0
+
+    if [ -f /etc/debian_version ]
+    then
+        sudo apt-get -y -qq install nodejs jq
+    elif [ -f /etc/redhat-release ]
+    then
+        sudo yum -y install nodejs jq
+    fi
+}
+
+function init_vagrant ()
+{
+    which vagrant && return 0
+
+    vagrant_pkg_name=vagrant_${vagrant_version}_x86_64.deb
+    vagrant_pkg=https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_pkg_name}
+
+    wget -c ${vagrant_pkg}
+    sudo dpkg -i ${vagrant_pkg_name}
+}
+
+function init_rbenv ()
+{
+    which rbenv && return 0
+
+    # clone rbenv
+    test -d ~/.rbenv/.git || git clone ${rbenv_git} ~/.rbenv
+
+    # clone ruby-build
+    mkdir -p ~/.rbenv/plugins
+    test -d ~/.rbenv/plugins/ruby-build/.git || git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
+
+    # build ruby-build
+    cd ~/.rbenv && src/configure && make -C src
+
+    # Add rbenv to bashrc
+    grep HOME/.rbenv/bin ~/.bash_profile || echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile
+
+    # Add rbenv to current environment
+    export PATH="$HOME/.rbenv/bin:$PATH"
+}
+
+function init_ruby ()
+{
+    # Install ruby build deps
+    sudo apt-get build-dep ruby
+    #sudo apt-get -y install \
+    #     autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev \
+    #     zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev
+
+    # Build ruby
+    curl -fsSL ${ruby_patch} | rbenv install --patch ${ruby_version}
+}
+
+function select_ruby ()
+{
+    # Select ruby ${ruby_version} from rbenv
+    rbenv local ${1}
+    rbenv global ${1}
+}
+
+function install_vagrant_plugins ()
+{
+    for plugin in vagrant-openstack-provider vagrant-cachier vagrant-mutate
+    do
+        vagrant plugin install ${plugin}
+    done
+}
+
+function import_vagrant_box ()
+{
+    # Add dummy box
+    vagrant box add dummy https://github.com/huit/vagrant-openstack/blob/master/dummy.box
+
+    cp ${CI_MGMT}/vagrant/examples/box/dummy/Vagrantfile ~/.vagrant.d/boxes/dummy/0/openstack/
+}
+
diff --git a/scripts/respin-jcloud-images.sh b/scripts/respin-jcloud-images.sh
index 64a5c92f3..2af9e4c1b 100755
--- a/scripts/respin-jcloud-images.sh
+++ b/scripts/respin-jcloud-images.sh
@@ -1,25 +1,43 @@
 #!/bin/bash
-
+# Copyright 2016 The Linux Foundation
 set -e
 
+SUPPORTED_PLATFORMS=(
+  'Ubuntu 14.04 amd64'
+  'Ubuntu 16.04 amd64'
+  'CentOS 7 x86_64'
+)
+
 CI_MGMT=$(realpath $(dirname $(realpath $0))/..)
 
-# Copyright 2016 The Linux Foundation <cjcollier@linuxfoundation.org>
 source ${CI_MGMT}/vagrant/lib/respin-functions.sh
+source ${PVERC}
+
+echo nova: $(which nova)
+
+export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")}
+
+for PLATFORM in "${SUPPORTED_PLATFORMS[@]}"
+do
+    read -ra DVA <<< "${PLATFORM}"
+    DIST="${DVA[0]}"
+    VERSION="${DVA[1]}"
+    ARCH="${DVA[2]}"
+    DTYPE=$(dist_type ${DIST})
 
-source ${PVE_PATH}/bin/activate
+    AGE_JSON=$(latest_src_age ${DIST} ${VERSION} ${ARCH});
 
-# Acquire bootstrap images
-#download_deb_image 'Ubuntu' '14.04' 'amd64'
-#download_deb_image 'Ubuntu' '16.04' 'amd64'
-#download_rh_image 'CentOS' '7' 'x86_64'
+    # only fetch new base image if our latest one is more than two weeks old
+    if [ $(echo ${AGE_JSON} | jq .week) -ge "3" ]
+    then
+        # Acquire bootstrap images
+        download_${DTYPE}_image "${DIST}" "${VERSION}" "${ARCH}"
 
-# Push images to openstack via glance
-#create_deb_image 'Ubuntu' '14.04' 'amd64'
-#create_deb_image 'Ubuntu' '16.04' 'amd64'
-#create_rh_image 'CentOS' '7' 'x86_64'
+        # Push images to openstack via glance
+        create_${DTYPE}_image "${DIST}" "${VERSION}" "${ARCH}"
+    fi
 
-# Respin images
-respin_deb_image 'Ubuntu' '14.04' 'amd64'
-respin_deb_image 'Ubuntu' '16.04' 'amd64'
-respin_rh_image 'CentOS' '7' 'x86_64'
+    # Respin images
+    cd ${CI_MGMT}/vagrant/basebuild
+    respin_${DTYPE}_image "${DIST}" "${VERSION}" "${ARCH}"
+done
diff --git a/scripts/setup-vagrant.sh b/scripts/setup-vagrant.sh
deleted file mode 100644
index b7145d49a..000000000
--- a/scripts/setup-vagrant.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# fetch and install vagrant package
-vagrant_version=1.8.1
-vagrant_pkg_name=vagrant_${vagrant_version}_x86_64.deb
-vagrant_pkg=https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_pkg_name}
-wget -c $vagrant_pkg
-sudo dpkg -i $vagrant_pkg_name
-
-# clone rbenv
-test -d ~/.rbenv/.git || git clone https://github.com/rbenv/rbenv.git ~/.rbenv
-
-# clone ruby-build
-mkdir -p ~/.rbenv/plugins
-test -d ~/.rbenv/plugins/ruby-build/.git || git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
-
-# build ruby-build
-cd ~/.rbenv && src/configure && make -C src
-
-# Add rbenv to bashrc
-echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile
-
-# Add rbenv to current environment
-export PATH="$HOME/.rbenv/bin:$PATH"
-
-# Install ruby build deps
-sudo apt-get build-dep ruby2.3
-#sudo apt-get -y install \
-#     autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev \
-#     zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev
-
-# Build ruby 2.1.5
-curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.1.5
-
-# Select ruby 2.1.5 from rbenv
-rbenv local 2.1.5
-rbenv global 2.1.5
-
-# Add dummy box
-vagrant box add dummy https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box
-cp ${CIADM_DIR}/vagrant/examples/box/dummy/Vagrantfile ~/.vagrant.d/boxes/dummy/0/openstack/
diff --git a/vagrant/lib/Respin.pm b/vagrant/lib/Respin.pm
new file mode 100644
index 000000000..e72c601a1
--- /dev/null
+++ b/vagrant/lib/Respin.pm
@@ -0,0 +1,30 @@
+package Respin;
+
+use strict;
+use warnings;
+use DateTime;
+use DateTime::Format::Duration;
+use DateTime::Duration;
+use JSON::XS;
+
+my $iso8601_rx = qr{^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})$};
+
+my $json = JSON::XS->new->utf8;
+
+my $dur_fmt = DateTime::Format::Duration->new(
+    normalize => 1,
+    pattern =>
+      q{{"week":"%V","day":"%u","hour":"%k","minute":"%M","second":"%S"}}
+);
+
+sub latest_src_age {
+    my ( $now, $src ) = @_;
+
+    my ( %now, %src );
+    @now{qw(year month day hour minute second)} = ( $now =~ $iso8601_rx );
+    @src{qw(year month day hour minute second)} = ( $src =~ $iso8601_rx );
+
+    print $dur_fmt->format_duration_from_deltas(
+        DateTime->new(%now)->subtract_datetime_absolute( DateTime->new(%src) )
+          ->deltas );
+}
diff --git a/vagrant/lib/respin-functions.sh b/vagrant/lib/respin-functions.sh
index 9a9f9809c..7aa742c32 100644
--- a/vagrant/lib/respin-functions.sh
+++ b/vagrant/lib/respin-functions.sh
@@ -1,53 +1,37 @@
 #!/bin/bash
 
-# Copyright 2016 The Linux Foundation <cjcollier@linuxfoundation.org>
-
-PVE_ROOT="${HOME}/src/python-virtual"
-CPPROJECT=${CPPROJECT:-fdio}
-PVENAME="${CPPROJECT}-openstack"
-PVE_PATH="${PVE_ROOT}/${PVENAME}"
-PVERC=${PVE_PATH}/bin/activate
-SERVER_NAME=${SERVER_NAME:-${USER}-vagrant}
-
-STACK_PROVIDER=vexxhost
-STACK_PORTAL=secure.${STACK_PROVIDER}.com
-STACK_ID_SERVER=auth.${STACK_PROVIDER}.net
-
-export OPENSTACK_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/"
-export OPENSTACK_FLAVOR='v1-standard-4'
-export STACK_REGION_NAME='ca-ymq-1'
-export AVAILABILITY_ZONE='ca-ymq-2'
-export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")}
-
-if [ ! -d ${PVE_PATH} ]
-then
-    mkdir -p $(dirname $PVE_PATH)
-    if [ -f /etc/debian_version ]
-    then
-        sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev
-    elif [ -f /etc/redhat-release ]
-    then
-        sudo yum -y install python-virtualenv
-    fi
+# Copyright 2016 The Linux Foundation
 
-    python-virtualenv ${PVE_PATH}
+source ${CI_MGMT}/vagrant/lib/vagrant-functions.sh
 
-    echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}"
-fi
 
-RH_ARCH64=x86_64
-RH_ARCH32=i686
-DEB_ARCH64=amd64
-DEB_ARCH32=i386
-LV_IMG_DIR=/var/lib/libvirt/images/
-SRC_TIMESTAMP=""
-DST_TIMESTAMP=""
+source ${PVERC}
+pip install -q --upgrade pip setuptools python-{cinder,glance,keystone,neutron,nova,openstack}client
+
+#
+# usage:
+#   AGE_JSON=$(latest_src_age ${DIST} ${VERSION} ${ARCH})
+#
+function latest_src_age ()
+{
+    SRC_TS=$(latest_src_timestamp "$@")
+    NOW_TS=$(new_timestamp)
+
+    perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )'
+
+    return 0
+}
+
+function new_timestamp ()
+{
+    date +'%F T %T' | sed -e 's/[-: ]//g'
+}
 
 function new_dst_timestamp ()
 {
     if [ -z "${DST_TIMESTAMP}" ]
     then
-        DST_TIMESTAMP=$(date +'%F T %T' | sed -e 's/[-: ]//g')
+        DST_TIMESTAMP=$(new_timestamp)
     fi
 
     echo ${DST_TIMESTAMP}
@@ -101,7 +85,6 @@ function setup_rh ()
     IMG_NAME="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload"
 }
 
-
 #
 # usage:
 #   create_rh_image ${DIST} ${VERSION} ${ARCH}
@@ -122,7 +105,7 @@ function download_rh_image ()
 {
     setup_rh "$@"
     echo "--> Fetching image file for ${DIST} ${VERSION}"
-    wget -cP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}"
+    wget -qcP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}"
 }
 
 
@@ -234,7 +217,7 @@ function download_deb_image ()
 
     if [ -z "$URL" ]; then echo "Cannot fetch qcow2 image for ${DIST} v${MICRO_VERSION}"; return -3; fi
     echo "--> Fetching image file for ${DIST} ${VERSION}"
-    wget -cP ${LV_IMG_DIR} "${URL}"
+    wget -qcP ${LV_IMG_DIR} "${URL}"
 }
 
 # Used to upload
@@ -251,6 +234,7 @@ function create_deb_image ()
 
     if [ ! -f ${IMG_PATH} ]; then download_deb_image "$@"; fi
 
+    echo "--> Pushing image ${IMG_NAME}"
     glance_image_create "${IMG_NAME}" "${IMG_PATH}"
 }
 
@@ -261,7 +245,7 @@ function respin_deb_image ()
     setup_deb "$@"
     export IMAGE="${IMG_NAME}"
     echo "--> creating instance of image '${IMAGE}' as server name '${SERVER_NAME}'"
-    vagrant up
+    vagrant up --provider=openstack
     if [ "Ubuntu" == "${DIST}" ]
     then
         DST_IMAGE="${DIST} ${VERSION} LTS - basebuild - ${DST_TIMESTAMP}"
@@ -285,10 +269,25 @@ function respin_rh_image ()
     setup_rh "$@"
     IMAGE="${IMG_NAME}"
     echo "--> creating instance of image '${IMG_NAME}' as server name '${SERVER_NAME}'"
-    vagrant up
+    vagrant up --provider=openstack
     DST_IMAGE="${DIST} ${VERSION} - basebuild - ${DST_TIMESTAMP}"
     echo "--> Taking snapshot of image '${IMG_NAME}' with name '${DST_IMAGE}'"
     nova image-create --poll "${SERVER_NAME}" "${DST_IMAGE}"
     echo "--> Bringing down vagrant instance"
     vagrant destroy
 }
+
+function dist_type ()
+{
+  case "${1}" in
+      CentOS | RHEL | SuSE)
+          echo "rh" ;;
+      Debian | Ubuntu | Kali | ProxMox | VyOS)
+          echo "deb" ;;
+      *)
+          echo "Unrecognized distribution: ${1}"
+          exit 2 ;;
+  esac
+
+}
+