Improve respin scripts

In order to create a new OpenStack basebuild Images, a multi-phase
process must be executed.

The first phase is to configure the system for use with dynamic
language environments.

Due to security precautions, We do not currently have the ability to
automate the process of respinning a new Image.  We have a goal of
fully automating the process so that a new Image will be generated on
every commit.  Until the process is self-hosted, we must manually
fetch credentials from the OpenStack provider's web UI in order to
make use of the python openstack bindings and to complete the
basebuild re-spin process using Vagrant.

The more generic portions of the Image are imported once every two to
three weeks from upstream distributors.  This duration is computed
thus:

  $(perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )') | jq .week

Once fetched, these qcow2 Images are pushed to OpenStack using glance
and labelled as:

  "$(lsb_release -i) $(lsb_release -r) (YYYYMMDDTHHMMSS) - LF upload"

The generated basebuild Images are pushed to OpenStack with glance
labelled as:

  "$(lsb_release -i) $(lsb_release -r) - basebuild - YYYYMMDDTHHMMSS"

====

This patch makes it possible to perform the above multi-phase
processes by executing a single script:

  bash scripts/respin-jcloud-images.sh

This script sources a function library from the following file:

  vagrant/lib/respin-functions.sh

Defined in this library are (among others):

  # Fetch latest image from upstream distributor
  download_deb_image( $dist, $version, $arch )
  download_rh_image( $dist, $version, $arch )

  # Fetch latest image from upstream distributor
  create_deb_image( $dist, $version, $arch )
  create_rh_image( $dist, $version, $arch )

  # Run Vagrant provisioning on upstream image
  respin_deb_image( $dist, $version, $arch )
  respin_rh_image( $dist, $version, $arch )

The following script initializes vagrant and the following dynamic language environments:

  * python-virtualenv
  * pip
  * setuptools
  * python-{cinder,glance,keystone,neutron,nova,openstack}client

  * rbenv
  * Vagrant
  * vagrant-openstack-provider

  * nodejs
  * jq

  * local::lib
  * JSON::XS
  * DateTime
  * DateTime::Format::Duration
  * DateTime::Duration

  scripts/init-respin-env.sh

The following module compares ISO 8601 dates (as included in Image labels) with
the current time and prints to STDOUT the count of weeks, days, hours,
minutes and seconds in JSON:

  vagrant/lib/Respin.pm

Change-Id: I434cf2882e5e337ae4b55a4a7acb774a62b528b7
Signed-off-by: C.J. Collier <cjcollier@linuxfoundation.org>

2 files changed, 74 insertions, 45 deletions

diff --git a/vagrant/lib/Respin.pm b/vagrant/lib/Respin.pm
new file mode 100644
index 000000000..e72c601a1
--- /dev/null
+++ b/vagrant/lib/Respin.pm
@@ -0,0 +1,30 @@
+package Respin;
+
+use strict;
+use warnings;
+use DateTime;
+use DateTime::Format::Duration;
+use DateTime::Duration;
+use JSON::XS;
+
+my $iso8601_rx = qr{^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})$};
+
+my $json = JSON::XS->new->utf8;
+
+my $dur_fmt = DateTime::Format::Duration->new(
+    normalize => 1,
+    pattern =>
+      q{{"week":"%V","day":"%u","hour":"%k","minute":"%M","second":"%S"}}
+);
+
+sub latest_src_age {
+    my ( $now, $src ) = @_;
+
+    my ( %now, %src );
+    @now{qw(year month day hour minute second)} = ( $now =~ $iso8601_rx );
+    @src{qw(year month day hour minute second)} = ( $src =~ $iso8601_rx );
+
+    print $dur_fmt->format_duration_from_deltas(
+        DateTime->new(%now)->subtract_datetime_absolute( DateTime->new(%src) )
+          ->deltas );
+}
diff --git a/vagrant/lib/respin-functions.sh b/vagrant/lib/respin-functions.sh
index 9a9f9809c..7aa742c32 100644
--- a/vagrant/lib/respin-functions.sh
+++ b/vagrant/lib/respin-functions.sh
@@ -1,53 +1,37 @@
 #!/bin/bash
 
-# Copyright 2016 The Linux Foundation <cjcollier@linuxfoundation.org>
-
-PVE_ROOT="${HOME}/src/python-virtual"
-CPPROJECT=${CPPROJECT:-fdio}
-PVENAME="${CPPROJECT}-openstack"
-PVE_PATH="${PVE_ROOT}/${PVENAME}"
-PVERC=${PVE_PATH}/bin/activate
-SERVER_NAME=${SERVER_NAME:-${USER}-vagrant}
-
-STACK_PROVIDER=vexxhost
-STACK_PORTAL=secure.${STACK_PROVIDER}.com
-STACK_ID_SERVER=auth.${STACK_PROVIDER}.net
-
-export OPENSTACK_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/"
-export OPENSTACK_FLAVOR='v1-standard-4'
-export STACK_REGION_NAME='ca-ymq-1'
-export AVAILABILITY_ZONE='ca-ymq-2'
-export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")}
-
-if [ ! -d ${PVE_PATH} ]
-then
-    mkdir -p $(dirname $PVE_PATH)
-    if [ -f /etc/debian_version ]
-    then
-        sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev
-    elif [ -f /etc/redhat-release ]
-    then
-        sudo yum -y install python-virtualenv
-    fi
+# Copyright 2016 The Linux Foundation
 
-    python-virtualenv ${PVE_PATH}
+source ${CI_MGMT}/vagrant/lib/vagrant-functions.sh
 
-    echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}"
-fi
 
-RH_ARCH64=x86_64
-RH_ARCH32=i686
-DEB_ARCH64=amd64
-DEB_ARCH32=i386
-LV_IMG_DIR=/var/lib/libvirt/images/
-SRC_TIMESTAMP=""
-DST_TIMESTAMP=""
+source ${PVERC}
+pip install -q --upgrade pip setuptools python-{cinder,glance,keystone,neutron,nova,openstack}client
+
+#
+# usage:
+#   AGE_JSON=$(latest_src_age ${DIST} ${VERSION} ${ARCH})
+#
+function latest_src_age ()
+{
+    SRC_TS=$(latest_src_timestamp "$@")
+    NOW_TS=$(new_timestamp)
+
+    perl -I${CI_MGMT}/vagrant/lib -MRespin -e 'Respin::latest_src_age( "${NOW_TS}", "${SRC_TS}" )'
+
+    return 0
+}
+
+function new_timestamp ()
+{
+    date +'%F T %T' | sed -e 's/[-: ]//g'
+}
 
 function new_dst_timestamp ()
 {
     if [ -z "${DST_TIMESTAMP}" ]
     then
-        DST_TIMESTAMP=$(date +'%F T %T' | sed -e 's/[-: ]//g')
+        DST_TIMESTAMP=$(new_timestamp)
     fi
 
     echo ${DST_TIMESTAMP}
@@ -101,7 +85,6 @@ function setup_rh ()
     IMG_NAME="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload"
 }
 
-
 #
 # usage:
 #   create_rh_image ${DIST} ${VERSION} ${ARCH}
@@ -122,7 +105,7 @@ function download_rh_image ()
 {
     setup_rh "$@"
     echo "--> Fetching image file for ${DIST} ${VERSION}"
-    wget -cP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}"
+    wget -qcP ${LV_IMG_DIR} "http://cloud.centos.org/centos/${VERSION}/images/${IMG_FNAME}"
 }
 
 
@@ -234,7 +217,7 @@ function download_deb_image ()
 
     if [ -z "$URL" ]; then echo "Cannot fetch qcow2 image for ${DIST} v${MICRO_VERSION}"; return -3; fi
     echo "--> Fetching image file for ${DIST} ${VERSION}"
-    wget -cP ${LV_IMG_DIR} "${URL}"
+    wget -qcP ${LV_IMG_DIR} "${URL}"
 }
 
 # Used to upload
@@ -251,6 +234,7 @@ function create_deb_image ()
 
     if [ ! -f ${IMG_PATH} ]; then download_deb_image "$@"; fi
 
+    echo "--> Pushing image ${IMG_NAME}"
     glance_image_create "${IMG_NAME}" "${IMG_PATH}"
 }
 
@@ -261,7 +245,7 @@ function respin_deb_image ()
     setup_deb "$@"
     export IMAGE="${IMG_NAME}"
     echo "--> creating instance of image '${IMAGE}' as server name '${SERVER_NAME}'"
-    vagrant up
+    vagrant up --provider=openstack
     if [ "Ubuntu" == "${DIST}" ]
     then
         DST_IMAGE="${DIST} ${VERSION} LTS - basebuild - ${DST_TIMESTAMP}"
@@ -285,10 +269,25 @@ function respin_rh_image ()
     setup_rh "$@"
     IMAGE="${IMG_NAME}"
     echo "--> creating instance of image '${IMG_NAME}' as server name '${SERVER_NAME}'"
-    vagrant up
+    vagrant up --provider=openstack
     DST_IMAGE="${DIST} ${VERSION} - basebuild - ${DST_TIMESTAMP}"
     echo "--> Taking snapshot of image '${IMG_NAME}' with name '${DST_IMAGE}'"
     nova image-create --poll "${SERVER_NAME}" "${DST_IMAGE}"
     echo "--> Bringing down vagrant instance"
     vagrant destroy
 }
+
+function dist_type ()
+{
+  case "${1}" in
+      CentOS | RHEL | SuSE)
+          echo "rh" ;;
+      Debian | Ubuntu | Kali | ProxMox | VyOS)
+          echo "deb" ;;
+      *)
+          echo "Unrecognized distribution: ${1}"
+          exit 2 ;;
+  esac
+
+}
+