From 995d396e9310ded5827f5ee17c345b06234f68ca Mon Sep 17 00:00:00 2001 From: "C.J. Collier" Date: Wed, 25 May 2016 21:56:54 -0700 Subject: Simplify README * configurable options have been moved to environment variables * processes have been simplified * Added a script to bootstrap openstack vagrant environment * Added an example openstack dummy box Vagrantfile Change-Id: I5dd850e9456ad205fe3a655ca9e16a9bb14f32e2 Signed-off-by: C.J. Collier --- scripts/respin-jcloud-images.sh | 4 +- scripts/setup-vagrant.sh | 43 +++++++++ vagrant/basebuild/README.txt | 171 ++++++++++++++++++++++++--------- vagrant/examples/box/dummy/Vagrantfile | 39 ++++++++ vagrant/lib/respin-functions.sh | 16 ++- 5 files changed, 224 insertions(+), 49 deletions(-) create mode 100644 scripts/setup-vagrant.sh create mode 100644 vagrant/examples/box/dummy/Vagrantfile diff --git a/scripts/respin-jcloud-images.sh b/scripts/respin-jcloud-images.sh index 3d1195847..64a5c92f3 100755 --- a/scripts/respin-jcloud-images.sh +++ b/scripts/respin-jcloud-images.sh @@ -2,8 +2,10 @@ set -e +CI_MGMT=$(realpath $(dirname $(realpath $0))/..) + # Copyright 2016 The Linux Foundation -source ../vagrant/lib/respin-functions.sh +source ${CI_MGMT}/vagrant/lib/respin-functions.sh source ${PVE_PATH}/bin/activate diff --git a/scripts/setup-vagrant.sh b/scripts/setup-vagrant.sh new file mode 100644 index 000000000..b7145d49a --- /dev/null +++ b/scripts/setup-vagrant.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +# fetch and install vagrant package +vagrant_version=1.8.1 +vagrant_pkg_name=vagrant_${vagrant_version}_x86_64.deb +vagrant_pkg=https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_pkg_name} +wget -c $vagrant_pkg +sudo dpkg -i $vagrant_pkg_name + +# clone rbenv +test -d ~/.rbenv/.git || git clone https://github.com/rbenv/rbenv.git ~/.rbenv + +# clone ruby-build +mkdir -p ~/.rbenv/plugins +test -d ~/.rbenv/plugins/ruby-build/.git || git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build + +# build ruby-build +cd ~/.rbenv && src/configure && make -C src + +# Add rbenv to bashrc +echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile + +# Add rbenv to current environment +export PATH="$HOME/.rbenv/bin:$PATH" + +# Install ruby build deps +sudo apt-get build-dep ruby2.3 +#sudo apt-get -y install \ +# autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev \ +# zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev + +# Build ruby 2.1.5 +curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.1.5 + +# Select ruby 2.1.5 from rbenv +rbenv local 2.1.5 +rbenv global 2.1.5 + +# Add dummy box +vagrant box add dummy https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box +cp ${CIADM_DIR}/vagrant/examples/box/dummy/Vagrantfile ~/.vagrant.d/boxes/dummy/0/openstack/ diff --git a/vagrant/basebuild/README.txt b/vagrant/basebuild/README.txt index adeb0bb81..e4fd83a1c 100644 --- a/vagrant/basebuild/README.txt +++ b/vagrant/basebuild/README.txt @@ -1,14 +1,61 @@ Standard Operating Procedures: +=== Environmental specifics === + +For the use of this document, the following environment variables +should be assumed set: + + CPPROJECT=fdio + PVENAME=openstack-${CPPROJECT} + + LFID=cjcollier + CP_NAME=FD.io + CP_DOMAIN=fd.io + VAGRANT_DEFAULT_PROVIDER=openstack + RESEAL=1 + RUBY_VER=2.1.5 + LF_GIT=/usr/src/git/lf + PVEPATH=${HOME}/src/python-virtual + PVE_DIR=${PVEPATH}/${PVENAME} + SERVER_NAME=${LFID}-vagrant + + RH_ARCH_32=i686 + RH_ARCH_64=x86_64 + DEB_ARCH_64=amd64 + DEB_ARCH_32=i386 + + STACK_PROVIDER=vexxhost + STACK_PORTAL=secure.${STACK_PROVIDER}.com + STACK_ID_SERVER=auth.${STACK_PROVIDER}.net + STACK_REGION_NAME=ca-ymq-1 + + VAGRANT_DISTRIBUTOR=hashicorp + VDIST_DOMAIN=releases.${VAGRANT_DISTRIBUTOR}.com + # vagrant_${VDIST_VER}_${RH_ARCH_32}.deb + # vagrant_${VDIST_VER}_${RH_ARCH_64}.deb + VDIST_FILENAME=vagrant_${VDIST_VER}_${RH_ARCH}.deb + # Ask the web server what its latest version is + VDIST_VER=$(curl https://${VDIST_DOMAIN}/vagrant|html2text|awk -F_ '/_/ {print $2}'|sort|tail -1) + VDIST_PATH=vagrant/${VDIST_VER}/${VDIST_FILENAME} + VAGRANT_DISTFILE_URL=https://{$VDIST_DOMAIN}/${VDIST_PATH} + + GERRIT_HOSTNAME=gerrit.${CP_DOMAIN} + GERRIT_DIR=${LF_GIT}/${GERRIT_HOSTNAME} + CIADM_NAME=ci-management + CIADM_DIR=${GERRIT_DIR}/${CIADM_NAME} + RH_ARCH=${RH_ARCH_64} + DEB_ARCH=${DEB_ARCH_64} + PVERC=${PVE_DIR}/bin/activate + === Establish Cloudstack Credentials === To get the correct configuration for the environment you will need openstack credentials. Those used for this project are managed by -vexxhost. Establish credentials through appropriate channels. Once -credentials are acquired, visit this page: +${STACK_PROVIDER}. Establish credentials through appropriate +channels. Once credentials are acquired, visit this page: -https://secure.vexxhost.com/console/#/account/credentials +https://${STACK_PORTAL}/console/#/account/credentials The environment variables you need will be at the bottom of the page, formatted as such: @@ -16,72 +63,95 @@ formatted as such: export OS_TENANT_NAME="00000000-0000-0000-0000-000000000000" export OS_USERNAME="00000000-0000-0000-0000-000000000000" export OS_PASSWORD="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - export OS_AUTH_URL="https://auth.vexxhost.net/v2.0/" - export OS_REGION_NAME="ca-ymq-1" + export OS_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/" + export OS_REGION_NAME="${STACK_REGION_NAME}" Save these lines to /tmp/openstack-credentials.sh === Establish a python-virtual environment === -For Debian-based systems, be sure that you install these packages: -* libpython-dev -* python-virtualenv - - CPPROJECT=fdio +For Debian-based systems, be sure that you install the +virtualenvwrapper packages - export PVENAME=openstack-${CPPROJECT} - export PVEPATH=/usr/src/git/lf/git.lf.org/cjcollier/python-virtual - export PVERC=${PVEPATH}/${PVENAME}/bin/activate + # Debian: sudo apt-get install virtualenvwrapper mkdir -p ${PVEPATH} cd ${PVEPATH} virtualenv ${PVENAME} cat /tmp/openstack-credentials.sh >> ${PVERC} - source ${PVERC} + source ${PVERC} pip install --upgrade pip setuptools pip install python-{cinder,glance,keystone,neutron,nova,openstack}client === Install your public key into the cloudstack environment === -https://secure.vexxhost.com/console/v2.html#/compute/keypairs +Via the web interface: https://${STACK_PORTAL}/console/v2.html#/compute/keypairs -This will install your yubikey-backed ssh key: +This command will install your yubikey-backed ssh key: - ssh-add -L | grep cardno | nova keypair-add --pub-key - $USER + ssh-add -L | grep cardno | nova keypair-add --pub-key - ${LFID} === Establish the GUID of our network === -Via the web interface: https://secure.vexxhost.com/console/#/networking/networks +Via the web interface: https://${STACK_PORTAL}/console/#/networking/networks Command line: - NETID=$(nova network-list | awk '/${CPPROJECT}/ {print $2}') + export NETID=$(nova network-list | awk "/${CPPROJECT}/ {print \$2}") + grep -v '^NETID=' ${PVERC} | cat - > ${PVERC} echo "NETID=${NETID}" >> ${PVERC} +=== Install rbenv === + +==== Red Hat ==== + + sudo yum install -y \ + git-core zlib zlib-devel gcc-c++ patch readline readline-devel \ + libyaml-devel libffi-devel openssl-devel make bzip2 autoconf \ + automake libtool bison curl sqlite-devel + git clone git://github.com/sstephenson/rbenv.git ~/.rbenv + eval $(echo 'export PATH="${HOME}/.rbenv/bin:${PATH}"' | tee -a ~/.bashrc) + eval $(rbenv init -) + time rbenv install ${RUBY_VER} + +==== Debian ==== + + sudo apt-get build-dep ruby + sudo apt-get install rbenv ruby-build libssl-dev libreadline-dev + eval $(rbenv init -) + curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt > /tmp/debian-ssl-patch + time rbenv install --patch ${RUBY_VER} < /tmp/debian-ssl-patch # 7m42.708s + +==== Common ==== + + rbenv local ${RUBY_VER} + rbenv global ${RUBY_VER} + grep 'rbenv init' ~/.bashrc || \ + echo 'eval "$(rbenv init -)"' >> ~/.bashrc && \ + echo "rbenv local ${RUBY_VER}" >> ~/.bashrc && \ + echo "rbenv global ${RUBY_VER}" >> ~/.bashrc + === Install Vagrant, vagrant-openstack-provider === -On Debian, you must uninstall the system vagrant and instead install the -upstream package: +On Debian, you must uninstall the system vagrant and instead install +the upstream package: - # https://releases.hashicorp.com/vagrant/1.8.1/vagrant_1.8.1_i686.deb - wget https://releases.hashicorp.com/vagrant/1.8.1/vagrant_1.8.1_${RH_ARCH}.deb - sudo dpkg -i vagrant_1.8.1_${RH_ARCH}.deb + wget -O ${VAGRANT_DISTFILE_NAME} ${VAGRANT_DISTFILE_URL} + sudo dpkg -i ${VAGRANT_DISTFILE_NAME} vagrant plugin install vagrant-openstack-provider === Configure openstack 'dummy' box === CLI: vagrant box add dummy https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box - wget -P ~/.vagrant.d/boxes/dummy/0/openstack/ https://raw.githubusercontent.com/cjac/scripts-n-things/master/vagrant/Vagrantfile - vi ~/.vagrant.d/boxes/dummy/0/openstack/Vagrantfile + cp ${CIADM_DIR}/vagrant/examples/box/dummy/Vagrantfile ~/.vagrant.d/boxes/dummy/0/openstack/ -=== Check out the ci-management branch from fdio gerrit === +=== Check out the ci-management repo from gerrit === CLI: - FDIO_GDIR=/usr/src/git/lf/gerrit.fd.io - mkdir -p ${FDIO_GDIR} - cd ${FDIO_GDIR} - git clone ssh://gerrit.fd.io:29418/ci-management + mkdir -p ${GERRIT_DIR} + cd ${GERRIT_DIR} + git clone ssh://${GERRIT_HOSTNAME}:29418/${CIADM_NAME} === Acquire base images === @@ -95,17 +165,20 @@ Gentoo: http://linuximages.de/openstack/gentoo/ CLI: - source ${FDIO_GDIR}/vagrant/lib/respin-functions.sh + source ${CIADM_DIR}/vagrant/lib/respin-functions.sh download_deb_image 'Ubuntu' '14.04' 'amd64' download_deb_image 'Ubuntu' '16.04' 'amd64' + download_deb_image 'Debian' 'stable' 'amd64' + download_deb_image 'Debian' 'testing' 'amd64' + download_deb_image 'Debian' 'unstable' 'amd64' download_rh_image 'CentOS' '7' 'x86_64' === Upload base images === CLI: - source ${FDIO_GDIR}/vagrant/lib/respin-functions.sh + source ${CIADM_DIR}/vagrant/lib/respin-functions.sh create_deb_image 'Ubuntu' '14.04' 'amd64' create_deb_image 'Ubuntu' '16.04' 'amd64' @@ -116,21 +189,29 @@ CLI: CLI: - export VAGRANT_DEFAULT_PROVIDER=openstack - export SERVER_NAME=kf7bmp-vagrant - export RESEAL=1 - cd $FDIO_GDIR/ci-management/vagrant/basebuild + source ${PVERC} - source ${FDIO_GDIR}/vagrant/lib/respin-functions.sh + # Respin images respin_deb_image 'Ubuntu' '14.04' 'amd64' respin_deb_image 'Ubuntu' '16.04' 'amd64' - respin_rh_image 'CentOS' '7' 'x86_64' + respin_rh_image 'CentOS' '7' 'x86_64' + +=== Manual bootstrap of Ubuntu 14.04 === + +CLI: + source ${CIADM_DIR}/vagrant/lib/respin-functions.sh + SRC_TIMESTAMP=$(latest_src_timestamp) + + DIST='Ubuntu' + VERSION='14.04' + IMAGE="${DIST} ${VERSION} (${SRC_TIMESTAMP}) - LF upload" + RESEAL=1 + cd ${CIADM_DIR}/vagrant/basebuild + vagrant up -=== Verify images === +On manual bootstrap failure, one can connect to the VM using + vagrant ssh -* Update *-staging images at https://jenkins.fd.io/configure -* Submit a probe patch to vpp (as DO NOT MERGE) and comment 'verify-images' to check the new images against vpp -* Submit a probe patch to honeycomb (as DO NOT MERGE) and comment 'verify-images' to check the new images against honeycomb -* If and only if both probe patches pass verify-images, update remaining images at https://jenkins.fd.io/configure -* Abandon the vpp and honeycomb probe patches +On success + nova image-create --poll ${SERVER_NAME} "qq{$dist $version - basebuild - $isodate};" diff --git a/vagrant/examples/box/dummy/Vagrantfile b/vagrant/examples/box/dummy/Vagrantfile new file mode 100644 index 000000000..fc53530c5 --- /dev/null +++ b/vagrant/examples/box/dummy/Vagrantfile @@ -0,0 +1,39 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby ts=2 sw=2 sts=2 et : + +# See ci-management/vagrant/lib/respin-functions.sh for examples of +# environment variable settings + +require 'vagrant-openstack-provider' + +Vagrant.configure("2") do |config| + + config.vm.provider :openstack do |os, override| + + # CPPROJECT=fdio + # STACK_PROVIDER=vexxhost + # STACK_PORTAL=secure.${STACK_PROVIDER}.com + # STACK_ID_SERVER=auth.${STACK_PROVIDER}.net + + # OPENSTACK_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/" + # OPENSTACK_FLAVOR='v1-standard-4' + # STACK_REGION_NAME='ca-ymq-1' + # AVAILABILITY_ZONE='ca-ymq-2' + + os.openstack_auth_url = ENV['OPENSTACK_AUTH_URL'] + os.flavor = ENV['OPENSTACK_FLAVOR'] + os.region = ENV['STACK_REGION_NAME'] + os.availability_zone = ENV['AVAILABILITY_ZONE'] + + # https://${STACK_PORTAL}/console/#/account/credentials + os.tenant_name = ENV['OS_TENANT_NAME'] + os.username = ENV['OS_USERNAME'] + os.password = ENV['OS_PASSWORD'] + + # NETID=$(nova network-list | awk '/${CPPROJECT}/ {print $2}') + os.networks = ENV['NETID'] + + # personal default instance names: for example, ${USER}-vagrant + os.server_name = ENV['SERVER_NAME'] + end +end diff --git a/vagrant/lib/respin-functions.sh b/vagrant/lib/respin-functions.sh index e4fdb2a31..9a9f9809c 100644 --- a/vagrant/lib/respin-functions.sh +++ b/vagrant/lib/respin-functions.sh @@ -2,19 +2,29 @@ # Copyright 2016 The Linux Foundation -PVE_ROOT=/usr/src/git/lf/git.lf.org/cjcollier/python-virtual +PVE_ROOT="${HOME}/src/python-virtual" CPPROJECT=${CPPROJECT:-fdio} PVENAME="${CPPROJECT}-openstack" PVE_PATH="${PVE_ROOT}/${PVENAME}" PVERC=${PVE_PATH}/bin/activate SERVER_NAME=${SERVER_NAME:-${USER}-vagrant} +STACK_PROVIDER=vexxhost +STACK_PORTAL=secure.${STACK_PROVIDER}.com +STACK_ID_SERVER=auth.${STACK_PROVIDER}.net + +export OPENSTACK_AUTH_URL="https://${STACK_ID_SERVER}/v2.0/" +export OPENSTACK_FLAVOR='v1-standard-4' +export STACK_REGION_NAME='ca-ymq-1' +export AVAILABILITY_ZONE='ca-ymq-2' +export NETID=${NETID:-$(nova network-list | awk "/${CPPROJECT}/ {print \$2}")} + if [ ! -d ${PVE_PATH} ] then mkdir -p $(dirname $PVE_PATH) if [ -f /etc/debian_version ] then - sudo apt-get -y -qq install python-virtualenv libpython-dev + sudo apt-get -y -qq install virtualenvwrapper python-virtualenv libpython-dev elif [ -f /etc/redhat-release ] then sudo yum -y install python-virtualenv @@ -22,7 +32,7 @@ then python-virtualenv ${PVE_PATH} - echo "Please visit https://secure.vexxhost.com/console/#/account/credentials and place all $OS_* variables at the end of ${PVERC}" + echo "Please copy all OS_* variables from https://secure.vexxhost.com/console/#/account/credentials to the end of ${PVERC}" fi RH_ARCH64=x86_64 -- cgit 1.2.3-korg