From ed1bb590d38c1581d35fd822564510ff22288510 Mon Sep 17 00:00:00 2001 From: Dave Wallace Date: Fri, 5 Feb 2016 01:00:27 -0500 Subject: Refactor vagrant files for reliability and maintenance. Add git-review to VM. Change-Id: I7ed6c9d00037bb9665019f9f96e30ba9c2546525 Signed-off-by: Dave Wallace --- vagrant/Vagrantfile | 89 +++++++++++--- vagrant/bootstrap.ubuntu1404.sh | 257 +++++++++++++++++++++++++--------------- 2 files changed, 235 insertions(+), 111 deletions(-) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 5a3203933..7b72c87ce 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -5,14 +5,80 @@ Vagrant.configure(2) do |config| config.ssh.forward_x11 = true + if ENV.key?('VAGRANT_VM_NAME') + vagrant_vm_name = ENV['VAGRANT_VM_NAME'] + else + vagrant_vm_name = "" + end + + if ENV.key?('KARAF_PACKAGES') + config.vm.synced_folder ENV['KARAF_PACKAGES'], "/karaf-packages", disabled: false + end + + # If specified, add a private network interface for the ODL VBD app + # to communicate with VPP Honeycomb agents. VPP is not installed + # in an ODL VDB application VM. + if ENV.key?('VAGRANT_VBD_VM') + vagrant_vbd_vm = "is_vbd_vm" + if ENV.key?('VAGRANT_VBD_ADDR') + config.vm.network "private_network", ip: ENV['VAGRANT_VBD_ADDR'] + end + vagrant_vpp_agent_addr = "" + + else + vagrant_vbd_vm = "is_vpp_agent_vm" + + if ENV.key?('VAGRANT_VPP_AGENT_ADDR') + vagrant_vpp_agent_addr = ENV['VAGRANT_VPP_AGENT_ADDR'] + config.vm.network "private_network", ip: "#{vagrant_vpp_agent_addr}" + else + vagrant_vpp_agent_addr = "" + end + + # Define some physical ports for your VMs to be used by DPDK + nics = 0 + if ENV.key?('VPP_VAGRANT_NICS') + nics = ENV['VPP_VAGRANT_NICS'].to_i(10) + end + for i in 1..nics + config.vm.network "private_network", type: "dhcp" + end + # Mount VPP repository if specified + if ENV.key?('VPP_REPO') + config.vm.synced_folder ENV['VPP_REPO'], "/vpp", disabled: false + end + end + # Pick the right distro and bootstrap, default is ubuntu1404 distro = ENV['HONEYCOMB_VAGRANT_DISTRO'] if distro == 'centos7' config.vm.box = "puppetlabs/centos-7.0-64-nocm" - config.vm.provision 'shell', path: 'bootstrap.centos7.sh' + config.vm.provision 'shell' do |s| + s.path = "bootstrap.centos7.sh" + s.args = ["#{vagrant_vm_name}", "#{vagrant_vbd_vm}", "#{vagrant_vpp_agent_addr}"] + end else config.vm.box = "puppetlabs/ubuntu-14.04-64-nocm" - config.vm.provision 'shell', path: 'bootstrap.ubuntu1404.sh' + config.vm.provision 'shell' do |s| + s.path = "bootstrap.ubuntu1404.sh" + s.args = ["#{vagrant_vm_name}", "#{vagrant_vbd_vm}", "#{vagrant_vpp_agent_addr}"] + end + end + + # Add .gnupg dir in so folks can sign patches + # Note, as gnupg puts socket files in that dir, we have + # to be cautious and make sure we are dealing with a plain file + homedir = File.expand_path("~/") + Dir["#{homedir}/.gnupg/**/*"].each do |fname| + if File.file?(fname) + destname = fname.sub(Regexp.escape("#{homedir}/"),'') + config.vm.provision "file", source: fname, destination: destname + end + end + + # Copy in the .gitconfig if it exists + if File.file?(File.expand_path("~/.gitconfig")) + config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig" end # vagrant-cachier caches apt/yum etc to speed subsequent @@ -24,33 +90,22 @@ Vagrant.configure(2) do |config| config.cache.scope = :box end - # Define some physical ports for your VMs to be used by DPDK - nics = 0 - if ENV.key?('VPP_VAGRANT_NICS') - nics = ENV['VPP_VAGRANT_NICS'].to_i(10) - end - for i in 1..nics - config.vm.network "private_network", type: "dhcp" - end - - # use http proxy if avaiable + # use http proxy if available if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf") config.proxy.http = "$http_proxy" config.proxy.https = "$https_proxy" config.proxy.no_proxy = "localhost,127.0.0.1" end - # Mount VPP repository if specified - if ENV.key?('VPP_REPO') - config.vm.synced_folder ENV['VPP_REPO'], "/vpp", disabled: false - end - # Mount Maven repository if specified if ENV.key?('HONEYCOMB_M2_REPO') config.vm.synced_folder ENV['HONEYCOMB_M2_REPO'], "/m2-repository", disabled: false end + # Mount Honeycomb repository config.vm.synced_folder "../", "/honeycomb", disabled: false + + # Memory/cpu config config.vm.provider "virtualbox" do |vb| vb.memory = "4096" end diff --git a/vagrant/bootstrap.ubuntu1404.sh b/vagrant/bootstrap.ubuntu1404.sh index da51e48df..630193892 100644 --- a/vagrant/bootstrap.ubuntu1404.sh +++ b/vagrant/bootstrap.ubuntu1404.sh @@ -1,128 +1,197 @@ -# set default route to be the GATEWAY -declare -r GATEWAY=${1:-} -# set default gateway for bridged interface, and make it happen at reboot -if [[ -n "${GATEWAY}" ]]; then - echo "Setting default gateway through bridged interface" - ip route delete default 2>&1 >/dev/null || true - ip route add default via ${GATEWAY} - echo "GATEWAY=${GATEWAY}" >/etc/default/vagrant-bridge - [ -r /vagrant/vagrant-bridge.conf ] && cp /vagrant/vagrant-bridge.conf /etc/init/ # symlinks don't work - initctl reload-configuration +# Args +VAGRANT_VM_NAME="$1" +VAGRANT_VBD_VM="$2" +VAGRANT_VPP_AGENT_ADDR="$3" +echo "Running bootstrap.ubuntu1404.sh..." +echo "VAGRANT_VM_NAME = '$VAGRANT_VM_NAME'" +echo "VAGRANT_VBD_VM = '$VAGRANT_VBD_VM'" +echo "VAGRANT_VPP_AGENT_ADDR = '$VAGRANT_VPP_AGENT_ADDR'" + +# Directory and file definitions +HONEYCOMB_MOUNT="/honeycomb" +KARAF_PACKAGES_MOUNT="/karaf-packages" +M2_SETTINGS_XML="$HONEYCOMB_MOUNT/vagrant/settings.xml" +M2_MOUNT="/m2-repository" +VPP_MOUNT="/vpp" +VPP_BUILD_ROOT="$VPP_MOUNT/build-root" +VAGRANT_HOME="/home/vagrant" +VAGRANT_BASHRC="$VAGRANT_HOME/.bashrc" +VAGRANT_BASH_ALIASES="$VAGRANT_HOME/.bash_aliases" +VAGRANT_M2_DIR="$VAGRANT_HOME/.m2" +VAGRANT_M2_REPOSITORY="$VAGRANT_M2_DIR/repository" +HONEYCOMB_INSTALL_DIR="/opt/honeycomb" +V3PO_TARGET_DIR="$HONEYCOMB_MOUNT/v3po/karaf/target" +V3PO_TARBALL="$KARAF_PACKAGES_MOUNT/v3po-karaf*.tar.gz" +[ ! -f $V3PO_TARBALL ] && V3PO_TARBALL="$V3PO_TARGET_DIR/v3po-karaf*.tar.gz" +VBD_TARGET_DIR="$HONEYCOMB_MOUNT/vbd/karaf/target" +VBD_TARBALL="$KARAF_PACKAGES_MOUNT/vbd-karaf*.tar.gz" +[ ! -f $VBD_TARBALL ] && VBD_TARBALL="$VBD_TARGET_DIR/vbd-karaf*.tar.gz" + +# Don't install VPP if this is an ODL VBD application VM +if [ "$VAGRANT_VBD_VM" != "is_vbd_vm" ] ; then + echo "Configuring hugepages for VPP" + # Setup for hugepages using upstart so it persists across reboots + echo "vm.nr_hugepages=1024" >> /etc/sysctl.d/20-hugepages.conf + sysctl --system + cat << EOF > /etc/init/hugepages.conf +start on runlevel [2345] + +task + +script + mkdir -p /run/hugepages/kvm || true + rm -f /run/hugepages/kvm/* || true + rm -f /dev/shm/* || true + mount -t hugetlbfs nodev /run/hugepages/kvm +end script +EOF + # Allocate hugepages.conf right now. Verify that + # all hugepages have been allocated. If the VM + # runs out of resources, the messages below will + # identify the issue. Front and Center! + nr_hugepages=$(cat /proc/sys/vm/nr_hugepages) + echo + while [ $nr_hugepages != 1024 ] ; do + echo -n "Allocating hugepages... " + start hugepages + nr_hugepages=$(cat /proc/sys/vm/nr_hugepages) + echo "nr_hugepages = $nr_hugepages" + done +fi + +# Set prompt to include VM name if provided. +sudo -H -u vagrant perl -i -pe 's/@\\h/@\\\h\$VM_NAME/g' $VAGRANT_BASHRC +sudo -H -u vagrant touch $VAGRANT_BASH_ALIASES +if [ "$VAGRANT_VM_NAME" != "" ] && [ "$(grep VM_NAME $VAGRANT_BASH_ALIASES)" = "" ] ; then + echo -e "\n# Include VM Name in prompt" >> $VAGRANT_BASH_ALIASES + echo "export VM_NAME=\"($VAGRANT_VM_NAME)\"" >> $VAGRANT_BASH_ALIASES fi # Fix grub-pc on Virtualbox with Ubuntu export DEBIAN_FRONTEND=noninteractive +# Add fd.io apt repo in case its needed +echo "deb http://nexus.fd.io/content/repositories/fd.io.dev/ ./" > /etc/apt/sources.list.d/99fd.io.list + # Standard update + upgrade dance apt-get update apt-get upgrade -y -# Fix the silly notion that /bin/sh should point to dash by pointing it to bash - -sudo update-alternatives --install /bin/sh sh /bin/bash 100 - # Install build tools -apt-get install -y build-essential autoconf automake bison libssl-dev ccache libtool git dkms debhelper emacs libganglia1-dev libapr1-dev libconfuse-dev - -# Install other stuff -# apt-get install -y qemu-kvm libvirt-bin ubuntu-vm-builder bridge-utils +apt-get install -y build-essential autoconf automake bison libssl-dev ccache libtool git dkms debhelper emacs libganglia1-dev libapr1-dev libconfuse-dev git-review # Install uio apt-get install -y linux-image-extra-`uname -r` # Install jdk and maven apt-get install -y openjdk-7-jdk -mkdir /usr/local/apache-maven +mkdir -p /usr/local/apache-maven cd /usr/local/apache-maven wget http://apache.go-parts.com/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz tar -xzvf apache-maven-3.3.9-bin.tar.gz -C /usr/local/apache-maven/ update-alternatives --install /usr/bin/mvn mvn /usr/local/apache-maven/apache-maven-3.3.9/bin/mvn 1 update-alternatives --config mvn -cd /home/vagrant -sudo -H -u vagrant mkdir .m2 -sudo -H -u vagrant cp /honeycomb/vagrant/settings.xml .m2 -cat << EOF > .bash_aliases +# Set up Maven +sudo -H -u vagrant mkdir -p $VAGRANT_M2_DIR +sudo -H -u vagrant cp $M2_SETTINGS_XML $VAGRANT_M2_DIR +if [ "$(grep M2_HOME $VAGRANT_BASH_ALIASES)" = "" ] ; then + cat << EOF >> $VAGRANT_BASH_ALIASES + +# Maven Environment variables export M2_HOME=/usr/local/apache-maven/apache-maven-3.3.9 export MAVEN_OPTS="-Xms256m -Xmx512m" # Very important to put the "m" on the end export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 EOF -chown vagrant:vagrant .bash_aliases + chown vagrant:vagrant $VAGRANT_BASH_ALIASES +fi -# Use the external Maven M2 repository if it has been mounted on /m2-repository +# Use the external Maven M2 repository as a seed if available. [ -d /m2-repository ] && sudo -u vagrant sed -i -e 's,/home/vagrant/.m2/repository,/m2-repository,g' /home/vagrant/.m2/settings.xml -# Setup for hugepages using upstart so it persists across reboots -echo "vm.nr_hugepages=1024" >> /etc/sysctl.d/20-hugepages.conf -sysctl --system - -cat << EOF > /etc/init/hugepages.conf -start on runlevel [2345] - -task - -script - mkdir -p /run/hugepages/kvm || true - rm -f /run/hugepages/kvm/* || true - rm -f /dev/shm/* || true - mount -t hugetlbfs nodev /run/hugepages/kvm -end script -EOF - -# Make sure we run that hugepages.conf right now -start hugepages - -# Setup the vpp code -cd ~vagrant/ - -sudo -u vagrant mkdir git -cd git/ - -# Check if git exists and remove it before attempting clone, else clone ineffective when "reload --provision" -[ -d honeycomb ] && rm -rf honeycomb -sudo -H -u vagrant git clone /honeycomb -cd honeycomb/ - -# Initial honeycomb build -sudo -H -u vagrant mvn clean install -DskipTests - -# Install honeycomb agent (v3po) -mkdir -p /opt/honeycomb/v3po -cp -a v3po/karaf /opt/honeycomb/v3po - -# Install ODL Virtual Bridge App -mkdir -p /opt/odl/vbd -cp -a vbd/karaf /opt/odl/vbd - -if [ -d /vpp ] ; then - cd .. - - # Check if git exists and remove it before attempting clone, else clone ineffective when "reload --provision" - [ -d vpp ] && rm -rf vpp - sudo -H -u vagrant git clone /vpp - cd vpp/ - - # Initial vpp build - if [ -d build-root ]; then - # Bootstrap vpp - cd build-root/ - sudo -H -u vagrant ./bootstrap.sh - - # Build vpp - sudo -H -u vagrant make V=0 PLATFORM=vpp TAG=vpp_debug install-deb - - # Install debian packages - dpkg -i *.deb - - # Disable all ethernet interfaces other than the default route - # so VPP will use those interfaces. - for intf in $(ls /sys/class/net) ; do - if [ -d /sys/class/net/$intf/device ] && [ "$(route | grep default | grep $intf)" == "" ] ; then +# Don't install VPP and ODL Honeycomb Agent if this is an ODL VBD application VM +if [ "$VAGRANT_VBD_VM" != "is_vbd_vm" ] ; then + # Disable all ethernet interfaces other than the default route + # interface so VPP will use those interfaces. The VPP auto-blacklist + # algorithm prevents the use of any physical interface contained in the + # routing table (i.e. "route --inet --inet6") preventing the theft of + # the management ethernet interface by VPP from the kernel. + for intf in $(ls /sys/class/net) ; do + if [ "$VAGRANT_VPP_AGENT_ADDR" != "" ] && [ "$(ifconfig $intf | grep $VAGRANT_VPP_AGENT_ADDR)" != "" ] ; then + continue; + fi + if [ -d /sys/class/net/$intf/device ] && [ "$(route --inet --inet6 | grep default | grep $intf)" == "" ] ; then ifconfig $intf down + fi + done + + # Install VPP + if [ -d $VPP_MOUNT ] ; then + # Fix the silly notion that /bin/sh should point to dash by pointing it to bash + sudo update-alternatives --install /bin/sh sh /bin/bash 100 + + # Build and install VPP if necessary + if [ -d $VPP_BUILD_ROOT ] ; then + if [ "$(ls $VPP_BUILD_ROOT/*.deb)" = "" ] ; then + echo "Building VPP" + # Bootstrap vpp + cd $VPP_BUILD_ROOT + sudo -H -u vagrant ./bootstrap.sh + # Build vpp + sudo -H -u vagrant make V=0 PLATFORM=vpp TAG=vpp_debug install-deb fi - done - - # Start vpp + # Install debian packages + echo "Installing VPP from $VPP_BUILD_ROOT" + dpkg -i $VPP_BUILD_ROOT/*.deb + fi + else + echo "Installing VPP from nexus.fd.io" + apt-get install vpp vpp-dpdk-dev vpp-dpdk-dkms vpp-dev vpp-dbg -y --force-yes + fi + # Start VPP if it is installed. + if [ "$(dpkg -l | grep vpp)" != "" ] ; then + echo "Starting VPP" start vpp fi + + # Build Honeycomb if necessary + if [ ! -f $V3PO_TARBALL ] ; then + echo "Building Honeycomb..." + cd $HONEYCOMB_MOUNT + sudo -H -u vagrant mvn clean install -DskipTests + fi + + # Install honeycomb agent (v3po) if available. + if [ -f $V3PO_TARBALL ] ; then + echo + echo "Installing Honeycomb VPP agent in $HONEYCOMB_INSTALL_DIR/$V3PO_SNAPSHOT" + [ ! -d $HONEYCOMB_INSTALL_DIR ] && mkdir -p $HONEYCOMB_INSTALL_DIR + cd $HONEYCOMB_INSTALL_DIR + [ -d $V3PO_SNAPSHOT ] && rm -rf $V3PO_SNAPSHOT + V3PO_KARAF_DIR="$HONEYCOMB_INSTALL_DIR/$(basename $V3PO_TARBALL|perl -pe 's/\.tar.gz//g')" + [ -d $V3PO_KARAF_DIR ] && rm -rf $V3PO_KARAF_DIR + tar xzf $V3PO_TARBALL + $V3PO_KARAF_DIR/bin/start + fi +else + # Build Honeycomb if necessary + if [ ! -f $VBD_TARBALL ] ; then + echo "Building Honeycomb..." + cd $HONEYCOMB_MOUNT + sudo -H -u vagrant mvn clean install -DskipTests + fi + # Install ODL Virtual Bridge Domain App if available. + if [ -f $VBD_TARBALL ] ; then + echo + echo "Installing ODL Virtual Bridge Domain application in $HONEYCOMB_INSTALL_DIR/$VBD_SNAPSHOT" + [ ! -d $HONEYCOMB_INSTALL_DIR ] && mkdir -p $HONEYCOMB_INSTALL_DIR + cd $HONEYCOMB_INSTALL_DIR + VBD_KARAF_DIR="$HONEYCOMB_INSTALL_DIR/$(basename $VBD_TARBALL|perl -pe 's/\.tar.gz//g')" + [ -d $VBD_KARAF_DIR ] && rm -rf $VBD_KARAF_DIR + tar xzf $VBD_TARBALL + $VBD_KARAF_DIR/bin/start + fi fi + +echo "VM Installation Complete!" +echo -- cgit 1.2.3-korg