vpp-bootstrap: initial commit

vpp-bootstrap is a vagrant based tool to automatically bootstrap a
vpp development environment. The tool is designed to be as data-driven
and extensible.

 * An Ubuntu environment featuring one or more containers connected
  via a Linux bridge is created.
 * New containers can be easily created, packages install and configured
  via a provisioning scripts.
 * Networking, auth credentials and hostnames are automatically configured
  to enable using tools like ssh and scp without password and ip addresses.
 * Support for vagrant-proxy and vagrant-aws provisioning.
 * Two containers are initially provided:-
  * The first container provides network test tools such as scapy
  are automatically installed
  * In the second container VPP-lite is automatically git cloned, built
  and installed.

Change-Id: Id484431f7dde8fdc9ff25ad198ff3cec55ba112c
Signed-off-by: Ray Kinsella <ray.kinsella@intel.com>

8 files changed, 396 insertions, 0 deletions

diff --git a/vpp-bootstrap/.gitignore b/vpp-bootstrap/.gitignore
new file mode 100644
index 0000000..ddbe19d
--- /dev/null
+++ b/vpp-bootstrap/.gitignore
@@ -0,0 +1,10 @@
+.vagrant/
+*~
+*#
+*.cmd
+
+node_modules/
+frontend/
+dist.dev/
+node-info.log
+node-error.log
\ No newline at end of file
diff --git a/vpp-bootstrap/README.md b/vpp-bootstrap/README.md
new file mode 100644
index 0000000..67b80ed
--- /dev/null
+++ b/vpp-bootstrap/README.md
@@ -0,0 +1,18 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2016 Intel Corporation
+ */
+# INTRO
+
+<insert useful content here>
diff --git a/vpp-bootstrap/Vagrantfile b/vpp-bootstrap/Vagrantfile
new file mode 100644
index 0000000..9438156
--- /dev/null
+++ b/vpp-bootstrap/Vagrantfile
@@ -0,0 +1,103 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+# Copyright (c) 2016 Intel Corporation
+
+Vagrant.configure(2) do |config|
+
+  # Pick the right distro and bootstrap, default is ubuntu1404
+  config.vm.box = "puppetlabs/ubuntu-14.04-64-nocm"
+  vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 8)
+  vmram=(ENV['VPP_VAGRANT_VMRAM'] || 16384)
+
+  # Define some physical ports for your VMs to be used by DPDK
+  config.vm.network "private_network", type: "dhcp"
+
+  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"provision.sh") , privileged: false
+
+  # vagrant-cachier caches apt/yum etc to speed subsequent
+  # vagrant up
+  # to enable, run
+  # vagrant plugin install vagrant-cachier
+  #
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+  end
+
+  # use http proxy if avaiable
+  if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf")
+   config.proxy.http     = ENV['http_proxy']
+   config.proxy.https    = ENV['https_proxy']
+   config.proxy.no_proxy = "localhost,127.0.0.1"
+  end
+
+  config.vm.provider :aws do |aws, override|
+      #disable any corporate proxies in the AWS cloud
+      if Vagrant.has_plugin?("vagrant-proxyconf")
+         override.proxy.enabled = false
+      end
+
+      #Use rsync instead of nfs to sync folders.
+      override.vm.synced_folder ".", "/vagrant", type: "rsync",
+         rsync__exclude: ".git/"
+
+      #We don't need a local box, use the vagrant-aws dummy instead
+      override.vm.box = "dummy"
+
+      #These are the credentials required to access AWS Instructure.
+      #vagrant-aws requires these to create the new instance.
+      #These can either be your AWS root account access key (not recommended)
+      #or an IAM user with sufficent rights to create EC2 instances.
+      aws.access_key_id = "abcdefg"
+      aws.secret_access_key = "abcdefg"
+
+      #Your preferred region, Ireland is always a good choice.
+      aws.region = "eu-west-1"
+
+      #The EC2 keypair used to provision remote access creds in the
+      #newly created EC2 instance. These creds permit remote access via ssh.
+      aws.keypair_name = "ec2"
+
+      #Security groups (ACLs) to provision new EC2 instance with.
+      #At least one of the security groups should allow SSH.
+      #to enable `vagrant ssh` to work.
+      aws.security_groups = [ "permit-ssh", "default" ]
+
+      #Amazon Machine Instance (AMI) to use, default is Ubuntu Trusty (HVM).
+      aws.ami = "ami-0c59f37f"
+
+      #EC2 instance type (how much cpu/mem resources to give the instance).
+      aws.instance_type = "t2.micro"
+
+      #Any proxy command required for ssh to workaround corporate firewalls
+      #override.ssh.proxy_command = "nc -x proxy.com:1080 %h %p"
+
+      #Ubuntu AMIs use ubuntu as the default username, not vagrant.
+      override.ssh.username = "ubuntu"
+
+      #Private key to access new EC2 instance via SSH, should be the private
+      #key from the keypair_name created above.
+      override.ssh.private_key_path = "/root/private_key.pem"
+  end
+  config.vm.provider "virtualbox" do |vb|
+      vb.name = "vpp-bootstrap"
+
+      vb.customize ["modifyvm", :id, "--ioapic", "on"]
+      vb.memory = "#{vmram}"
+      vb.cpus = "#{vmcpu}"
+
+      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
+      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
+  end
+  config.vm.provider "vmware_fusion" do |fusion,override|
+    fusion.vmx["memsize"] = "#{vmram}"
+    fusion.vmx["numvcpus"] = "#{vmcpu}"
+  end
+  config.vm.provider "libvirt" do |lv|
+    lv.memory = "#{vmram}"
+    lv.cpus = "#{vmcpu}"
+  end
+  config.vm.provider "vmware_workstation" do |vws,override|
+    vws.vmx["memsize"] = "#{vmram}"
+    vws.vmx["numvcpus"] = "#{vmcpu}"
+  end
+end
diff --git a/vpp-bootstrap/containers/cone.cntr b/vpp-bootstrap/containers/cone.cntr
new file mode 100644
index 0000000..237496c
--- /dev/null
+++ b/vpp-bootstrap/containers/cone.cntr
@@ -0,0 +1,4 @@
+DESC: This container is used for vpp testing with scapy.
+DIST: ubuntu
+VER: trusty
+PACKAGES: python-scapy
diff --git a/vpp-bootstrap/containers/ctwo.cntr b/vpp-bootstrap/containers/ctwo.cntr
new file mode 100644
index 0000000..56a83c8
--- /dev/null
+++ b/vpp-bootstrap/containers/ctwo.cntr
@@ -0,0 +1,4 @@
+DESC: This is the vpp build/test container. 
+DIST: ubuntu
+VER: trusty
+PACKAGES: make gcc autotools-dev autoconf linux-headers-kernver
diff --git a/vpp-bootstrap/containers/ctwo.provision.sh b/vpp-bootstrap/containers/ctwo.provision.sh
new file mode 100755
index 0000000..2ba66c6
--- /dev/null
+++ b/vpp-bootstrap/containers/ctwo.provision.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright (c) 2016 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+VPP_DIR=~/vpp
+VPP_GIT="https://git.fd.io/vpp"
+#PLATFORM=vpp_lite
+PLATFORM=vpp
+
+echo Cloning $VPP_GIT
+git clone $VPP_GIT $VPP_DIR
+
+# Install dependencies
+echo Building $VPP_DIR
+cd $VPP_DIR
+make UNATTENDED=yes install-dep
+
+make wipe
+(cd build-root/; make distclean)
+rm -f build-root/.bootstrap.ok
+
+# Build and install packaging
+make PLATFORM=$PLATFORM bootstrap
+make PLATFORM=$PLATFORM pkg-deb
+
+# Install VPPP
+(cd ${VPP_DIR}/build-root/;sudo dpkg -i *.deb)
+
+sudo sed -i 's/dpdk {/dpdk {\n\tno-pci\n/' /etc/vpp/startup.conf
+sudo service vpp start
diff --git a/vpp-bootstrap/provision.sh b/vpp-bootstrap/provision.sh
new file mode 100755
index 0000000..7b52782
--- /dev/null
+++ b/vpp-bootstrap/provision.sh
@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2016 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOME_DIR="/home/$USER"
+SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+APT_PROXY_CONF="/etc/apt/apt.conf.d/01proxy"
+ENV_FILE="/etc/environment"
+UNAMER=$(uname -r)
+DEV_BRIDGE="lxcbr1"
+
+sudo sysctl -w vm.nr_hugepages=1024
+HUGEPAGES=`sudo sysctl -n  vm.nr_hugepages`
+if [ $HUGEPAGES != 1024 ]; then
+    echo "ERROR: Unable to get 1024 hugepages, only got $HUGEPAGES.  Cannot finish."
+    exit
+fi
+
+sudo apt-get -qq update
+sudo apt-get -qq install -y --force-yes lxc bridge-utils
+
+echo -e "lxc.network.name=veth0" | sudo tee -a /etc/lxc/default.conf
+echo -e "lxc.network.type = veth" | sudo tee -a /etc/lxc/default.conf
+echo -e "lxc.network.link = lxcbr1" | sudo tee -a /etc/lxc/default.conf
+echo -e "lxc.network.flags = up" | sudo tee -a /etc/lxc/default.conf
+echo -e "lxc.network.hwaddr = 00:17:3e:xx:xx:xx\n" | sudo tee -a /etc/lxc/default.conf
+echo -e "lxc.network.name=veth_link1" | sudo tee -a /etc/lxc/default.conf
+
+sudo lxc-checkconfig
+
+sudo brctl addbr $DEV_BRIDGE
+sudo ip link set $DEV_BRIDGE up
+
+ssh-keygen -t rsa -b 1024 -N "" -f ~/.ssh/id_rsa
+openssh_pubkey=`cat ~/.ssh/id_rsa.pub`
+
+function lxc_exec() {
+
+    cntr="$1"
+    rCMD="$2"
+
+    CMD="sudo lxc-attach -n $cntr  -- $rCMD"
+
+    echo "$CMD"
+    eval "${CMD}"
+}
+
+function get_field() {
+    file="$1"
+    field="$2"
+    
+    value=$(grep $field $file | awk -F : '{print $2}' | sed -e 's/^[ ]*//' | sed -e 's/kernver/"$UNAMER"/')
+    echo $value
+}
+
+# LXC gives backend interfaces horrible names, give them a better name.
+function rename_veth_interface() {
+
+    local cntr="$1"
+    local nifname="$2"
+
+    ifr_index=`sudo lxc-attach -n $cntr -- ip -o link | tail -n 1 | awk -F : '{print $1}'`
+    ifr_index=$((ifr_index+1))
+
+    for dir in /sys/class/net/*/
+    do
+        ifindex=`cat $dir/ifindex`
+        if [ $ifindex == $ifr_index ]
+            then ifname=`basename $dir`
+        fi
+    done
+
+    sudo ip link set $ifname down
+    sudo ip link set $ifname name $nifname
+    sudo ip link set $nifname up
+}
+
+for f in $(ls /vagrant/containers/*.cntr)
+do
+    i=$(basename $f | sed s/.cntr//)
+    dist=$(get_field $f DIST)
+    ver=$(get_field $f VER)
+    packages=$(get_field $f PACKAGES)
+    provision_file="/vagrant/containers/"$i".provision.sh"
+ 
+    sudo lxc-create -t download -n $i -- --dist $dist --release $ver --arch amd64
+    sudo lxc-start -n $i -d
+	
+    lxc_exec $i "resolvconf -d veth0"
+    lxc_exec $i "dhclient veth0"
+
+    lxc_exec $i "apt-get -qq install -y git openssh-server"
+    lxc_exec $i "apt-get -qq update"
+
+    lxc_exec $i "adduser --disabled-password --gecos \"\" $USER"
+
+    lxc_exec $i "mkdir -p /root/.ssh/"
+    lxc_exec $i "mkdir -p $HOME_DIR/.ssh/"
+
+    lxc_exec $i "sh -c 'echo $openssh_pubkey >> /root/.ssh/authorized_keys'"
+    lxc_exec $i "sh -c 'echo $openssh_pubkey >> $HOME_DIR/.ssh/authorized_keys'"
+
+    lxc_exec $i "chmod 0600 /root/.ssh/authorized_keys"
+    lxc_exec $i "chmod 0600 $HOME_DIR/.ssh/authorized_keys"
+
+    lxc_exec $i "chown -R $USER.$USER $HOME_DIR/.ssh/"
+
+    lxc_exec $i "sh -c 'echo \"%$USER ALL=(ALL) NOPASSWD: ALL\" > /etc/sudoers.d/10_$USER'"
+
+    lxc_exec $i "update-alternatives --install /bin/sh sh /bin/bash 100"
+
+    lxc_exec $i "apt-get -qq install $packages"
+
+    ip_address=$(sudo lxc-ls -f | grep $i | awk '{print $3}')
+    echo $ip_address $i | sudo tee -a /etc/hosts
+
+    if [ -s $APT_PROXY_CONF ]
+    then
+        scp $SSH_OPTIONS $APT_PROXY_CONF root@$i:$APT_PROXY_CONF
+    fi
+
+    if [ -s $ENV_FILE ]
+    then
+        scp $SSH_OPTIONS $ENV_FILE root@$i:$ENV_FILE
+    fi
+
+#    backend_intr="link_"$cntr
+#    rename_veth_interface $i $backend_intr
+#    sudo brctl addif $DEV_BRIDGE $backend_intr
+
+    if [ -s $provision_file ]
+    then
+       tmpname=$(mktemp)".sh"
+       scp $SSH_OPTIONS $provision_file $USER@$i:$tmpname
+       ssh $SSH_OPTIONS $USER@$i "sh -c $tmpname"
+    fi	
+done
+
+echo "List of containers deployed in the dev environment:"
+for f in $(ls /vagrant/containers/*.cntr)
+do
+     i=$(basename $f | sed s/.cntr//)
+     desc=$(get_field $f DESC)
+     echo $i":\t"$desc
+done
+
+echo "To get access the dev environment, type 'vagrant ssh'"
diff --git a/vpp-bootstrap/update.sh b/vpp-bootstrap/update.sh
new file mode 100755
index 0000000..d3e0094
--- /dev/null
+++ b/vpp-bootstrap/update.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make sure that we get the hugepages we need on provision boot
+# Note: The package install should take care of this at the end
+#       But sometimes after all the work of provisioning, we can't
+#       get the requested number of hugepages without rebooting.
+#       So do it here just in case
+sysctl -w vm.nr_hugepages=1024
+HUGEPAGES=`sysctl -n  vm.nr_hugepages`
+if [ $HUGEPAGES != 1024 ]; then
+    echo "ERROR: Unable to get 1024 hugepages, only got $HUGEPAGES.  Cannot finish."
+    exit
+fi
+
+exit 0
+
+# Figure out what system we are running on
+if [ -f /etc/lsb-release ];then
+    . /etc/lsb-release
+elif [ -f /etc/redhat-release ];then
+    yum install -y redhat-lsb
+    DISTRIB_ID=`lsb_release -si`
+    DISTRIB_RELEASE=`lsb_release -sr`
+    DISTRIB_CODENAME=`lsb_release -sc`
+    DISTRIB_DESCRIPTION=`lsb_release -sd`
+fi
+
+# Do initial setup for the system
+if [ $DISTRIB_ID == "Ubuntu" ]; then
+    # Fix grub-pc on Virtualbox with Ubuntu
+    export DEBIAN_FRONTEND=noninteractive
+
+    # Standard update + upgrade dance
+    apt-get update
+    apt-get upgrade -y
+
+    # Fix the silly notion that /bin/sh should point to dash by pointing it to bash
+
+    update-alternatives --install /bin/sh sh /bin/bash 100
+
+    # Install useful but non-mandatory tools
+    apt-get install -y emacs git-review gdb gdbserver brctl
+elif [ $DISTRIB_ID == "CentOS" ]; then
+    # Standard update + upgrade dance
+    yum check-update
+    yum update -y
+fi