Add Vagrantfile for local testing.

Vagrantfile contains 3 VMs as of now, 2 DUTs 1 TG, with these notes:
 - login is csit/csit
 - by default provision script installs all deb packages from the dir
    where Vagrantfile is
 - developed for, and only tested on vbox (someone can pick up vmware)
 - All nodes have 1 shared mgmt network: 192.168.255.0/24
 - hosts have these IP addresses in host-only network
    TG : 192.168.255.100
    DUT1 : 192.168.255.101
    DUT2 : 192.168.255.102
 - script created to download MAC address information
 - PCI addresses are always the same for vbox (not sure about vmware)

HOWTO (will create a wiki page once one is created for CSIT project):
 - copy Vagrantfile to separate dir on host
 - vagrant up --parallel
    sit-back-and-relax
 - from VM that has access to the same host-only network (192.168.255.0 above)
    - copy your ssh-key to csit@192.168.255.{101,102,250} using
        ssh-copy-id
    - cd ${csit_dir}
    - virtualenv & pip as in README
    - export PYTHONPATH=${csit_dir}
    - resources/tools/topology/update_topology.py -v -f
        -o topologies/available/vagrant_pci.yaml \
        topologies/available/vagrant.yaml
    - pybot -L TRACE \
        -v TOPOLOGY_PATH:topologies/available/vagrant_pci.yaml -s \
        "ipv4" tests
    - see tests results

Change-Id: Ic27626605a9c820bca977b38f4e8ca37d1504ff5
Signed-off-by: Stefan Kobza <skobza@cisco.com>

7 files changed, 440 insertions, 17 deletions

diff --git a/docs/testing_in_vagrant b/docs/testing_in_vagrant
new file mode 100644
index 0000000000..1007621874
--- /dev/null
+++ b/docs/testing_in_vagrant
@@ -0,0 +1,16 @@
+HOWTO (will create a wiki page once one is created for CSIT project):
+ - copy Vagrantfile to separate dir on host
+ - vagrant up --parallel
+    sit-back-and-relax
+ - from VM that has access to the same host-only network (192.168.255.0 above)
+    - copy your ssh-key to csit@192.168.255.{101,102,250}
+    - cd ${csit_dir}
+    - virtualenv & pip as in README
+    - PYTHONPATH=`pwd` resources/tools/topology/update_topology.py \
+        topologies/available/vagrant.yaml \
+        -o topologies/available/vagrant_pci.yaml
+    - PYTHONPATH=`pwd` pybot -L TRACE \
+        -v TOPOLOGY_PATH:topologies/available/vagrant_pci.yaml -s \
+        "bridge domain" tests
+    - see tests results
+
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 2a0bd4284e..b3df489685 100644
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -11,19 +11,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""This module exists to provide setup utilities for the framework on topology
+nodes. All tasks required to be run before the actual tests are started is
+supposed to end up here.
+"""
+
 import shlex
 from subprocess import Popen, PIPE, call
 from multiprocessing import Pool
 from tempfile import NamedTemporaryFile
 from os.path import basename
+
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
-from ssh import SSH
-from constants import Constants as con
-from topology import NodeType
 
-__all__ = ["SetupFramework"]
+from resources.libraries.python.ssh import SSH
+from resources.libraries.python.constants import Constants as con
+from resources.libraries.python.topology import NodeType
 
+__all__ = ["SetupFramework"]
 
 def pack_framework_dir():
     """Pack the testing WS into temp file, return its name."""
@@ -48,6 +54,14 @@ def pack_framework_dir():
 
 
 def copy_tarball_to_node(tarball, node):
+    """Copy tarball file from local host to remote node.
+
+    :param tarball: path to tarball to upload
+    :param node: dictionary created from topology
+    :type tarball: string
+    :type node: dict
+    :return: nothing
+    """
     logger.console('Copying tarball to {0}'.format(node['host']))
     ssh = SSH()
     ssh.connect(node)
@@ -56,6 +70,16 @@ def copy_tarball_to_node(tarball, node):
 
 
 def extract_tarball_at_node(tarball, node):
+    """Extract tarball at given node.
+
+    Extracts tarball using tar on given node to specific CSIT loocation.
+
+    :param tarball: path to tarball to upload
+    :param node: dictionary created from topology
+    :type tarball: string
+    :type node: dict
+    :return: nothing
+    """
     logger.console('Extracting tarball to {0} on {1}'.format(
         con.REMOTE_FW_DIR, node['host']))
     ssh = SSH()
@@ -63,7 +87,7 @@ def extract_tarball_at_node(tarball, node):
 
     cmd = 'sudo rm -rf {1}; mkdir {1} ; tar -zxf {0} -C {1}; ' \
         'rm -f {0}'.format(tarball, con.REMOTE_FW_DIR)
-    (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=30)
+    (ret_code, _, stderr) = ssh.exec_command(cmd, timeout=30)
     if 0 != ret_code:
         logger.error('Unpack error: {0}'.format(stderr))
         raise Exception('Failed to unpack {0} at node {1}'.format(
@@ -77,9 +101,9 @@ def create_env_directory_at_node(node):
     ssh = SSH()
     ssh.connect(node)
     (ret_code, stdout, stderr) = ssh.exec_command(
-            'cd {0} && rm -rf env && virtualenv env && '
-            '. env/bin/activate && '
-            'pip install -r requirements.txt'.format(con.REMOTE_FW_DIR), timeout=100)
+        'cd {0} && rm -rf env && virtualenv env && . env/bin/activate && '
+        'pip install -r requirements.txt'.format(con.REMOTE_FW_DIR),
+                                                 timeout=100)
     if 0 != ret_code:
         logger.error('Virtualenv creation error: {0}'.format(stdout + stderr))
         raise Exception('Virtualenv setup failed')
@@ -87,18 +111,32 @@ def create_env_directory_at_node(node):
         logger.console('Virtualenv created on {0}'.format(node['host']))
 
 def setup_node(args):
+    """Run all set-up methods for a node.
+
+    This method is used as map_async parameter. It receives tuple with all
+    parameters as passed to map_async function.
+
+    :param args: all parameters needed to setup one node
+    :type args: tuple
+    :return: nothing
+    """
     tarball, remote_tarball, node = args
     copy_tarball_to_node(tarball, node)
     extract_tarball_at_node(remote_tarball, node)
     if node['type'] == NodeType.TG:
         create_env_directory_at_node(node)
-
+    logger.console('Setup of node {0} done'.format(node['host']))
 
 def delete_local_tarball(tarball):
-    call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))
+    """Delete local tarball to prevent disk pollution.
 
+    :param tarball: path to tarball to upload
+    :type tarball: string
+    :return: nothing
+    """
+    call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))
 
-class SetupFramework(object):
+class SetupFramework(object): # pylint: disable=too-few-public-methods
     """Setup suite run on topology nodes.
 
     Many VAT/CLI based tests need the scripts at remote hosts before executing
@@ -109,7 +147,8 @@ class SetupFramework(object):
     def __init__(self):
         pass
 
-    def setup_framework(self, nodes):
+    @staticmethod
+    def setup_framework(nodes):
         """Pack the whole directory and extract in temp on each node."""
 
         tarball = pack_framework_dir()
@@ -136,3 +175,5 @@ class SetupFramework(object):
         BuiltIn().set_log_level(log_level)
         logger.trace('Test framework copied to all topology nodes')
         delete_local_tarball(tarball)
+        logger.console('All nodes are ready')
+
diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py
index a94eec4e91..6914d528d6 100644
--- a/resources/libraries/python/ssh.py
+++ b/resources/libraries/python/ssh.py
@@ -10,6 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import socket
 import paramiko
 from paramiko import RSAKey
 import StringIO
@@ -95,11 +96,17 @@ class SSH(object):
             self._ssh.get_transport().getpeername(), end-start))
 
         stdout = ""
-        while True:
-            buf = chan.recv(self.__MAX_RECV_BUF)
-            stdout += buf
-            if not buf:
-                break
+        try:
+            while True:
+                buf = chan.recv(self.__MAX_RECV_BUF)
+                stdout += buf
+                if not buf:
+                    break
+        except socket.timeout:
+            logger.error('Caught timeout exception, current contents '
+                         'of buffer: {0}'.format(stdout))
+            raise
+
 
         stderr = ""
         while True:
diff --git a/resources/tools/topology/update_topology.py b/resources/tools/topology/update_topology.py
new file mode 100755
index 0000000000..d7a3929643
--- /dev/null
+++ b/resources/tools/topology/update_topology.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python2.7
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This executable python module gathers MAC address data from topology nodes.
+It requires that all interfaces/port elements in topology have driver field.
+This script binds the port in given node to set linux kernel driver and
+extracts MAC address from it."""
+
+import sys
+import os
+import re
+from argparse import ArgumentParser
+
+import yaml
+
+from resources.libraries.python.ssh import SSH
+
+def load_topology(args):
+    """Load topology file referenced to by parameter passed to this script.
+
+    :param args: arguments parsed from commandline
+    :type args: ArgumentParser().parse_args()
+    :return: Python representation of topology yaml
+    :rtype: dict
+    """
+    data = None
+    with open(args.topology, 'r') as stream:
+        try:
+            data = yaml.load(stream)
+        except yaml.YAMLError as exc:
+            print 'Failed to load topology file: {0}'.format(args.topology)
+            print exc
+            raise
+
+    return data
+
+def ssh_no_error(ssh, cmd):
+    """Execute a command over ssh channel, and log and exit if the command
+    fials.
+
+    :param ssh: SSH() object connected to a node
+    :param cmd: Command line to execute on remote node
+    :type ssh: SSH() object
+    :type cmd: str
+    :return: stdout from the SSH command
+    :rtype: str
+    """
+    ret, stdo, stde = ssh.exec_command(cmd)
+    if 0 != ret:
+        print 'Command execution failed: "{}"'.format(cmd)
+        print 'stdout: {0}'.format(stdo)
+        print 'stderr: {0}'.format(stde)
+        raise RuntimeError('Unexpected ssh command failure')
+
+    return stdo
+
+def update_mac_addresses_for_node(node):
+    """For given node loop over all ports with PCI address and look for its MAC
+    address.
+
+    This function firstly unbinds the PCI device from its current driver
+    and binds it to linux kernel driver. After the device is bound to specific
+    linux kernel driver the MAC address is extracted from /sys/bus/pci location
+    and stored within the node dictionary that was passed to this function.
+    :param node: Node from topology
+    :type node: dict
+    :return: None
+    """
+    for port_name, port in node['interfaces'].items():
+        if not port.has_key('driver'):
+            err_msg = '{0} port {1} has no driver element, exiting'.format(
+                    node['host'], port_name)
+            raise RuntimeError(err_msg)
+
+        ssh = SSH()
+        ssh.connect(node)
+
+        # TODO: make following SSH commands into one-liner to save on SSH opers
+
+        # First unbind from current driver
+        drvr_dir_path = '/sys/bus/pci/devices/{0}/driver'.format(
+                port['pci_address'])
+        cmd = '''\
+            if [ -d {0} ]; then
+                echo {1} | sudo tee {0}/unbind ;
+            else
+                true Do not have to do anything, port already unbound ;
+            fi'''.format(drvr_dir_path, port['pci_address'])
+        ssh_no_error(ssh, cmd)
+
+        # Then bind to the 'driver' from topology for given port
+        cmd = 'echo {0} | sudo tee /sys/bus/pci/drivers/{1}/bind'.\
+                format(port['pci_address'], port['driver'])
+        ssh_no_error(ssh, cmd)
+
+        # Then extract the mac address and store it in the topology
+        cmd = 'cat /sys/bus/pci/devices/{0}/net/*/address'.format(
+                port['pci_address'])
+        mac = ssh_no_error(ssh, cmd).strip()
+        pattern = re.compile("^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$")
+        if not pattern.match(mac):
+            raise RuntimeError('MAC address read from host {0} {1} is in '
+                                   'bad format "{2}"'.format(node['host'],
+                                       port['pci_address'], mac))
+        print '{0}: Found MAC address of PCI device {1}: {2}'.format(
+                node['host'], port['pci_address'], mac)
+        port['mac_address'] = mac
+
+def update_nodes_mac_addresses(topology):
+    """Loop over nodes in topology and get mac addresses for all listed ports
+    based on PCI addresses.
+
+    :param topology: Topology information with nodes
+    :type topology: dict
+    :return: None
+    """
+
+    for node in topology['nodes'].values():
+        update_mac_addresses_for_node(node)
+
+def dump_updated_topology(topology, args):
+    """Writes or prints out updated topology file.
+
+    :param topology: Topology information with nodes
+    :param args: arguments parsed from command line
+    :type topology: dict
+    :type args: ArgumentParser().parse_args()
+    :return: 1 if error occured, 0 if successful
+    :rtype: int
+    """
+
+    if args.output_file:
+        if not args.force:
+            if os.path.isfile(args.output_file):
+                print ('File {0} already exists. If you want to overwrite this '
+                       'file, add -f as a parameter to this script'.format(
+                           args.output_file))
+                return 1
+        with open(args.output_file, 'w') as stream:
+            yaml.dump(topology, stream, default_flow_style=False)
+    else:
+        print yaml.dump(topology, default_flow_style=False)
+    return 0
+
+def main():
+    """Main function"""
+    parser = ArgumentParser()
+    parser.add_argument('topology', help="Topology yaml file to read")
+    parser.add_argument('--output-file', '-o', help='Output file')
+    parser.add_argument('-f', '--force', help='Overwrite existing file',
+                        action='store_const', const=True)
+    parser.add_argument('--verbose', '-v', action='store_true')
+    args = parser.parse_args()
+
+    topology = load_topology(args)
+    update_nodes_mac_addresses(topology)
+    ret = dump_updated_topology(topology, args)
+
+    return ret
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
+
diff --git a/resources/tools/vagrant/Vagrantfile b/resources/tools/vagrant/Vagrantfile
new file mode 100644
index 0000000000..3e18192bec
--- /dev/null
+++ b/resources/tools/vagrant/Vagrantfile
@@ -0,0 +1,86 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- mode: ruby -*-
+# vi: set ts=2 sw=2 sts=2 et ft=ruby :
+
+$user_addition = <<-SHELL
+    sudo deluser csit
+    sudo adduser --disabled-password --gecos "" csit
+    echo csit:csit | sudo chpasswd
+    sudo adduser csit vagrant
+    id csit
+SHELL
+
+$install_prereqs = <<-SHELL
+    sudo apt-get -y update
+    sudo apt-get -y -f install
+    sudo apt-get -y install python-virtualenv python-dev iproute2 debhelper dkms
+    sudo update-alternatives --install /bin/sh sh /bin/bash 100
+SHELL
+
+$install_vpp = <<-SHELL
+    sudo apt-get -y purge vpp\*
+    cd /vagrant
+    if [ -e /vagrant/vpp-*.deb ]; then
+        sudo dpkg -i vpp-*.deb
+    fi
+SHELL
+
+
+def add_dut(config, name, mgmt_ip, net1, net2)
+  config.vm.define name do |node|
+    node.vm.box = "puppetlabs/ubuntu-14.04-64-nocm"
+    node.vm.hostname = name
+    node.vm.provision "shell", inline: $user_addition
+    node.vm.provision "shell", inline: $install_prereqs
+    node.vm.provision "shell", inline: $install_vpp
+
+    node.vm.network "private_network", ip: mgmt_ip
+    node.vm.network "private_network", type: "dhcp", auto_config: false,
+        virtualbox__intnet: net1
+    node.vm.network "private_network", type: "dhcp", auto_config: false,
+        virtualbox__intnet: net2
+    node.vm.provider "virtualbox" do |vb|
+      vb.memory = "2048"
+      vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+      vb.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"]
+    end
+  end
+
+end
+
+Vagrant.configure(2) do |config|
+  config.vm.define "tg" do |tg|
+    tg.vm.box = "puppetlabs/ubuntu-14.04-64-nocm"
+    tg.vm.hostname = "tg"
+
+    tg.vm.provision "shell", inline: $user_addition
+    tg.vm.provision "shell", inline: $install_prereqs
+    tg.vm.network "private_network", ip: '192.168.255.100/24'
+    tg.vm.network "private_network", type: "dhcp", auto_config: false,
+        virtualbox__intnet: "tg_dut1"
+    tg.vm.network "private_network", type: "dhcp", auto_config: false,
+        virtualbox__intnet: "tg_dut2"
+    tg.vm.provider "virtualbox" do |vb|
+      vb.memory = "2048"
+      vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+      vb.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"]
+    end
+
+  end
+
+  add_dut(config, "dut1", "192.168.255.101/24", "tg_dut1", "dut1_dut2")
+  add_dut(config, "dut2", "192.168.255.102/24", "tg_dut2", "dut1_dut2")
+end
+
diff --git a/resources/tools/vagrant/install_debs.sh b/resources/tools/vagrant/install_debs.sh
new file mode 100755
index 0000000000..5ace4bae0d
--- /dev/null
+++ b/resources/tools/vagrant/install_debs.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -x
+
+USERNAME=csit
+
+function ssh_do_duts {
+    ssh ${USERNAME}@192.168.255.101 ${@} || exit
+    ssh ${USERNAME}@192.168.255.102 ${@} || exit
+}
+
+rsync -avz ${@} ${USERNAME}@192.168.255.101:/tmp/ || exit
+rsync -avz ${@} ${USERNAME}@192.168.255.102:/tmp/ || exit
+
+ssh_do_duts "sudo apt-get -y purge 'vpp.*' ; exit 0"
+ssh_do_duts "sudo dpkg -i /tmp/vpp*.deb"
+ssh_do_duts "echo 128 | sudo tee /proc/sys/vm/nr_hugepages"
+ssh_do_duts "sudo rm -f /etc/vpp/startup.conf.orig ; sudo cp /etc/vpp/startup.conf /etc/vpp/startup.conf.orig"
+ssh_do_duts "sudo rm /etc/vpp/startup.conf"
+ssh_do_duts "sudo sed -e 's/socket-mem [0-9]*/socket-mem 128/' /etc/vpp/startup.conf.orig | sudo tee /etc/vpp/startup.conf"
+ssh_do_duts "echo heapsize 512M | sudo tee -a /etc/vpp/startup.conf"
+ssh_do_duts "sudo sed -e 's/vm.nr_hugepages=.*/vm.nr_hugepages=128/' -i /etc/sysctl.d/80-vpp.conf"
+ssh_do_duts "sudo sed -e 's/vm.max_map_count=.*/vm.max_map_count=256/' -i /etc/sysctl.d/80-vpp.conf"
+
+
+echo Success!
diff --git a/topologies/available/vagrant.yaml b/topologies/available/vagrant.yaml
new file mode 100644
index 0000000000..6c724c495f
--- /dev/null
+++ b/topologies/available/vagrant.yaml
@@ -0,0 +1,60 @@
+---
+metadata:
+  version: 0.1
+  schema:
+    - resources/topology_schemas/3_node_topology.sch.yaml
+    - resources/topology_schemas/topology.sch.yaml
+  tags: [vagrant, 3-node]
+
+nodes:
+  TG:
+    type: TG
+    host: "192.168.255.100"
+    port: 22
+    username: csit
+    password: csit
+    interfaces:
+      port3:
+        mac_address: ""
+        pci_address: "0000:00:09.0"
+        link: link1
+        driver: e1000
+      port5:
+        mac_address: ""
+        pci_address: "0000:00:0a.0"
+        link: link2
+        driver: e1000
+  DUT1:
+    type: DUT
+    host: "192.168.255.101"
+    port: 22
+    username: csit
+    password: csit
+    interfaces:
+      port1:
+        mac_address: ""
+        pci_address: "0000:00:09.0"
+        link: link1
+        driver: e1000
+      port3:
+        mac_address: ""
+        pci_address: "0000:00:0a.0"
+        link: link3
+        driver: e1000
+  DUT2:
+    type: DUT
+    host: "192.168.255.102"
+    port: 22
+    username: csit
+    password: csit
+    interfaces:
+      port1:
+        mac_address: ""
+        pci_address: "0000:00:09.0"
+        link: link2
+        driver: e1000
+      port3:
+        mac_address: ""
+        pci_address: "0000:00:0a.0"
+        link: link3
+        driver: e1000