Add multi-vm Vagrantfile for vcl-test.

- Existing Vagrantfile is symbolic link to the
  default Vagrantfile.
- In order to run the multi-host vcl test,
  change Vagrantfile -> Vagrantfile.vcl_test
- Fix socket_test.sh & vppcom bugs.

Change-Id: I965b7f799135c86e989c08bf6c5909677ef38dea
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

6 files changed, 272 insertions, 144 deletions

diff --git a/extras/vagrant/Vagrantfile b/extras/vagrant/Vagrantfile
index 07db660c..a0bc2252 100644..120000
--- a/extras/vagrant/Vagrantfile
+++ b/extras/vagrant/Vagrantfile
@@ -1,113 +1 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-Vagrant.configure(2) do |config|
-
-  # Pick the right distro and bootstrap, default is ubuntu1604
-  distro = ( ENV['VPP_VAGRANT_DISTRO'] || "ubuntu1604")
-  if distro == 'centos7'
-    config.vm.box = "puppetlabs/centos-7.2-64-nocm"
-    config.ssh.insert_key = false
-  else
-    config.vm.box = "puppetlabs/ubuntu-16.04-64-nocm"
-  end
-  config.vm.box_check_update = false
-
-  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"update.sh")
-  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"build.sh"), :args => "/vpp vagrant"
-
-  post_build = ( ENV['VPP_VAGRANT_POST_BUILD'] )
-  if post_build == "test"
-    config.vm.provision "shell", inline: "echo Testing VPP; cd /vpp; make test"
-  elsif post_build == "install"
-    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"install.sh"), :args => "/vpp"
-    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"clearinterfaces.sh")
-    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"run.sh")
-  elsif post_build == "vcl-test"
-    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"vcl_test.sh"), :args => "/vpp vagrant"
-  end
-
-  # Add .gnupg dir in so folks can sign patches
-  # Note, as gnupg puts socket files in that dir, we have
-  # to be cautious and make sure we are dealing with a plain file
-  homedir = File.expand_path("~/")
-  Dir["#{homedir}/.gnupg/**/*"].each do |fname|
-    if File.file?(fname)
-      destname = fname.sub(Regexp.escape("#{homedir}/"),'')
-      config.vm.provision "file", source: fname, destination: destname
-    end
-  end
-
-  # Copy in the .gitconfig if it exists
-  if File.file?(File.expand_path("~/.gitconfig"))
-    config.vm.provision  "file", source: "~/.gitconfig", destination: ".gitconfig"
-  end
-
-  # vagrant-cachier caches apt/yum etc to speed subsequent
-  # vagrant up
-  # to enable, run
-  # vagrant plugin install vagrant-cachier
-  #
-  if Vagrant.has_plugin?("vagrant-cachier")
-    config.cache.scope = :box
-  end
-
-  # Define some physical ports for your VMs to be used by DPDK
-  nics = (ENV['VPP_VAGRANT_NICS'] || "2").to_i(10)
-  for i in 1..nics
-    config.vm.network "private_network", type: "dhcp"
-  end
-
-  # use http proxy if avaiable
-  if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf")
-   config.proxy.http     = ENV['http_proxy']
-   config.proxy.https    = ENV['https_proxy']
-   config.proxy.no_proxy = "localhost,127.0.0.1"
-  end
-
-  vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 2)
-  vmram=(ENV['VPP_VAGRANT_VMRAM'] || 4096)
-
-  config.ssh.forward_agent = true
-  config.ssh.forward_x11 = true
-
-  config.vm.provider "virtualbox" do |vb|
-      vb.customize ["modifyvm", :id, "--ioapic", "on"]
-      vb.memory = "#{vmram}"
-      vb.cpus = "#{vmcpu}"
-
-      # rsync the vpp directory if provision hasn't happened yet
-      unless File.exist? (".vagrant/machines/default/virtualbox/action_provision")
-        config.vm.synced_folder "../../", "/vpp", type: "rsync",
-         rsync__auto: false,
-         rsync__exclude: [
-          "build-root/build*/",
-          "build-root/install*/",
-          "build-root/images*/",
-          "build-root/*.deb",
-          "build-root/*.rpm",
-          "build-root/*.changes",
-          "build-root/python",
-          "build-root/deb/debian/*.dkms",
-          "build-root/deb/debian/*.install",
-          "build-root/deb/debian/changes",
-          "build-root/tools"]
-      end
-
-      #support for the SSE4.x instruction is required in some versions of VB.
-      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
-      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
-  end
-  config.vm.provider "vmware_fusion" do |fusion,override|
-    fusion.vmx["memsize"] = "#{vmram}"
-    fusion.vmx["numvcpus"] = "#{vmcpu}"
-  end
-  config.vm.provider "libvirt" do |lv|
-    lv.memory = "#{vmram}"
-    lv.cpus = "#{vmcpu}"
-  end
-  config.vm.provider "vmware_workstation" do |vws,override|
-    vws.vmx["memsize"] = "#{vmram}"
-    vws.vmx["numvcpus"] = "#{vmcpu}"
-  end
-end
+Vagrantfile.default
\ No newline at end of file
diff --git a/extras/vagrant/Vagrantfile.default b/extras/vagrant/Vagrantfile.default
new file mode 100644
index 00000000..8d6d02a6
--- /dev/null
+++ b/extras/vagrant/Vagrantfile.default
@@ -0,0 +1,111 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+
+  # Pick the right distro and bootstrap, default is ubuntu1604
+  distro = ( ENV['VPP_VAGRANT_DISTRO'] || "ubuntu1604")
+  if distro == 'centos7'
+    config.vm.box = "puppetlabs/centos-7.2-64-nocm"
+    config.ssh.insert_key = false
+  else
+    config.vm.box = "puppetlabs/ubuntu-16.04-64-nocm"
+  end
+  config.vm.box_check_update = false
+
+  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"update.sh")
+  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"build.sh"), :args => "/vpp vagrant"
+
+  post_build = ( ENV['VPP_VAGRANT_POST_BUILD'] )
+  if post_build == "test"
+    config.vm.provision "shell", inline: "echo Testing VPP; cd /vpp; make test"
+  elsif post_build == "install"
+    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"install.sh"), :args => "/vpp"
+    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"clearinterfaces.sh")
+    config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"run.sh")
+  end
+
+  # Add .gnupg dir in so folks can sign patches
+  # Note, as gnupg puts socket files in that dir, we have
+  # to be cautious and make sure we are dealing with a plain file
+  homedir = File.expand_path("~/")
+  Dir["#{homedir}/.gnupg/**/*"].each do |fname|
+    if File.file?(fname)
+      destname = fname.sub(Regexp.escape("#{homedir}/"),'')
+      config.vm.provision "file", source: fname, destination: destname
+    end
+  end
+
+  # Copy in the .gitconfig if it exists
+  if File.file?(File.expand_path("~/.gitconfig"))
+    config.vm.provision  "file", source: "~/.gitconfig", destination: ".gitconfig"
+  end
+
+  # vagrant-cachier caches apt/yum etc to speed subsequent
+  # vagrant up
+  # to enable, run
+  # vagrant plugin install vagrant-cachier
+  #
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+  end
+
+  # Define some physical ports for your VMs to be used by DPDK
+  nics = (ENV['VPP_VAGRANT_NICS'] || "2").to_i(10)
+  for i in 1..nics
+    config.vm.network "private_network", type: "dhcp"
+  end
+
+  # use http proxy if avaiable
+  if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf")
+   config.proxy.http     = ENV['http_proxy']
+   config.proxy.https    = ENV['https_proxy']
+   config.proxy.no_proxy = "localhost,127.0.0.1"
+  end
+
+  vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 2)
+  vmram=(ENV['VPP_VAGRANT_VMRAM'] || 4096)
+
+  config.ssh.forward_agent = true
+  config.ssh.forward_x11 = true
+
+  config.vm.provider "virtualbox" do |vb|
+      vb.customize ["modifyvm", :id, "--ioapic", "on"]
+      vb.memory = "#{vmram}"
+      vb.cpus = "#{vmcpu}"
+
+      # rsync the vpp directory if provision hasn't happened yet
+      unless File.exist? (".vagrant/machines/default/virtualbox/action_provision")
+        config.vm.synced_folder "../../", "/vpp", type: "rsync",
+         rsync__auto: false,
+         rsync__exclude: [
+          "build-root/build*/",
+          "build-root/install*/",
+          "build-root/images*/",
+          "build-root/*.deb",
+          "build-root/*.rpm",
+          "build-root/*.changes",
+          "build-root/python",
+          "build-root/deb/debian/*.dkms",
+          "build-root/deb/debian/*.install",
+          "build-root/deb/debian/changes",
+          "build-root/tools"]
+      end
+
+      #support for the SSE4.x instruction is required in some versions of VB.
+      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
+      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
+  end
+  config.vm.provider "vmware_fusion" do |fusion,override|
+    fusion.vmx["memsize"] = "#{vmram}"
+    fusion.vmx["numvcpus"] = "#{vmcpu}"
+  end
+  config.vm.provider "libvirt" do |lv|
+    lv.memory = "#{vmram}"
+    lv.cpus = "#{vmcpu}"
+  end
+  config.vm.provider "vmware_workstation" do |vws,override|
+    vws.vmx["memsize"] = "#{vmram}"
+    vws.vmx["numvcpus"] = "#{vmcpu}"
+  end
+end
diff --git a/extras/vagrant/Vagrantfile.vcl_test b/extras/vagrant/Vagrantfile.vcl_test
new file mode 100644
index 00000000..92d6832a
--- /dev/null
+++ b/extras/vagrant/Vagrantfile.vcl_test
@@ -0,0 +1,110 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+
+  # Pick the right distro and bootstrap, default is ubuntu1604
+  distro = ( ENV['VPP_VAGRANT_DISTRO'] || "ubuntu1604")
+  if distro == 'centos7'
+    config.vm.box = "puppetlabs/centos-7.2-64-nocm"
+    config.ssh.insert_key = false
+  else
+    config.vm.box = "puppetlabs/ubuntu-16.04-64-nocm"
+  end
+  config.vm.box_check_update = false
+
+  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"update.sh")
+  config.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"build.sh"), :args => "/vpp vagrant"
+
+  # Create VCL client and server VM's
+  config.vm.define "vcl-server" do |server|
+    server.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"vcl_test.sh"), :args => "/vpp vagrant vcl-server"
+  end
+  config.vm.define "vcl-client" do |client|
+    client.vm.provision :shell, :path => File.join(File.dirname(__FILE__),"vcl_test.sh"), :args => "/vpp vagrant vcl-client"
+  end
+  
+  # Add .gnupg dir in so folks can sign patches
+  # Note, as gnupg puts socket files in that dir, we have
+  # to be cautious and make sure we are dealing with a plain file
+  homedir = File.expand_path("~/")
+  Dir["#{homedir}/.gnupg/**/*"].each do |fname|
+    if File.file?(fname)
+      destname = fname.sub(Regexp.escape("#{homedir}/"),'')
+      config.vm.provision "file", source: fname, destination: destname
+    end
+  end
+
+  # Copy in the .gitconfig if it exists
+  if File.file?(File.expand_path("~/.gitconfig"))
+    config.vm.provision  "file", source: "~/.gitconfig", destination: ".gitconfig"
+  end
+
+  # vagrant-cachier caches apt/yum etc to speed subsequent
+  # vagrant up
+  # to enable, run
+  # vagrant plugin install vagrant-cachier
+  #
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+  end
+
+  # Define some physical ports for your VMs to be used by DPDK
+  nics = (ENV['VPP_VAGRANT_NICS'] || "2").to_i(10)
+  for i in 1..nics
+    config.vm.network "private_network", type: "dhcp"
+  end
+
+  # use http proxy if avaiable
+  if ENV['http_proxy'] && Vagrant.has_plugin?("vagrant-proxyconf")
+   config.proxy.http     = ENV['http_proxy']
+   config.proxy.https    = ENV['https_proxy']
+   config.proxy.no_proxy = "localhost,127.0.0.1"
+  end
+
+  vmcpu=(ENV['VPP_VAGRANT_VMCPU'] || 2)
+  vmram=(ENV['VPP_VAGRANT_VMRAM'] || 4096)
+
+  config.ssh.forward_agent = true
+  config.ssh.forward_x11 = true
+
+  config.vm.provider "virtualbox" do |vb|
+      vb.customize ["modifyvm", :id, "--ioapic", "on"]
+      vb.memory = "#{vmram}"
+      vb.cpus = "#{vmcpu}"
+
+      # rsync the vpp directory if provision hasn't happened yet
+      unless File.exist? (".vagrant/machines/client/virtualbox/action_provision")
+        config.vm.synced_folder "../../", "/vpp", type: "rsync",
+         rsync__auto: false,
+         rsync__exclude: [
+          "build-root/build*/",
+          "build-root/install*/",
+          "build-root/images*/",
+          "build-root/*.deb",
+          "build-root/*.rpm",
+          "build-root/*.changes",
+          "build-root/python",
+          "build-root/deb/debian/*.dkms",
+          "build-root/deb/debian/*.install",
+          "build-root/deb/debian/changes",
+          "build-root/tools"]
+      end
+
+      #support for the SSE4.x instruction is required in some versions of VB.
+      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
+      vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
+  end
+  config.vm.provider "vmware_fusion" do |fusion,override|
+    fusion.vmx["memsize"] = "#{vmram}"
+    fusion.vmx["numvcpus"] = "#{vmcpu}"
+  end
+  config.vm.provider "libvirt" do |lv|
+    lv.memory = "#{vmram}"
+    lv.cpus = "#{vmcpu}"
+  end
+  config.vm.provider "vmware_workstation" do |vws,override|
+    vws.vmx["memsize"] = "#{vmram}"
+    vws.vmx["numvcpus"] = "#{vmcpu}"
+  end  
+end
diff --git a/extras/vagrant/vcl_test.sh b/extras/vagrant/vcl_test.sh
index 9fb924ae..3be18517 100644
--- a/extras/vagrant/vcl_test.sh
+++ b/extras/vagrant/vcl_test.sh
@@ -1,23 +1,25 @@
 #!/bin/bash
 
-# Get Command Line arguements if present
-VPP_DIR=$1
-if [ "x$1" != "x" ]; then
+if [ -n "$1" ]; then
     VPP_DIR=$1
 else
     VPP_DIR=`dirname $0`/../../
 fi
 
-if [ "x$2" != "x" ]; then
+if [ -n "$2" ]; then
     SUDOCMD="sudo -H -u $2"
 fi
 
 echo 'Building VCL test apps'
 cd $VPP_DIR
 $SUDOCMD perl -pi -e 's/noinst_PROGRAMS/bin_PROGRAMS/g' $VPP_DIR/src/uri.am
-$SUDOCMD make build-release
-echo "export WS_ROOT=$VPP_DIR" | sudo -H -u vagrant tee /home/vagrant/.bash_aliases
-source /home/vagrant/.bash_aliases
-sudo cp $VPP_DIR/src/vpp/conf/80-vpp.conf /etc/sysctl.d
-sudo sysctl -p/etc/sysctl.d/80-vpp.conf
+$SUDOCMD make dpdk-install-dev build-release
+sudo sysctl -p$VPP_DIR/src/vpp/conf/80-vpp.conf
 sudo modprobe uio_pci_generic
+
+if [ "$2" = "vagrant" ] && [ -d "/home/vagrant" ] ; then
+    dot_bash_aliases="/home/$2/.bash_aliases"
+    echo "export WS_ROOT=$VPP_DIR" | $SUDOCMD tee $dot_bash_aliases
+    source $dot_bash_aliases
+fi
+
diff --git a/src/uri/vppcom.c b/src/uri/vppcom.c
index aa307f1d..8a8a806c 100644
--- a/src/uri/vppcom.c
+++ b/src/uri/vppcom.c
@@ -852,19 +852,15 @@ vl_api_unbind_sock_reply_t_handler (vl_api_unbind_sock_reply_t * mp)
 
   clib_spinlock_lock (&vcm->sessions_lockp);
   rv = vppcom_session_at_index (vcm->bind_session_index, &session);
-  if (PREDICT_FALSE (rv))
+  if (rv == VPPCOM_OK)
     {
-      if (VPPCOM_DEBUG > 1)
-	clib_warning ("[%d] invalid session, sid (%d) has been closed!",
-		      vcm->my_pid, vcm->bind_session_index);
-    }
+      if ((VPPCOM_DEBUG > 1) && (mp->retval))
+	clib_warning ("[%d] unbind failed: %U", vcm->my_pid, format_api_error,
+		      ntohl (mp->retval));
 
-  if (mp->retval)
-    clib_warning ("[%d] unbind failed: %U", vcm->my_pid, format_api_error,
-		  ntohl (mp->retval));
-
-  vcm->bind_session_index = ~0;
-  session->state = STATE_START;
+      vcm->bind_session_index = ~0;
+      session->state = STATE_START;
+    }
   clib_spinlock_unlock (&vcm->sessions_lockp);
 }
 
@@ -993,6 +989,7 @@ vl_api_accept_session_t_handler (vl_api_accept_session_t * mp)
 					       unix_shared_memory_queue_t *);
   session->state = STATE_ACCEPT;
   session->is_cut_thru = 0;
+  session->is_server = 1;
   session->port = ntohs (mp->port);
   session->is_ip4 = mp->is_ip4;
   clib_memcpy (session->ip, mp->ip, sizeof (session->ip));
@@ -1277,10 +1274,6 @@ vppcom_session_disconnect (u32 session_index)
 		      vcm->my_pid, vppcom_retval_str (rv), rv);
       return rv;
     }
-
-  clib_spinlock_lock (&vcm->sessions_lockp);
-  pool_put_index (vcm->sessions, session_index);
-  clib_spinlock_unlock (&vcm->sessions_lockp);
   return VPPCOM_OK;
 }
 
@@ -1853,14 +1846,31 @@ vppcom_session_close (uint32_t session_index)
   if (session->is_cut_thru)
     {
       if (session->is_server)
-	rv = vppcom_session_unbind_cut_thru (session);
+	{
+	  rv = vppcom_session_unbind_cut_thru (session);
+	  if ((VPPCOM_DEBUG > 0) && (rv < 0))
+	    clib_warning ("[%d] unbind cut-thru (session %d) failed, "
+			  "rv = %s (%d)",
+			  vcm->my_pid, session_index,
+			  vppcom_retval_str (rv), rv);
+	}
+    }
+  else if (session->is_server)
+    {
+      rv = vppcom_session_unbind (session_index);
+      if ((VPPCOM_DEBUG > 0) && (rv < 0))
+	clib_warning ("[%d] unbind (session %d) failed, rv = %s (%d)",
+		      vcm->my_pid, session_index, vppcom_retval_str (rv), rv);
     }
   else
     {
-      rv = (session->is_server) ?
-	vppcom_session_unbind (session_index) :
-	vppcom_session_disconnect (session_index);
+      rv = vppcom_session_disconnect (session_index);
+      if ((VPPCOM_DEBUG > 0) && (rv < 0))
+	clib_warning ("[%d] disconnect (session %d) failed, rv = %s (%d)",
+		      vcm->my_pid, session_index, vppcom_retval_str (rv), rv);
     }
+  if (rv < 0)
+    return rv;
 
   clib_spinlock_lock (&vcm->sessions_lockp);
   pool_put_index (vcm->sessions, session_index);
diff --git a/test/scripts/socket_test.sh b/test/scripts/socket_test.sh
index 8c3f20a2..1573b48a 100755
--- a/test/scripts/socket_test.sh
+++ b/test/scripts/socket_test.sh
@@ -9,7 +9,7 @@ vpp_shm_dir="/dev/shm/"
 vpp_run_dir="/run/vpp"
 lib64_dir="$WS_ROOT/build-root/install-vpp-native/vpp/lib64/"
 lib64_debug_dir="$WS_ROOT/build-root/install-vpp_debug-native/vpp/lib64/"
-dpdk_devbind="$WS_ROOT/build-root/install-vpp-native/dpdk/share/dpdk/usertools/dpdk-devbind.py"
+dpdk_devbind="/usr/share/dpdk/usertools/dpdk-devbind.py"
 docker_vpp_dir="/vpp/"
 docker_app_dir="/vpp/"
 docker_lib64_dir="/vpp-lib64/"
@@ -302,6 +302,13 @@ if [[ $run_test =~ .*"_vcl" ]] && [ $iperf3 -eq 1 ] ; then
     env_test_failed="true"
 fi
 
+if [ -n "$mult_host"] && [ ! -f "$dpdk_devbind" ] ; then
+    echo "ERROR: Can't find dpdk-devbind.py!"
+    echo "       Run \"cd \$WS_ROOT; make dpdk-install-dev\" to install it."
+    echo
+    env_test_failed="true"
+fi
+
 if [ -n "$env_test_failed" ] ; then
     exit 1
 fi
@@ -458,8 +465,8 @@ write_script_header() {
     echo "$bash_header" > $1
     echo -e "#\n# $1 generated on $(date)\n#" >> $1
     if [ $leave_tmp_files -eq 0 ] ; then
-        if [ -n "$multi_host" ] ; then
-            echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file; sudo $dpdk_devbind -e $vpp_eth_kernel_driver $vpp_eth_pci_id; sudo ifup $vpp_eth_name\" $trap_signals" >> $1
+        if [ -n "$multi_host" ] && [[ "$3" == VPP* ]] ; then
+            echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file; sudo $dpdk_devbind -b $vpp_eth_kernel_driver $vpp_eth_pci_id; sudo ifup $vpp_eth_name\" $trap_signals" >> $1
         else
             echo "trap \"rm -f $1 $2 $tmp_vpp_exec_file\" $trap_signals" >> $1
         fi