Func vhost tests with needed changes in perf tests

Change-Id: I8c44448ca77f0fbd7ce074b8eef10b5ba491747d
Signed-off-by: Jan Gelety <jgelety@cisco.com>

9 files changed, 466 insertions, 46 deletions

diff --git a/bootstrap-centos.sh b/bootstrap-centos.sh
index 6bec72060c..b0b9562274 100755
--- a/bootstrap-centos.sh
+++ b/bootstrap-centos.sh
@@ -33,7 +33,7 @@ VIRL_RELEASE=$(cat ${SCRIPT_DIR}/VIRL_RELEASE_CENTOS)
 
 SSH_OPTIONS="-i ${VIRL_PKEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o LogLevel=error"
 
-TEST_GROUPS=("l2bd,dhcp,gre,honeycomb,l2xc,lisp,softwire" "cop,telemetry,ipsec,ipv6,rpf,tap,vrf" "fds,iacl,ipv4,policer,vlan,vxlan")
+TEST_GROUPS=("l2bd,dhcp,gre,honeycomb,l2xc,lisp,softwire" "cop,telemetry,ipsec,ipv6,rpf,tap,vrf" "fds,iacl,ipv4,policer,vlan,vxlan,vhost")
 SUITE_PATH="tests.func"
 
 # Create tmp dir
diff --git a/bootstrap-ubuntu.sh b/bootstrap-ubuntu.sh
index dfd20a45cd..3e130f83f9 100755
--- a/bootstrap-ubuntu.sh
+++ b/bootstrap-ubuntu.sh
@@ -35,7 +35,7 @@ VIRL_RELEASE=$(cat ${SCRIPT_DIR}/VIRL_RELEASE_UBUNTU)
 
 SSH_OPTIONS="-i ${VIRL_PKEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o LogLevel=error"
 
-TEST_GROUPS=("l2bd,dhcp,gre,honeycomb,l2xc,lisp,softwire" "cop,telemetry,ipsec,ipv6,rpf,tap,vrf" "fds,iacl,ipv4,policer,vlan,vxlan")
+TEST_GROUPS=("l2bd,dhcp,gre,honeycomb,l2xc,lisp,softwire" "cop,telemetry,ipsec,ipv6,rpf,tap,vrf" "fds,iacl,ipv4,policer,vlan,vxlan,vhost")
 SUITE_PATH="tests.func"
 
 # Create tmp dir
diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index 244ece2258..7a93a1ba93 100644
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -34,6 +34,8 @@ class QemuUtils(object):
         self._qmp_sock = '/tmp/qmp{0}.sock'.format(self._qemu_id)
         # QEMU Guest Agent socket
         self._qga_sock = '/tmp/qga{0}.sock'.format(self._qemu_id)
+        # QEMU PID file
+        self._pid_file = '/tmp/qemu{0}.pid'.format(self._qemu_id)
         self._qemu_opt = {}
         # Default 1 CPU.
         self._qemu_opt['smp'] = '-smp 1,sockets=1,cores=1,threads=1'
@@ -41,11 +43,11 @@ class QemuUtils(object):
         # management interface.
         self._qemu_opt['options'] = '-cpu host -daemonize -enable-kvm ' \
             '-machine pc,accel=kvm,usb=off,mem-merge=off ' \
-            '-net nic,macaddr=52:54:00:00:00:{0:02x} -balloon none'\
+            '-net nic,macaddr=52:54:00:00:{0:02x}:ff -balloon none'\
             .format(self._qemu_id)
-        self._qemu_opt['ssh_fwd_port'] = 10022
+        self._qemu_opt['ssh_fwd_port'] = 10021 + qemu_id
         # Default serial console port
-        self._qemu_opt['serial_port'] = 4556
+        self._qemu_opt['serial_port'] = 4555 + qemu_id
         # Default 512MB virtual RAM
         self._qemu_opt['mem_size'] = 512
         # Default huge page mount point, required for Vhost-user interfaces.
@@ -205,8 +207,8 @@ class QemuUtils(object):
             chardev += ',server'
         self._qemu_opt['options'] += chardev
         # Create Vhost-user network backend.
-        netdev = ' -netdev vhost-user,id=vhost{0},chardev=char{0},'\
-            'queues={1}'.format(self._vhost_id, self._qemu_opt['queues'])
+        netdev = (' -netdev vhost-user,id=vhost{0},chardev=char{0},queues={1}'
+                  .format(self._vhost_id, self._qemu_opt['queues']))
         self._qemu_opt['options'] += netdev
         # If MAC is not specified use auto-generated MAC address based on
         # template 52:54:00:00:<qemu_id>:<vhost_id>, e.g. vhost1 MAC of QEMU
@@ -525,12 +527,14 @@ class QemuUtils(object):
             '-device isa-serial,chardev=qga0'.format(self._qga_sock)
         # Graphic setup
         graphic = '-monitor none -display none -vga none'
+        # PID file
+        pid = '-pidfile {}'.format(self._pid_file)
 
         # Run QEMU
-        cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9}'.format(
+        cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'.format(
             self._qemu_bin, self._qemu_opt.get('smp'), mem, ssh_fwd,
             self._qemu_opt.get('options'),
-            drive, qmp, serial, qga, graphic)
+            drive, qmp, serial, qga, graphic, pid)
         (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd, timeout=300)
         if int(ret_code) != 0:
             logger.debug('QEMU start failed {0}'.format(stderr))
@@ -541,7 +545,7 @@ class QemuUtils(object):
         try:
             self._wait_until_vm_boot()
         except (RuntimeError, SSHTimeout):
-            self.qemu_kill()
+            self.qemu_kill_all()
             self.qemu_clear_socks()
             raise
         # Update interface names in VM node dict
@@ -578,9 +582,18 @@ class QemuUtils(object):
 
     def qemu_kill(self):
         """Kill qemu process."""
-        # TODO: add PID storage so that we can kill specific PID
         # Note: in QEMU start phase there are 3 QEMU processes because we
         # daemonize QEMU
+        self._ssh.exec_command_sudo('chmod +r {}'.format(self._pid_file))
+        self._ssh.exec_command_sudo('kill -SIGKILL $(cat {})'
+                                    .format(self._pid_file))
+        # Delete PID file
+        cmd = 'rm -f {}'.format(self._pid_file)
+        self._ssh.exec_command_sudo(cmd)
+
+    def qemu_kill_all(self, node=None):
+        if node:
+            self.qemu_set_node(node)
         self._ssh.exec_command_sudo('pkill -SIGKILL qemu')
 
     def qemu_clear_socks(self):
diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py
index 961f7e2ee8..0009bde59b 100644
--- a/resources/libraries/python/ssh.py
+++ b/resources/libraries/python/ssh.py
@@ -56,40 +56,46 @@ class SSH(object):
 
         return hash(frozenset([node['host'], node['port']]))
 
-    def connect(self, node):
+    def connect(self, node, attempts=5):
         """Connect to node prior to running exec_command or scp.
 
         If there already is a connection to the node, this method reuses it.
         """
-        self._node = node
-        node_hash = self._node_hash(node)
-        if node_hash in SSH.__existing_connections:
-            self._ssh = SSH.__existing_connections[node_hash]
-            logger.debug('reusing ssh: {0}'.format(self._ssh))
-        else:
-            start = time()
-            pkey = None
-            if 'priv_key' in node:
-                pkey = RSAKey.from_private_key(
-                    StringIO.StringIO(node['priv_key']))
-
-            self._ssh = paramiko.SSHClient()
-            self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-            self._ssh.connect(node['host'], username=node['username'],
-                              password=node.get('password'), pkey=pkey,
-                              port=node['port'])
-
-            self._ssh.get_transport().set_keepalive(10)
-
-            SSH.__existing_connections[node_hash] = self._ssh
-
-            logger.trace('connect took {} seconds'.format(time() - start))
-            logger.debug('new ssh: {0}'.format(self._ssh))
-
-        logger.debug('Connect peer: {0}'.
-                     format(self._ssh.get_transport().getpeername()))
-        logger.debug('Connections: {0}'.format(str(SSH.__existing_connections)))
+        try:
+            self._node = node
+            node_hash = self._node_hash(node)
+            if node_hash in SSH.__existing_connections:
+                self._ssh = SSH.__existing_connections[node_hash]
+                logger.debug('reusing ssh: {0}'.format(self._ssh))
+            else:
+                start = time()
+                pkey = None
+                if 'priv_key' in node:
+                    pkey = RSAKey.from_private_key(
+                        StringIO.StringIO(node['priv_key']))
+
+                self._ssh = paramiko.SSHClient()
+                self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+                self._ssh.connect(node['host'], username=node['username'],
+                                  password=node.get('password'), pkey=pkey,
+                                  port=node['port'])
+
+                self._ssh.get_transport().set_keepalive(10)
+
+                SSH.__existing_connections[node_hash] = self._ssh
+
+                logger.trace('connect took {} seconds'.format(time() - start))
+                logger.debug('new ssh: {0}'.format(self._ssh))
+
+            logger.debug('Connect peer: {0}'.
+                         format(self._ssh.get_transport().getpeername()))
+            logger.debug('Connections: {0}'.format(str(SSH.__existing_connections)))
+        except:
+            if attempts > 0:
+                self._reconnect(attempts-1)
+            else:
+                raise
 
     def disconnect(self, node):
         """Close SSH connection to the node.
@@ -104,12 +110,12 @@ class SSH(object):
             ssh = SSH.__existing_connections.pop(node_hash)
             ssh.close()
 
-    def _reconnect(self):
+    def _reconnect(self, attempts=0):
         """Close the SSH connection and open it again."""
 
         node = self._node
         self.disconnect(node)
-        self.connect(node)
+        self.connect(node, attempts)
         logger.debug('Reconnecting peer done: {}'.
                      format(self._ssh.get_transport().getpeername()))
 
diff --git a/resources/libraries/robot/performance.robot b/resources/libraries/robot/performance.robot
index 0d80436e99..39ea14cd7a 100644
--- a/resources/libraries/robot/performance.robot
+++ b/resources/libraries/robot/performance.robot
@@ -1854,7 +1854,7 @@
 | | ... | ${eth0_mac} | ${eth1_mac} | ${skip}=${6} | ${count}=${5}
 | | ... | ${qemu_id}=${1}
 | | ...
-| | Import Library | resources.libraries.python.QemuUtils
+| | Import Library | resources.libraries.python.QemuUtils | qemu_id=${qemu_id}
 | | ... | WITH NAME | ${vm_name}
 | | ${serial_port}= | Evaluate | ${qemu_id} + ${4555}
 | | Run keyword | ${vm_name}.Qemu Set Serial Port | ${serial_port}
@@ -2084,7 +2084,7 @@
 | | | Dpdk Testpmd Stop | ${vm}
 | | | Run Keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | | Run Keyword | ${vm_name}.Qemu Clear Socks
-| | | Run Keyword If | '${index}' == '${vms_number}' | ${vm_name}.Qemu Kill
+| | | Run Keyword If | '${index}' == '${vms_number}' | ${vm_name}.Qemu Kill All
 
 | Guest VM Teardown
 | | [Documentation]
@@ -2108,7 +2108,7 @@
 | | | ${index}= | Evaluate | ${index} + 1
 | | | Run Keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | | Run Keyword | ${vm_name}.Qemu Clear Socks
-| | | Run Keyword If | '${index}' == '${vms_number}' | ${vm_name}.Qemu Kill
+| | | Run Keyword If | '${index}' == '${vms_number}' | ${vm_name}.Qemu Kill All
 
 | Lisp IPv4 forwarding initialized in a 3-node circular topology
 | | [Documentation] | Custom setup of IPv4 addresses on all DUT nodes and TG \
diff --git a/resources/libraries/robot/vxlan.robot b/resources/libraries/robot/vxlan.robot
index 0b53c85494..36a31584d0 100644
--- a/resources/libraries/robot/vxlan.robot
+++ b/resources/libraries/robot/vxlan.robot
@@ -94,3 +94,50 @@
 | | Set Test Variable | ${dut1s_vlan_index}
 | | Set Test Variable | ${dut2s_vlan_name}
 | | Set Test Variable | ${dut2s_vlan_index}
+
+| Send VXLAN receive VXLAN Packet
+| | [Documentation] | Send VXLAN encapsulated Ethernet frame and check \
+| | ... | received one.
+| | ...
+| | ... | *Arguments:*
+| | ... | - tg_node - Node where to run traffic script. Type: dictionary
+| | ... | - tx_if - Interface from where send VXLAN packet. Type: string
+| | ... | - rx_if - Interface where receive VXLAN packet. Type: string
+| | ... | - tx_src_mac - Source MAC address of sent packet. Type: string
+| | ... | - tx_dst_mac - Destination MAC address of sent packet. Type: string
+| | ... | - tx_src_ip - Source IP address of sent VXLAN packet. Type: string
+| | ... | - tx_dst_ip - Destination IP address of sent VXLAN packet.
+| | ... |   Type: string
+| | ... | - tx_vni - VNI of sent VXLAN packet. Type: string
+| | ... | - rx_src_ip - Source IP address of received VXLAN packet. Type: string
+| | ... | - rx_dst_ip - Destination IP address of received VXLAN packet.
+| | ... |   Type: string
+| | ... | - rx_vni - VNI of received VXLAN packet. Type: string
+| | ...
+| | ... | *Return:*
+| | ... | - No value returned
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Send VXLAN receive VXLAN Packet \| ${tg_node} \| port4 \| port4  \
+| | ... | \| fa:16:3e:6d:f9:c5 \| fa:16:3e:e6:6d:9a \| 192.168.0.1  \
+| | ... | \| 192.168.0.2 \| ${101} \| 192.168.0.2 \| 192.168.0.1 \| ${102} \|
+| | ...
+| | [Arguments] | ${tg_node} | ${tx_if} | ${rx_if}
+| | ... | ${tx_src_mac} | ${tx_dst_mac}
+| | ... | ${tx_src_ip} | ${tx_dst_ip} | ${tx_vni}
+| | ... | ${rx_src_ip} | ${rx_dst_ip} | ${rx_vni}
+| | ${tx_if_name}= | Get interface name | ${tg_node} | ${tx_if}
+| | ${rx_if_name}= | Get interface name | ${tg_node} | ${rx_if}
+| | ${args}= | Catenate
+| | ... | --tx_if ${tx_if_name}
+| | ... | --rx_if ${rx_if_name}
+| | ... | --tx_src_mac ${tx_src_mac}
+| | ... | --tx_dst_mac ${tx_dst_mac}
+| | ... | --tx_src_ip ${tx_src_ip}
+| | ... | --tx_dst_ip ${tx_dst_ip}
+| | ... | --tx_vni ${tx_vni}
+| | ... | --rx_src_ip ${rx_src_ip}
+| | ... | --rx_dst_ip ${rx_dst_ip}
+| | ... | --rx_vni ${rx_vni}
+| | Run Traffic Script On Node | send_vxlan_check_vxlan.py | ${tg_node} | ${args}
diff --git a/resources/traffic_scripts/send_vxlan_check_vxlan.py b/resources/traffic_scripts/send_vxlan_check_vxlan.py
new file mode 100644
index 0000000000..3bf273d17a
--- /dev/null
+++ b/resources/traffic_scripts/send_vxlan_check_vxlan.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Traffic script that sends an IP ICMPv4/ICMPv6 packet from one interface to
+the other one. Dot1q or Dot1ad tagging of the ethernet frame can be set.
+"""
+
+import sys
+
+import vxlan
+
+from scapy.layers.inet import IP, UDP, Raw
+from scapy.layers.l2 import Ether
+from resources.libraries.python.PacketVerifier import RxQueue, TxQueue
+from resources.libraries.python.TrafficScriptArg import TrafficScriptArg
+
+
+def main():
+    """Send IP ICMPv4/ICMPv6 packet from one traffic generator interface to
+    the other one. Dot1q or Dot1ad tagging of the ethernet frame can be set.
+    """
+    args = TrafficScriptArg(['tx_src_mac', 'tx_dst_mac', 'tx_src_ip',
+                             'tx_dst_ip', 'tx_vni', 'rx_src_ip', 'rx_dst_ip',
+                             'rx_vni'])
+
+    tx_if = args.get_arg('tx_if')
+    rx_if = args.get_arg('rx_if')
+    tx_src_mac = args.get_arg('tx_src_mac')
+    tx_dst_mac = args.get_arg('tx_dst_mac')
+    tx_src_ip = args.get_arg('tx_src_ip')
+    tx_dst_ip = args.get_arg('tx_dst_ip')
+    tx_vni = args.get_arg('tx_vni')
+    rx_src_ip = args.get_arg('rx_src_ip')
+    rx_dst_ip = args.get_arg('rx_dst_ip')
+    rx_vni = args.get_arg('rx_vni')
+
+    rxq = RxQueue(rx_if)
+    txq = TxQueue(tx_if)
+
+    sent_packets = []
+
+    tx_pkt_p = (Ether(src='02:00:00:00:00:01', dst='02:00:00:00:00:02') /
+                IP(src='192.168.1.1', dst='192.168.1.2') /
+                UDP(sport=12345, dport=1234) /
+                Raw('rew data'))
+
+    pkt_raw = (Ether(src=tx_src_mac, dst=tx_dst_mac) /
+               IP(src=tx_src_ip, dst=tx_dst_ip) /
+               UDP(sport=23456) /
+               vxlan.VXLAN(vni=int(tx_vni)) /
+               tx_pkt_p)
+
+    # Send created packet on one interface and receive on the other
+    sent_packets.append(pkt_raw)
+    txq.send(pkt_raw)
+
+    ether = rxq.recv(2, ignore=sent_packets)
+
+    # Check whether received packet contains layers Ether, IP and VXLAN
+    if ether is None:
+        raise RuntimeError('Packet Rx timeout')
+    ip = ether.payload
+
+    if ip.src != rx_src_ip:
+        raise RuntimeError('IP src mismatch {} != {}'.format(ip.src, rx_src_ip))
+    if ip.dst != rx_dst_ip:
+        raise RuntimeError('IP dst mismatch {} != {}'.format(ip.dst, rx_dst_ip))
+    if ip.payload.dport != 4789:
+        raise RuntimeError('VXLAN UDP port mismatch {} != {}'.
+                           format(ip.payload.dport, 4789))
+    vxlan_pkt = ip.payload.payload
+
+    if int(vxlan_pkt.vni) != int(rx_vni):
+        raise RuntimeError('vxlan mismatch')
+    rx_pkt_p = vxlan_pkt.payload
+
+    if rx_pkt_p.src != tx_pkt_p.src:
+        raise RuntimeError('RX encapsulated MAC src mismatch {} != {}'.
+                           format(rx_pkt_p.src, tx_pkt_p.src))
+    if rx_pkt_p.dst != tx_pkt_p.dst:
+        raise RuntimeError('RX encapsulated MAC dst mismatch {} != {}'.
+                           format(rx_pkt_p.dst, tx_pkt_p.dst))
+    if rx_pkt_p['IP'].src != tx_pkt_p['IP'].src:
+        raise RuntimeError('RX encapsulated IP src mismatch {} != {}'.
+                           format(rx_pkt_p['IP'].src, tx_pkt_p['IP'].src))
+    if rx_pkt_p['IP'].dst != tx_pkt_p['IP'].dst:
+        raise RuntimeError('RX encapsulated IP dst mismatch {} != {}'.
+                           format(rx_pkt_p['IP'].dst, tx_pkt_p['IP'].dst))
+
+    # TODO: verify inner Ether()
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    main()
diff --git a/resources/traffic_scripts/vxlan.py b/resources/traffic_scripts/vxlan.py
new file mode 100644
index 0000000000..bf86f179a8
--- /dev/null
+++ b/resources/traffic_scripts/vxlan.py
@@ -0,0 +1,17 @@
+from scapy.fields import BitField, XByteField, X3BytesField
+from scapy.packet import Packet, bind_layers
+from scapy.layers.l2 import Ether
+from scapy.layers.inet import UDP
+
+
+class VXLAN(Packet):
+    name = "VXLAN"
+    fields_desc = [BitField("flags", 0x08000000, 32),
+                   X3BytesField("vni", 0),
+                   XByteField("reserved", 0x00)]
+
+    def mysummary(self):
+        return self.sprintf("VXLAN (vni=%VXLAN.vni%)")
+
+bind_layers(UDP, VXLAN, dport=4789)
+bind_layers(VXLAN, Ether)
diff --git a/tests/func/vhost/eth2p-ethip4-l2bdbase-vhost-client-reconnect-2vm-func.robot b/tests/func/vhost/eth2p-ethip4-l2bdbase-vhost-client-reconnect-2vm-func.robot
new file mode 100644
index 0000000000..8239a465a0
--- /dev/null
+++ b/tests/func/vhost/eth2p-ethip4-l2bdbase-vhost-client-reconnect-2vm-func.robot
@@ -0,0 +1,231 @@
+# Copyright (c) 2017 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Resource | resources/libraries/robot/default.robot
+| Resource | resources/libraries/robot/bridge_domain.robot
+| Resource | resources/libraries/robot/testing_path.robot
+| Resource | resources/libraries/robot/qemu.robot
+| Resource | resources/libraries/robot/vxlan.robot
+| Library  | resources.libraries.python.Trace
+| Force Tags | HW_ENV | VM_ENV | 3_NODE_DOUBLE_LINK_TOPO | VPP_VM_ENV
+| Test Setup | Func Test Setup
+| Test Teardown | Run Keywords
+| ... | resources.libraries.python.QemuUtils.Qemu Kill All | ${dut_node} | AND
+| ... | Func Test Teardown
+| Documentation | *Vhost-User Interface Traffic Tests*
+| ... | *[Top] Network Topologies:* TG=DUT1 2-node topology with two links
+| ... | between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-VXLAN-ETH-IP on TG-DUT link.
+| ... | ETH-IP on VirtualEthernet-VM interface.
+| ... | *[Cfg] DUT configuration:* On DUT is running 2 VM with 2 vhost-user
+| ... | interface on each VM. DUT is configured with VXLAN and vhost-user
+| ... | interfaces in bridge-domain (L2BD).
+| ... | *[Cfg] VM configuration:* VM has both vhost-user interfaces added into
+| ... | Linux Bridge.
+| ... | *[Ver] TG verification:*
+| ... | VXLAN packet is send to DUT where is decapsulated and send bridged to
+| ... | vhost-user inteface. VM forwards frame to its second interface and VPP
+| ... | encapsulates it to another VXLAN tunnel. Packets
+| ... | are sent and received by TG on link to DUT.
+
+*** Variables ***
+| ${tg_if1_ip}= | 192.168.0.1
+| ${dut_if1_ip}= | 192.168.0.2
+| ${prefix_length}= | ${24}
+
+| ${sock_vm1_1}= | /tmp/sock1
+| ${sock_vm1_2}= | /tmp/sock2
+| ${sock_vm2_1}= | /tmp/sock3
+| ${sock_vm2_2}= | /tmp/sock4
+
+*** Test Cases ***
+| TC01: Qemu reconnects to VPPs vhost-user when Qemu is killed and restarted
+| | Given Path for 2-node testing is set
+| | ... | ${nodes['TG']} | ${nodes['DUT1']} | ${nodes['TG']}
+| | And Interfaces in 2-node path are up
+| | And Set Interface Address | ${dut_node} | ${dut_to_tg_if1} | ${dut_if1_ip}
+| | ... | ${prefix_length}
+| | And Add IP Neighbor | ${dut_node} | ${dut_to_tg_if1} | ${tg_if1_ip}
+| | ... | ${tg_to_dut_if1_mac}
+| | ${vxlan1}= | And Create VXLAN interface | ${dut_node} | ${101}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan2}= | And Create VXLAN interface | ${dut_node} | ${102}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan3}= | And Create VXLAN interface | ${dut_node} | ${103}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan4}= | And Create VXLAN interface | ${dut_node} | ${104}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | And Set Interface State | ${dut_node} | ${vxlan1} | up
+| | And Set Interface State | ${dut_node} | ${vxlan2} | up
+| | And Set Interface State | ${dut_node} | ${vxlan3} | up
+| | And Set Interface State | ${dut_node} | ${vxlan4} | up
+| | ${vhost_if1}= | And Vpp Create Vhost User Interface
+| | ... | ${dut_node} | ${sock_vm1_1}
+| | ${vhost_if2}= | And Vpp Create Vhost User Interface
+| | ... | ${dut_node} | ${sock_vm1_2}
+| | ${vhost_if3}= | And Vpp Create Vhost User Interface
+| | ... | ${dut_node} | ${sock_vm2_1}
+| | ${vhost_if4}= | And Vpp Create Vhost User Interface
+| | ... | ${dut_node} | ${sock_vm2_2}
+| | And Set Interface State | ${dut_node} | ${vhost_if1} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if2} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if3} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if4} | up
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${101} | ${vxlan1}
+| | ... | ${vhost_if1}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${102} | ${vxlan2}
+| | ... | ${vhost_if2}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${103} | ${vxlan3}
+| | ... | ${vhost_if3}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${104} | ${vxlan4}
+| | ... | ${vhost_if4}
+| | And Setup QEMU Vhost and Run VM | ${dut_node} | ${sock_vm1_1}
+| | ... | ${sock_vm1_2} | ${1}
+| | And Setup QEMU Vhost and Run VM | ${dut_node} | ${sock_vm2_1}
+| | ... | ${sock_vm2_2} | ${2}
+| | And Check traffic through VM
+| | When Run keyword | qemu-1.Qemu Kill
+| | ${vm1}= | And Run Keyword | qemu-1.Qemu Start
+| | ${vhost_int_1}= | And Get Vhost User If Name By Sock | ${vm1}
+| | ... | ${sock_vm1_1}
+| | ${vhost_int_2}= | And Get Vhost User If Name By Sock | ${vm1}
+| | ... | ${sock_vm1_2}
+| | And Linux Add Bridge | ${vm1} | br0 | ${vhost_int_1} | ${vhost_int_2}
+| | And Set Interface State | ${vm1} | ${vhost_int_1} | up | if_type=name
+| | And Set Interface State | ${vm1} | ${vhost_int_2} | up | if_type=name
+| | Then Check traffic through VM
+
+
+| TC02: VPP reconnects to Qemu vhost-user when VPP is restarted and reconfigured
+| | [Tags] | EXPECTED_FAILING
+| | [Documentation]
+| | ... | *Failing:* Qemu doesn't support reconnect prior to version 2.7.
+| | Given Path for 2-node testing is set
+| | ... | ${nodes['TG']} | ${nodes['DUT1']} | ${nodes['TG']}
+| | And Interfaces in 2-node path are up
+| | And Set Interface Address | ${dut_node} | ${dut_to_tg_if1} | ${dut_if1_ip}
+| | ... | ${prefix_length}
+| | And Add IP Neighbor | ${dut_node} | ${dut_to_tg_if1} | ${tg_if1_ip}
+| | ... | ${tg_to_dut_if1_mac}
+| | ${vxlan1}= | And Create VXLAN interface | ${dut_node} | ${101}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan2}= | And Create VXLAN interface | ${dut_node} | ${102}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan3}= | And Create VXLAN interface | ${dut_node} | ${103}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan4}= | And Create VXLAN interface | ${dut_node} | ${104}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | And Set Interface State | ${dut_node} | ${vxlan1} | up
+| | And Set Interface State | ${dut_node} | ${vxlan2} | up
+| | And Set Interface State | ${dut_node} | ${vxlan3} | up
+| | And Set Interface State | ${dut_node} | ${vxlan4} | up
+| | ${vhost_if1}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm1_1}
+| | ${vhost_if2}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm1_2}
+| | ${vhost_if3}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm2_1}
+| | ${vhost_if4}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm2_2}
+| | And Set Interface State | ${dut_node} | ${vhost_if1} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if2} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if3} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if4} | up
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${101} | ${vxlan1}
+| | ... | ${vhost_if1}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${102} | ${vxlan2}
+| | ... | ${vhost_if2}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${103} | ${vxlan3}
+| | ... | ${vhost_if3}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${104} | ${vxlan4}
+| | ... | ${vhost_if4}
+| | And Setup QEMU Vhost and Run VM | ${dut_node} | ${sock_vm1_1}
+| | ... | ${sock_vm1_2} | ${1}
+| | And Setup QEMU Vhost and Run VM | ${dut_node} | ${sock_vm2_1}
+| | ... | ${sock_vm2_2} | ${2}
+| | And Check traffic through VM
+| | And Check VPP PID in Teardown
+| | When Setup All Duts ${nodes}
+| | And Save VPP PIDs
+| | And Interfaces in 2-node path are up
+| | And Set Interface Address | ${dut_node} | ${dut_to_tg_if1} | ${dut_if1_ip}
+| | ... | ${prefix_length}
+| | And Add IP Neighbor | ${dut_node} | ${dut_to_tg_if1} | ${tg_if1_ip}
+| | ... | ${tg_to_dut_if1_mac}
+| | ${vxlan1}= | And Create VXLAN interface | ${dut_node} | ${101}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan2}= | And Create VXLAN interface | ${dut_node} | ${102}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan3}= | And Create VXLAN interface | ${dut_node} | ${103}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | ${vxlan4}= | And Create VXLAN interface | ${dut_node} | ${104}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip}
+| | And Set Interface State | ${dut_node} | ${vxlan1} | up
+| | And Set Interface State | ${dut_node} | ${vxlan2} | up
+| | And Set Interface State | ${dut_node} | ${vxlan3} | up
+| | And Set Interface State | ${dut_node} | ${vxlan4} | up
+| | ${vhost_if1}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm1_1}
+| | ${vhost_if2}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm1_2}
+| | ${vhost_if3}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm2_1}
+| | ${vhost_if4}= | And Vpp Create Vhost User Interface | ${dut_node}
+| | ... | ${sock_vm2_2}
+| | And Set Interface State | ${dut_node} | ${vhost_if1} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if2} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if3} | up
+| | And Set Interface State | ${dut_node} | ${vhost_if4} | up
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${101} | ${vxlan1}
+| | ... | ${vhost_if1}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${102} | ${vxlan2}
+| | ... | ${vhost_if2}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${103} | ${vxlan3}
+| | ... | ${vhost_if3}
+| | And Vpp Add L2 Bridge Domain | ${dut_node} | ${104} | ${vxlan4}
+| | ... | ${vhost_if4}
+| | Then Check traffic through VM
+
+
+*** Keywords ***
+| Setup QEMU Vhost and Run VM
+| | [Arguments] | ${dut_node} | ${sock1} | ${sock2} | ${qemu_id}
+| | Import Library | resources.libraries.python.QemuUtils | qemu_id=${qemu_id}
+| | ... | WITH NAME | qemu-${qemu_id}
+| | ${q_add_vhost}= | Replace Variables | qemu-${qemu_id}.Qemu Add Vhost User If
+| | ${q_set_node}= | Replace Variables | qemu-${qemu_id}.Qemu Set Node
+| | ${q_start}= | Replace Variables | qemu-${qemu_id}.Qemu Start
+| | Run keyword | ${q_set_node} | ${dut_node}
+| | Run keyword | ${q_add_vhost} | ${sock1}
+| | Run keyword | ${q_add_vhost} | ${sock2}
+| | ${vm}= | Run keyword | ${q_start}
+| | ${vhost1}= | Get Vhost User If Name By Sock | ${vm} | ${sock1}
+| | ${vhost2}= | Get Vhost User If Name By Sock | ${vm} | ${sock2}
+| | Linux Add Bridge | ${vm} | br0 | ${vhost1} | ${vhost2}
+| | Set Interface State | ${vm} | ${vhost1} | up | if_type=name
+| | Set Interface State | ${vm} | ${vhost2} | up | if_type=name
+| | Set Test Variable | ${qemu-${qemu_id}} | ${vm}
+
+| Check traffic through VM
+| | [Documentation] | Send VXLAN traffic through both configured VMs.
+| | Send VXLAN receive VXLAN Packet | ${tg_node}
+| | ... | ${tg_to_dut_if1} | ${tg_to_dut_if1}
+| | ... | ${tg_to_dut_if1_mac} | ${dut_to_tg_if1_mac}
+| | ... | ${tg_if1_ip} | ${dut_if1_ip} | ${101}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip} | ${102}
+| | Send VXLAN receive VXLAN Packet | ${tg_node}
+| | ... | ${tg_to_dut_if1} | ${tg_to_dut_if1}
+| | ... | ${tg_to_dut_if1_mac} | ${dut_to_tg_if1_mac}
+| | ... | ${tg_if1_ip} | ${dut_if1_ip} | ${103}
+| | ... | ${dut_if1_ip} | ${tg_if1_ip} | ${104}