CSIT-366 IPv4dp - baseline vhost-user

- Add test cases for vhost-user testing with
  NIC(X520)-to-VM(testpmd,virtio)

Change-Id: I598e70221b939b2b08aa324d2c98059eee8b6857
Signed-off-by: pmikus <pmikus@cisco.com>

4 files changed, 883 insertions, 1 deletions

diff --git a/resources/libraries/python/DpdkUtil.py b/resources/libraries/python/DpdkUtil.py
index 5eea0d6c26..e7727c1101 100644
--- a/resources/libraries/python/DpdkUtil.py
+++ b/resources/libraries/python/DpdkUtil.py
@@ -62,6 +62,11 @@ class DpdkUtil(object):
         # Disable RSS (Receive Side Scaling).
         pmd_disable_rss = '--disable-rss '\
             if args.get('pmd_disable_rss', '') else ''
+        # Set the MAC address XX:XX:XX:XX:XX:XX of the peer port N
+        pmd_eth_peer_0 = '--eth-peer={} '.format(args['pmd_eth_peer_0'])\
+            if args.get('pmd_eth_peer_0', '') else ''
+        pmd_eth_peer_1 = '--eth-peer={} '.format(args['pmd_eth_peer_1'])\
+            if args.get('pmd_eth_peer_1', '') else ''
         # Set the hexadecimal bitmask of the cores running forwarding. Master
         # lcore=0 is reserved, so highest bit is set to 0.
         pmd_coremask = '--coremask={} '.format(\
@@ -86,6 +91,8 @@ class DpdkUtil(object):
             + pmd_portmask\
             + pmd_disable_hw_vlan\
             + pmd_disable_rss\
+            + pmd_eth_peer_0\
+            + pmd_eth_peer_1\
             + pmd_coremask\
             + pmd_nb_cores
         ssh = SSH()
diff --git a/resources/libraries/python/VhostUser.py b/resources/libraries/python/VhostUser.py
index 1a311e03fc..020df6100e 100644
--- a/resources/libraries/python/VhostUser.py
+++ b/resources/libraries/python/VhostUser.py
@@ -13,7 +13,7 @@
 
 """Vhost-user interfaces library."""
 
-from resources.libraries.python.VatExecutor import VatExecutor
+from resources.libraries.python.VatExecutor import VatExecutor, VatTerminal
 
 
 class VhostUser(object):
@@ -53,3 +53,25 @@ class VhostUser(object):
             if interface.get('socket') == socket:
                 return interface.get('name')
         return None
+
+    @staticmethod
+    def get_vhost_user_mac_by_sw_index(node, sw_if_index):
+        """Get Vhost-user l2_address for the given interface from actual
+        interface dump.
+
+        :param node: VPP node to get interface data from.
+        :param sw_if_index: Idx of the specific interface.
+        :type node: dict
+        :type sw_if_index: str
+        :return: l2_address of the given interface.
+        :rtype: str
+        """
+
+        with VatTerminal(node) as vat:
+            if_data = vat.vat_terminal_exec_cmd_from_template(
+                "interface_dump.vat")
+        for iface in if_data[0]:
+            if iface["sw_if_index"] == sw_if_index:
+                return ':'.join("%02x" % (b) for b in iface["l2_address"][:6])
+
+        return None
diff --git a/resources/libraries/robot/performance.robot b/resources/libraries/robot/performance.robot
index 579e2dbd92..6849e9820a 100644
--- a/resources/libraries/robot/performance.robot
+++ b/resources/libraries/robot/performance.robot
@@ -460,6 +460,133 @@
 | | ...           | count=${count}
 | | All Vpp Interfaces Ready Wait | ${nodes}
 
+| IPv4 forwarding with vhost initialized in a 3-node circular topology
+| | [Documentation]
+| | ... | Create vhost-user interfaces in VPP. Set UP state of all VPP
+| | ... | interfaces in path on nodes in 3-node circular topology. Create 2
+| | ... | FIB tables on each DUT with multipath routing. Assign pair of
+| | ... | Physical and Virtual interfaces on both nodes to each FIB table.
+| | ... | Setup IPv4 addresses with /30 prefix on DUT-TG links and /30 prefix
+| | ... | on DUT1-DUT2 link. Set routing on all DUT nodes in all FIB tables
+| | ... | with prefix /24 and next hop of neighbour IPv4 address. Setup
+| | ... | ARP on all VPP interfaces.
+| | ...
+| | ... | *Arguments:*
+| | ... | - sock1 - Sock path for first Vhost-User interface. Type: string
+| | ... | - sock2 - Sock path for second Vhost-User interface. Type: string
+| | ...
+| | ... | *Return:*
+| | ... | - No value returned
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| IPv4 forwarding with vhost initialized in a 3-node circular \
+| | ... | topology \| /tmp/sock1 \| /tmp/sock2 \|
+| | [Arguments] | ${sock1} | ${sock2}
+| | VPP interfaces in path are up in a 3-node circular topology
+| | VPP Vhost interfaces for L2BD forwarding are setup | ${dut1}
+| | ...                                                | ${sock1}
+| | ...                                                | ${sock2}
+| | ${dut1_vif1}= | Set Variable | ${vhost_if1}
+| | ${dut1_vif2}= | Set Variable | ${vhost_if2}
+| | Set Interface State | ${dut1} | ${dut1_vif1} | up
+| | Set Interface State | ${dut1} | ${dut1_vif2} | up
+| | VPP Vhost interfaces for L2BD forwarding are setup | ${dut2}
+| | ...                                                | ${sock1}
+| | ...                                                | ${sock2}
+| | ${dut2_vif1}= | Set Variable | ${vhost_if1}
+| | ${dut2_vif2}= | Set Variable | ${vhost_if2}
+| | Set Interface State | ${dut2} | ${dut2_vif1} | up
+| | Set Interface State | ${dut2} | ${dut2_vif2} | up
+| | ${dut1_vif1_idx}= | Get Interface SW Index | ${dut1} | ${dut1_vif1}
+| | ${dut1_vif2_idx}= | Get Interface SW Index | ${dut1} | ${dut1_vif2}
+| | ${dut1_if1_idx}= | Get Interface SW Index | ${dut1} | ${dut1_if1}
+| | ${dut1_if2_idx}= | Get Interface SW Index | ${dut1} | ${dut1_if2}
+| | ${dut2_vif1_idx}= | Get Interface SW Index | ${dut2} | ${dut2_vif1}
+| | ${dut2_vif2_idx}= | Get Interface SW Index | ${dut2} | ${dut2_vif2}
+| | ${dut2_if1_idx}= | Get Interface SW Index | ${dut2} | ${dut2_if1}
+| | ${dut2_if2_idx}= | Get Interface SW Index | ${dut2} | ${dut2_if2}
+| | Add fib table | ${dut1} | 20.20.20.0 | 24 | ${fib_table_1}
+| | ... | via 4.4.4.2 sw_if_index ${dut1_vif1_idx} multipath
+| | Add fib table | ${dut1} | 10.10.10.0 | 24 | ${fib_table_1}
+| | ... | via 1.1.1.2 sw_if_index ${dut1_if1_idx} multipath
+| | Add fib table | ${dut1} | 20.20.20.0 | 24 | ${fib_table_2}
+| | ... | via 2.2.2.2 sw_if_index ${dut1_if2_idx} multipath
+| | Add fib table | ${dut1} | 10.10.10.0 | 24 | ${fib_table_2}
+| | ... | via 5.5.5.2 sw_if_index ${dut1_vif2_idx} multipath
+| | Add fib table | ${dut2} | 10.10.10.0 | 24 | ${fib_table_1}
+| | ... | via 2.2.2.1 sw_if_index ${dut2_if1_idx} multipath
+| | Add fib table | ${dut2} | 20.20.20.0 | 24 | ${fib_table_1}
+| | ... | via 4.4.4.1 sw_if_index ${dut2_vif1_idx} multipath
+| | Add fib table | ${dut2} | 10.10.10.0 | 24 | ${fib_table_2}
+| | ... | via 5.5.5.2 sw_if_index ${dut2_vif2_idx} multipath
+| | Add fib table | ${dut2} | 20.20.20.0 | 24 | ${fib_table_2}
+| | ... | via 3.3.3.2 sw_if_index ${dut2_if2_idx} multipath
+| | Assign Interface To Fib Table | ${dut1} | ${dut1_if1} | ${fib_table_1}
+| | Assign Interface To Fib Table | ${dut1} | ${dut1_vif1} | ${fib_table_1}
+| | Assign Interface To Fib Table | ${dut1} | ${dut1_if2} | ${fib_table_2}
+| | Assign Interface To Fib Table | ${dut1} | ${dut1_vif2} | ${fib_table_2}
+| | Assign Interface To Fib Table | ${dut2} | ${dut2_if1} | ${fib_table_1}
+| | Assign Interface To Fib Table | ${dut2} | ${dut2_vif1} | ${fib_table_1}
+| | Assign Interface To Fib Table | ${dut2} | ${dut2_if2} | ${fib_table_2}
+| | Assign Interface To Fib Table | ${dut2} | ${dut2_vif2} | ${fib_table_2}
+| | IP addresses are set on interfaces | ${dut1} | ${dut1_if1} | 1.1.1.2 | 30
+| | IP addresses are set on interfaces | ${dut1} | ${dut1_if2} | 2.2.2.1 | 30
+| | IP addresses are set on interfaces | ${dut1} | ${dut1_vif1} | 4.4.4.1 | 30
+| | IP addresses are set on interfaces | ${dut1} | ${dut1_vif2} | 5.5.5.1 | 30
+| | IP addresses are set on interfaces | ${dut2} | ${dut2_if1} | 2.2.2.2 | 30
+| | IP addresses are set on interfaces | ${dut2} | ${dut2_if2} | 3.3.3.1 | 30
+| | IP addresses are set on interfaces | ${dut2} | ${dut2_vif1} | 4.4.4.1 | 30
+| | IP addresses are set on interfaces | ${dut2} | ${dut2_vif2} | 5.5.5.1 | 30
+| | ${tg1_if1_mac}= | Get Interface MAC | ${tg} | ${tg_if1}
+| | ${dut1_if2_mac}= | Get Interface MAC | ${dut1} | ${dut1_if2}
+| | ${tg1_if2_mac}= | Get Interface MAC | ${tg} | ${tg_if2}
+| | ${dut2_if1_mac}= | Get Interface MAC | ${dut2} | ${dut2_if1}
+| | ${dut1_vif1_mac}= | Get Vhost User Mac By Sw Index | ${dut1}
+| | ... | ${dut1_vif1_idx}
+| | ${dut1_vif2_mac}= | Get Vhost User Mac By Sw Index | ${dut1}
+| | ... | ${dut1_vif2_idx}
+| | ${dut2_vif1_mac}= | Get Vhost User Mac By Sw Index | ${dut2}
+| | ... | ${dut2_vif1_idx}
+| | ${dut2_vif2_mac}= | Get Vhost User Mac By Sw Index | ${dut2}
+| | ... | ${dut2_vif2_idx}
+| | Set Test Variable | ${dut1_vif1_mac}
+| | Set Test Variable | ${dut1_vif2_mac}
+| | Set Test Variable | ${dut2_vif1_mac}
+| | Set Test Variable | ${dut2_vif2_mac}
+| | Add arp on dut | ${dut1} | ${dut1_if1} | 1.1.1.1 | ${tg1_if1_mac}
+| | ... | vrf=${fib_table_1}
+| | Add arp on dut | ${dut1} | ${dut1_if2} | 2.2.2.2 | ${dut2_if1_mac}
+| | ... | vrf=${fib_table_2}
+| | Add arp on dut | ${dut1} | ${dut1_vif1} | 4.4.4.2 | 52:54:00:00:04:01
+| | ... | vrf=${fib_table_1}
+| | Add arp on dut | ${dut1} | ${dut1_vif2} | 5.5.5.2 | 52:54:00:00:04:02
+| | ... | vrf=${fib_table_2}
+| | Add arp on dut | ${dut2} | ${dut2_if1} | 2.2.2.1 | ${dut1_if2_mac}
+| | ... | vrf=${fib_table_1}
+| | Add arp on dut | ${dut2} | ${dut2_if2} | 3.3.3.2 | ${tg1_if2_mac}
+| | ... | vrf=${fib_table_2}
+| | Add arp on dut | ${dut2} | ${dut2_vif1} | 4.4.4.2 | 52:54:00:00:04:01
+| | ... | vrf=${fib_table_1}
+| | Add arp on dut | ${dut2} | ${dut2_vif2} | 5.5.5.2 | 52:54:00:00:04:02
+| | ... | vrf=${fib_table_2}
+| | Vpp Route Add | ${dut1} | 20.20.20.0 | 24 | 4.4.4.2 | ${dut1_vif1}
+| | ... | vrf=${fib_table_1}
+| | Vpp Route Add | ${dut1} | 10.10.10.0 | 24 | 1.1.1.1 | ${dut1_if1}
+| | ... | vrf=${fib_table_1}
+| | Vpp Route Add | ${dut1} | 20.20.20.0 | 24 | 2.2.2.2 | ${dut1_if2}
+| | ... | vrf=${fib_table_2}
+| | Vpp Route Add | ${dut1} | 10.10.10.0 | 24 | 5.5.5.2 | ${dut1_vif2}
+| | ... | vrf=${fib_table_2}
+| | Vpp Route Add | ${dut2} | 20.20.20.0 | 24 | 4.4.4.2 | ${dut2_vif1}
+| | ... | vrf=${fib_table_1}
+| | Vpp Route Add | ${dut2} | 10.10.10.0 | 24 | 2.2.2.1 | ${dut2_if1}
+| | ... | vrf=${fib_table_1}
+| | Vpp Route Add | ${dut2} | 20.20.20.0 | 24 | 3.3.3.2 | ${dut2_if2}
+| | ... | vrf=${fib_table_2}
+| | Vpp Route Add | ${dut2} | 10.10.10.0 | 24 | 5.5.5.2 | ${dut2_vif2}
+| | ... | vrf=${fib_table_2}
+
 | IPv6 forwarding initialized in a 3-node circular topology
 | | [Documentation]
 | | ... | Set UP state on VPP interfaces in path on nodes in 3-node circular
@@ -1051,6 +1178,51 @@
 | | ...                | pmd_disable_hw_vlan=${True}
 | | Return From Keyword | ${vm}
 
+| Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | [Documentation]
+| | ... | Start QEMU guest with two vhost-user interfaces and interconnecting
+| | ... | DPDK testpmd. Qemu Guest is using 3 cores pinned to physical cores 5,
+| | ... | 6, 7 and 2048M. Testpmd is using 3 cores (1 main core and 2 cores
+| | ... | dedicated to io) socket-mem=1024, mem-channel=4, txq/rxq=2048,
+| | ... | burst=64, disable-hw-vlan, total-num-mbufs=65K, driver
+| | ... | usr/lib/librte_pmd_virtio.so and fwd mode is mac rewrite.
+| | ...
+| | ... | *Arguments:*
+| | ... | - dut_node - DUT node to start guest VM on. Type: dictionary
+| | ... | - sock1 - Socket path for first Vhost-User interface. Type: string
+| | ... | - sock2 - Socket path for second Vhost-User interface. Type: string
+| | ... | - vm_name - QemuUtil instance name. Type: string
+| | ... | - eth0_mac - MAC address of first Vhost interface. Type: string
+| | ... | - eth1_mac - MAC address of second Vhost interface. Type: string
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Guest VM with dpdk-testpmd for Vhost L2BD forwarding is setup \
+| | ... | \| ${nodes['DUT1']} \| /tmp/sock1 \| /tmp/sock2 \| DUT1_VM \
+| | ... | \| 00:00:00:00:00:01 \| 00:00:00:00:00:02 \|
+| | [Arguments] | ${dut_node} | ${sock1} | ${sock2} | ${vm_name}
+| | ...         | ${eth0_mac} | ${eth1_mac}
+| | Import Library | resources.libraries.python.QemuUtils
+| | ...            | WITH NAME | ${vm_name}
+| | Run keyword | ${vm_name}.Qemu Add Vhost User If | ${sock1}
+| | Run keyword | ${vm_name}.Qemu Add Vhost User If | ${sock2}
+| | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
+| | Run keyword | ${vm_name}.Qemu Set Smp | 3 | 3 | 1 | 1
+| | Run keyword | ${vm_name}.Qemu Set Mem Size | 2048
+| | Run keyword | ${vm_name}.Qemu Set Huge Allocate
+| | Run keyword | ${vm_name}.Qemu Set Disk Image
+| | ...         | /var/lib/vm/csit-nested-1.3.img
+| | ${vm}= | Run keyword | ${vm_name}.Qemu Start
+| | Run keyword | ${vm_name}.Qemu Set Affinity | 5 | 6 | 7
+| | Dpdk Testpmd Start | ${vm} | eal_coremask=0x7
+| | ...                | eal_mem_channels=4
+| | ...                | eal_socket_mem=1024
+| | ...                | pmd_fwd_mode=mac
+| | ...                | pmd_eth_peer_0=0,${eth0_mac}
+| | ...                | pmd_eth_peer_1=1,${eth1_mac}
+| | ...                | pmd_disable_hw_vlan=${True}
+| | Return From Keyword | ${vm}
+
 | Guest VM with Linux Bridge connected via vhost-user is setup
 | | [Documentation]
 | | ... | Start QEMU guest with two vhost-user interfaces and interconnecting
diff --git a/tests/perf/Long_IPv4_Vhost_Intel-X520-DA2.robot b/tests/perf/Long_IPv4_Vhost_Intel-X520-DA2.robot
new file mode 100644
index 0000000000..2370780b91
--- /dev/null
+++ b/tests/perf/Long_IPv4_Vhost_Intel-X520-DA2.robot
@@ -0,0 +1,681 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Resource | resources/libraries/robot/performance.robot
+| Library | resources.libraries.python.QemuUtils
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | PERFTEST_LONG
+| ...        | PERFTEST_VHOST | NIC_Intel-X520-DA2
+| Suite Setup | Run Keywords | 3-node Performance Suite Setup with DUT's NIC model
+| ... | L3 | Intel-X520-DA2
+| ...         | AND          | Kill QEMU on all DUTs
+| ...         | AND          | Build QEMU on all DUTs
+| Suite Teardown | 3-node Performance Suite Teardown
+| Test Setup | Setup all DUTs before test
+| Test Teardown | Run Keywords | Remove startup configuration of VPP from all DUTs
+| ...           | AND          | Guest VM with dpdk-testpmd Teardown | ${dut1}
+| ...                          | ${dut1_vm_refs}
+| ...           | AND          | Guest VM with dpdk-testpmd Teardown | ${dut2}
+| ...                          | ${dut2_vm_refs}
+| Documentation | *RFC2544: Pkt throughput IPv4 test cases with vhost*
+| ...
+| ... | *[Top] Network Topologies:* TG-DUT1-DUT2-TG 3-node circular topology
+| ... | with single links between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4 for IPv4 routing.
+| ... | *[Cfg] DUT configuration:* DUT1 and DUT2 are configured with IPv4
+| ... | routing and two static IPv4 /24 route entries. Qemu Guest is connected
+| ... | to VPP via vhost-user interfaces. Guest is running DPDK testpmd
+| ... | interconnecting vhost-user interfaces using 3 cores pinned to cpus 5,6,7
+| ... |  and 2048M memory. Testpmd is using socket-mem=1024M (512x2M hugepages),
+| ... | 3 cores (1 main core and 2 cores dedicated for io), forwarding mode is
+| ... | set to mac, rxq/txq=2048, burst=64. DUT1, DUT2 are tested with 2p10GE
+| ... | NIC X520 Niantic by Intel.
+| ... | *[Ver] TG verification:* TG finds and reports throughput NDR (Non Drop
+| ... | Rate) with zero packet loss tolerance or throughput PDR (Partial Drop
+| ... | Rate) with non-zero packet loss tolerance (LT) expressed in percentage
+| ... | of packets transmitted. NDR and PDR are discovered for different
+| ... | Ethernet L2 frame sizes using either binary search or linear search
+| ... | algorithms with configured starting rate and final step that determines
+| ... | throughput measurement resolution. Test packets are generated by TG on
+| ... | links to DUTs. TG traffic profile contains two L3 flow-groups
+| ... | (flow-group per direction, 253 flows per flow-group) with all packets
+| ... | containing Ethernet header, IPv4 header with IP protocol=61 and static
+| ... | payload. MAC addresses are matching MAC addresses of the TG node
+| ... | interfaces.
+| ... | *[Ref] Applicable standard specifications:* RFC2544.
+
+*** Variables ***
+| ${sock1}= | /tmp/sock-1
+| ${sock2}= | /tmp/sock-2
+| ${fib_table_1}= | 100
+| ${fib_table_2}= | 101
+
+*** Test Cases ***
+| TC01: 64B NDR binary search - DUT IPv4-VHOST - 1thread 1core 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 1 thread, 1 phy core, \
+| | ... | 1 receive queue per NIC port. [Ver] Find NDR for 64 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps.
+| | [Tags] | 1_THREAD_NOHTT_RXQUEUES_1 | SINGLE_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 64
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_64B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '1' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC02: 64B PDR binary search - DUT IPv4-VHOST - 1thread 1core 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 1 thread, 1 phy core, \
+| | ... | 1 receive queue per NIC port. [Ver] Find PDR for 64 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps, LT=0.5%.
+| | [Tags] | 1_THREAD_NOHTT_RXQUEUES_1 | SINGLE_THREAD | PDR
+| | ${framesize}= | Set Variable | 64
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_64B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '1' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC03: 1518B NDR binary search - DUT IPv4-VHOST - 1thread 1core 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 1 thread, 1 phy core, \
+| | ... | 1 receive queue per NIC port. [Ver] Find NDR for 1518 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps.
+| | [Tags] | 1_THREAD_NOHTT_RXQUEUES_1 | SINGLE_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 1518
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_1518B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '1' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC04: 1518B PDR binary search - DUT IPv4-VHOST - 1thread 1core 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 1 thread, 1 phy core, \
+| | ... | 1 receive queue per NIC port. [Ver] Find PDR for 1518 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps, LT=0.5%.
+| | [Tags] | 1_THREAD_NOHTT_RXQUEUES_1 | SINGLE_THREAD | PDR
+| | ${framesize}= | Set Variable | 1518
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_1518B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '1' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC05: IMIX_v4_1 NDR binary search - DUT IPv4-VHOST - 1thread 1core 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 1 thread, 1 phy core, \
+| | ... | 1 receive queue per NIC port. [Ver] Find NDR for IMIX_v4_1 frame \
+| | ... | size using binary search start at 10GE linerate, step 5kpps.
+| | ... | IMIX_v4_1 = (28x64B;16x570B;4x1518B)
+| | [Tags] | 1_THREAD_NOHTT_RXQUEUES_1 | SINGLE_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | IMIX_v4_1
+| | ${min_rate}= | Set Variable | 5000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_IMIX_v4_1}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '1' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC06: IMIX_v4_1 PDR binary search - DUT IPv4-VHOST - 1thread 1core 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 1 thread, 1 phy core, \
+| | ... | 1 receive queue per NIC port. [Ver] Find PDR for IMIX_v4_1 frame \
+| | ... | size using binary search start at 10GE linerate, step 5kpps, LT=0.5%.
+| | ... | IMIX_v4_1 = (28x64B;16x570B;4x1518B)
+| | [Tags] | 1_THREAD_NOHTT_RXQUEUES_1 | SINGLE_THREAD | PDR
+| | ${framesize}= | Set Variable | IMIX_v4_1
+| | ${min_rate}= | Set Variable | 5000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_IMIX_v4_1}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '1' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC07: 64B NDR binary search - DUT IPv4-VHOST - 2threads 2cores 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 2 threads, 2 phy cores, \
+| | ... | 1 receive queue per NIC port. [Ver] Find NDR for 64 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps.
+| | [Tags] | 2_THREAD_NOHTT_RXQUEUES_1 | MULTI_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 64
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_64B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '2' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC08: 64B PDR binary search - DUT IPv4-VHOST - 2threads 2cores 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 2 threads, 2 phy cores, \
+| | ... | 1 receive queue per NIC port. [Ver] Find PDR for 64 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps, LT=0.5%.
+| | [Tags] | 2_THREAD_NOHTT_RXQUEUES_1 | MULTI_THREAD | PDR
+| | ${framesize}= | Set Variable | 64
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_64B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '2' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC09: 1518B NDR binary search - DUT IPv4-VHOST - 2threads 2cores 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 2 threads, 2 phy cores, \
+| | ... | 1 receive queue per NIC port. [Ver] Find NDR for 1518 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps.
+| | [Tags] | 2_THREAD_NOHTT_RXQUEUES_1 | MULTI_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 1518
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_1518B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '2' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC10: 1518B PDR binary search - DUT IPv4-VHOST - 2threads 2cores 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 2 threads, 2 phy cores, \
+| | ... | 1 receive queue per NIC port. [Ver] Find PDR for 1518 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps, LT=0.5%.
+| | [Tags] | 2_THREAD_NOHTT_RXQUEUES_1 | MULTI_THREAD | PDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 1518
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_1518B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '2' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC11: IMIX_v4_1 NDR binary search - DUT IPv4-VHOST - 2threads 2cores 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 2 threads, 2 phy cores, \
+| | ... | 1 receive queue per NIC port. [Ver] Find NDR for IMIX_v4_1 frame \
+| | ... | size using binary search start at 10GE linerate, step 5kpps.
+| | ... | IMIX_v4_1 = (28x64B;16x570B;4x1518B)
+| | [Tags] | 2_THREAD_NOHTT_RXQUEUES_1 | MULTI_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | IMIX_v4_1
+| | ${min_rate}= | Set Variable | 5000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_IMIX_v4_1}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '2' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC12: IMIX_v4_1 PDR binary search - DUT IPv4-VHOST - 2threads 2cores 1rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 2 threads, 2 phy cores, \
+| | ... | 1 receive queue per NIC port. [Ver] Find PDR for IMIX_v4_1 frame \
+| | ... | size using binary search start at 10GE linerate, step 5kpps, LT=0.5%.
+| | ... | IMIX_v4_1 = (28x64B;16x570B;4x1518B)
+| | [Tags] | 2_THREAD_NOHTT_RXQUEUES_1 | MULTI_THREAD | PDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | IMIX_v4_1
+| | ${min_rate}= | Set Variable | 5000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_IMIX_v4_1}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '2' worker threads and rxqueues '1' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC13: 64B NDR binary search - DUT IPv4-VHOST - 4threads 4cores 2rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 4 threads, 4 phy cores, \
+| | ... | 2 receive queues per NIC port. [Ver] Find NDR for 64 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps.
+| | [Tags] | 4_THREAD_NOHTT_RXQUEUES_2 | MULTI_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 64
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_64B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '4' worker threads and rxqueues '2' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC14: 64B PDR binary search - DUT IPv4-VHOST - 4threads 4cores 2rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 4 threads, 4 phy cores, \
+| | ... | 2 receive queues per NIC port. [Ver] Find PDR for 64 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps, LT=0.5%.
+| | [Tags] | 4_THREAD_NOHTT_RXQUEUES_2 | MULTI_THREAD | PDR
+| | ${framesize}= | Set Variable | 64
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_64B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '4' worker threads and rxqueues '2' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC15: 1518B NDR binary search - DUT IPv4-VHOST - 4threads 4cores 2rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 4 threads, 4 phy cores, \
+| | ... | 2 receive queues per NIC port. [Ver] Find NDR for 1518 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps.
+| | [Tags] | 4_THREAD_NOHTT_RXQUEUES_2 | MULTI_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 1518
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_1518B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '4' worker threads and rxqueues '2' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC16: 1518B PDR binary search - DUT IPv4-VHOST - 4threads 4cores 2rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 4 threads, 4 phy cores, \
+| | ... | 2 receive queues per NIC port. [Ver] Find PDR for 1518 Byte frames \
+| | ... | using binary search start at 10GE linerate, step 10kpps, LT=0.5%.
+| | [Tags] | 4_THREAD_NOHTT_RXQUEUES_2 | MULTI_THREAD | PDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | 1518
+| | ${min_rate}= | Set Variable | 10000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_1518B}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '4' worker threads and rxqueues '2' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+| TC17: IMIX_v4_1 NDR binary search - DUT IPv4-VHOST - 4threads 4cores 2rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 4 threads, 4 phy cores, \
+| | ... | 2 receive queues per NIC port. [Ver] Find NDR for IMIX_v4_1 frame \
+| | ... | size using binary search start at 10GE linerate, step 5kpps.
+| | ... | IMIX_v4_1 = (28x64B;16x570B;4x1518B)
+| | [Tags] | 4_THREAD_NOHTT_RXQUEUES_2 | MULTI_THREAD | NDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | IMIX_v4_1
+| | ${min_rate}= | Set Variable | 5000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_IMIX_v4_1}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '4' worker threads and rxqueues '2' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find NDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+
+| TC18: IMIX_v4_1 PDR binary search - DUT IPv4-VHOST - 4threads 4cores 2rxq
+| | [Documentation]
+| | ... | [Cfg] DUT runs IPv4 routing config with 4 threads, 4 phy cores, \
+| | ... | 2 receive queues per NIC port. [Ver] Find PDR for IMIX_v4_1 frame \
+| | ... | size using binary search start at 10GE linerate, step 5kpps, LT=0.5%.
+| | ... | IMIX_v4_1 = (28x64B;16x570B;4x1518B)
+| | [Tags] | 4_THREAD_NOHTT_RXQUEUES_2 | MULTI_THREAD | PDR | SKIP_PATCH
+| | ${framesize}= | Set Variable | IMIX_v4_1
+| | ${min_rate}= | Set Variable | 5000
+| | ${max_rate}= | Set Variable | ${10Ge_linerate_pps_IMIX_v4_1}
+| | ${binary_min}= | Set Variable | ${min_rate}
+| | ${binary_max}= | Set Variable | ${max_rate}
+| | ${threshold}= | Set Variable | ${min_rate}
+| | ${dut1_vm_refs}= | Create Dictionary
+| | ${dut2_vm_refs}= | Create Dictionary
+| | Given Add '4' worker threads and rxqueues '2' without HTT to all DUTs
+| | And   Add PCI devices to DUTs from 3-node single link topology
+| | And   Add No Multi Seg to all DUTs
+| | And   Apply startup configuration on all VPP DUTs
+| | When  IPv4 forwarding with Vhost initialized in a 3-node circular topology
+| | ...   | ${sock1} | ${sock2}
+| | ${vm1}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut1} | ${sock1} | ${sock2} | DUT1_VM1 | ${dut1_vif1_mac}
+| | ...     | ${dut1_vif2_mac}
+| | Set To Dictionary | ${dut1_vm_refs} | DUT1_VM1 | ${vm1}
+| | ${vm2}= | And Guest VM with dpdk-testpmd-mac connected via vhost-user is setup
+| | ...     | ${dut2} | ${sock1} | ${sock2} | DUT2_VM1 | ${dut2_vif1_mac}
+| | ...     | ${dut2_vif2_mac}
+| | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM1 | ${vm2}
+| | Then Find PDR using binary search and pps | ${framesize} | ${binary_min}
+| | ...                                       | ${binary_max} | 3-node-IPv4
+| | ...                                       | ${min_rate} | ${max_rate}
+| | ...                                       | ${threshold}
+| | ...                                       | ${glob_loss_acceptance}
+| | ...                                       | ${glob_loss_acceptance_type}
+
+