Convert existing nf_density testpmd-mac into KernelVM

Change-Id: I9ec1701d9012de59cc42cc5f9a4744f01695ff9a
Signed-off-by: Peter Mikus <pmikus@cisco.com>

6 files changed, 164 insertions, 74 deletions

diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 9822f1c3a0..8a22a4288d 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -52,6 +52,9 @@ class Constants(object):
     # QEMU performance test VM nested image path
     QEMU_PERF_VM_IMAGE = '/var/lib/vm/csit-nested-1.7.img'
 
+    # QEMU performance test VM DPDK path
+    QEMU_PERF_VM_DPDK = '/opt/dpdk-19.02'
+
     # TRex install version
     TREX_INSTALL_VERSION = '2.35'
 
diff --git a/resources/libraries/python/DpdkUtil.py b/resources/libraries/python/DpdkUtil.py
index 41c79a84cf..30efbde378 100644
--- a/resources/libraries/python/DpdkUtil.py
+++ b/resources/libraries/python/DpdkUtil.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -28,26 +28,25 @@ class DpdkUtil(object):
         :returns: EAL parameters string.
         :rtype: str
         """
+        eal_version = '-v'
         # Set the hexadecimal bitmask of the cores to run on.
-        eal_corelist = '-l {} '.format(args['eal_corelist'])\
+        eal_corelist = '-l {}'.format(
+            args.get('eal_corelist', ''))\
             if args.get('eal_corelist', '') else ''
         # Set master core.
-        eal_master_core = '--master-lcore 0 '
-        # Set the number of memory channels to use.
-        eal_mem_channels = '-n {} '.format(args['eal_mem_channels'])\
-            if args.get('eal_mem_channels', '') else ''
-        # Set the memory to allocate on specific sockets (comma separated).
-        eal_socket_mem = '--socket-mem {} '.format(args['eal_socket_mem'])\
-            if args.get('eal_socket_mem', '') else ''
+        eal_master_core = '--master-lcore 0'
         # Load an external driver. Multiple -d options are allowed.
-        eal_driver = '-d /usr/lib/librte_pmd_virtio.so '
-        eal_options = '-v '\
-            + eal_corelist\
-            + eal_master_core\
-            + eal_mem_channels\
-            + eal_socket_mem\
-            + eal_driver
-        return eal_options
+        eal_driver = '-d /usr/lib/librte_pmd_virtio.so'\
+            if args.get('eal_driver', True) else ''
+        # Run in memory.
+        eal_in_memory = '--in-memory'\
+            if args.get('eal_in_memory', False) else ''
+
+        return ' '.join([eal_version,
+                         eal_corelist,
+                         eal_master_core,
+                         eal_driver,
+                         eal_in_memory])
 
     @staticmethod
     def get_pmd_options(**args):
@@ -60,65 +59,80 @@ class DpdkUtil(object):
         """
         # Set the forwarding mode: io, mac, mac_retry, mac_swap, flowgen,
         # rxonly, txonly, csum, icmpecho, ieee1588
-        pmd_fwd_mode = '--forward-mode={} '.format(args['pmd_fwd_mode'])\
-            if args.get('pmd_fwd_mode', '') else ''
+        pmd_fwd_mode = '--forward-mode={}'.format(
+            args.get('pmd_fwd_mode', 'io')) \
+            if args.get('pmd_fwd_mode', 'io') else ''
         # Set the number of packets per burst to N.
-        pmd_burst = '--burst=64 '
+        pmd_burst = '--burst=64'
         # Set the number of descriptors in the TX rings to N.
-        pmd_txd = '--txd={} '.format(args.get('pmd_txd', '256')) \
-            if args.get('pmd_txd', '256') else ''
+        pmd_txd = '--txd={}'.format(
+            args.get('pmd_txd', '1024')) \
+            if args.get('pmd_txd', '1024') else ''
         # Set the number of descriptors in the RX rings to N.
-        pmd_rxd = '--rxd={} '.format(args.get('pmd_rxd', '256')) \
-            if args.get('pmd_rxd', '256') else ''
+        pmd_rxd = '--rxd={}'.format(
+            args.get('pmd_rxd', '1024')) \
+            if args.get('pmd_rxd', '1024') else ''
         # Set the number of queues in the TX to N.
-        pmd_txq = '--txq={} '.format(args.get('pmd_txq', '1')) \
+        pmd_txq = '--txq={}'.format(
+            args.get('pmd_txq', '1')) \
             if args.get('pmd_txq', '1') else ''
         # Set the number of queues in the RX to N.
-        pmd_rxq = '--rxq={} '.format(args.get('pmd_rxq', '1')) \
+        pmd_rxq = '--rxq={}'.format(
+            args.get('pmd_rxq', '1')) \
             if args.get('pmd_rxq', '1') else ''
-        # Set the hexadecimal bitmask of TX queue flags.
-        pmd_txqflags = '--txqflags=0xf00 '
+        # Set the hexadecimal bitmask of TX offloads.
+        pmd_tx_offloads = '--txqflags=0xf00'\
+            if args.get('pmd_tx_offloads', True) else ''
         # Set the number of mbufs to be allocated in the mbuf pools.
-        pmd_total_num_mbufs = '--total-num-mbufs={} '.format(
-            args['pmd_num_mbufs']) if args.get('pmd_num_mbufs', '') else ''
+        pmd_total_num_mbufs = '--total-num-mbufs={}'.format(
+            args.get('pmd_num_mbufs', '')) \
+            if args.get('pmd_num_mbufs', '') else ''
         # Set the max packet length.
-        pmd_max_pkt_len = "--max-pkt-len={0}".format(args["pmd_max_pkt_len"]) \
+        pmd_max_pkt_len = '--max-pkt-len={}'.format(
+            args.get("pmd_max_pkt_len", "")) \
             if args.get("pmd_max_pkt_len", "") else ""
-        # Set the hexadecimal bitmask of the ports for forwarding.
-        pmd_portmask = '--portmask={} '.format(args['pmd_portmask'])\
-            if args.get('pmd_portmask', '') else ''
         # Disable hardware VLAN.
-        pmd_disable_hw_vlan = '--disable-hw-vlan '\
-            if args.get('pmd_disable_hw_vlan', '') else ''
-        # Disable RSS (Receive Side Scaling).
-        pmd_disable_rss = '--disable-rss '\
-            if args.get('pmd_disable_rss', '') else ''
+        pmd_disable_hw_vlan = '--disable-hw-vlan'\
+            if args.get('pmd_disable_hw_vlan', True) else ''
         # Set the MAC address XX:XX:XX:XX:XX:XX of the peer port N
-        pmd_eth_peer_0 = '--eth-peer={} '.format(args['pmd_eth_peer_0'])\
+        pmd_eth_peer_0 = '--eth-peer={}'.format(
+            args.get('pmd_eth_peer_0', ''))\
             if args.get('pmd_eth_peer_0', '') else ''
-        pmd_eth_peer_1 = '--eth-peer={} '.format(args['pmd_eth_peer_1'])\
+        pmd_eth_peer_1 = '--eth-peer={}'.format(
+            args.get('pmd_eth_peer_1', ''))\
             if args.get('pmd_eth_peer_1', '') else ''
         # Set the number of forwarding cores based on coremask.
-        pmd_nb_cores = '--nb-cores={} '.format(
-            bin(int(args['eal_coremask'], 0)).count('1')-1)\
-            if args.get('eal_coremask', '') else ''
-        pmd_options = '-- '\
-            + pmd_fwd_mode\
-            + pmd_burst\
-            + pmd_txd\
-            + pmd_rxd\
-            + pmd_txq\
-            + pmd_rxq\
-            + pmd_txqflags\
-            + pmd_total_num_mbufs\
-            + pmd_portmask\
-            + pmd_disable_hw_vlan\
-            + pmd_disable_rss\
-            + pmd_eth_peer_0\
-            + pmd_eth_peer_1\
-            + pmd_nb_cores\
-            + pmd_max_pkt_len
-        return pmd_options
+        pmd_nb_cores = '--nb-cores={}'.format(
+            args.get('pmd_nb_cores', ''))\
+            if args.get('pmd_nb_cores', '') else ''
+
+        return ' '.join([pmd_fwd_mode,
+                         pmd_burst,
+                         pmd_txd,
+                         pmd_rxd,
+                         pmd_txq,
+                         pmd_rxq,
+                         pmd_tx_offloads,
+                         pmd_total_num_mbufs,
+                         pmd_disable_hw_vlan,
+                         pmd_eth_peer_0,
+                         pmd_eth_peer_1,
+                         pmd_max_pkt_len,
+                         pmd_nb_cores])
+
+    @staticmethod
+    def get_testpmd_cmdline(**kwargs):
+        """Get DPDK testpmd command line arguments.
+
+        :param args: Key-value testpmd parameters.
+        :type args: dict
+        :returns: Command line string.
+        :rtype: str
+        """
+        eal_options = DpdkUtil.get_eal_options(**kwargs)
+        pmd_options = DpdkUtil.get_pmd_options(**kwargs)
+
+        return 'testpmd {0} -- {1}'.format(eal_options, pmd_options)
 
     @staticmethod
     def dpdk_testpmd_start(node, **kwargs):
@@ -135,7 +149,7 @@ class DpdkUtil(object):
 
         ssh = SSH()
         ssh.connect(node)
-        cmd = "/start-testpmd.sh {0} {1}".format(eal_options, pmd_options)
+        cmd = "/start-testpmd.sh {0} -- {1}".format(eal_options, pmd_options)
         exec_cmd_no_error(node, cmd, sudo=True)
         ssh.disconnect(node)
 
diff --git a/resources/libraries/python/QemuManager.py b/resources/libraries/python/QemuManager.py
index e81f0a84ce..55c282bc24 100644
--- a/resources/libraries/python/QemuManager.py
+++ b/resources/libraries/python/QemuManager.py
@@ -26,7 +26,7 @@ __all__ = ["QemuManager"]
 
 
 def get_affinity_vm(nodes, node, nf_chains=1, nf_nodes=1, nf_chain=1, nf_node=1,
-                    cpu_count_int=1):
+                    cpu_count_int=1, vnf_count_int=1):
     """Get affinity of VM. Result will be used to compute the amount of
     CPUs and also affinity.
 
@@ -37,6 +37,7 @@ def get_affinity_vm(nodes, node, nf_chains=1, nf_nodes=1, nf_chain=1, nf_node=1,
     :param nf_chain: Chain ID.
     :param nf_node: Node ID.
     :param cpu_count_int: Amount of Dataplane threads of vswitch.
+    :param vnf_count_int: Amount of Dataplane threads of vnf.
     :type nodes: dict
     :type node: dict
     :type nf_chains: int
@@ -44,6 +45,7 @@ def get_affinity_vm(nodes, node, nf_chains=1, nf_nodes=1, nf_chain=1, nf_node=1,
     :type nf_chain: int
     :type nf_node: int
     :type cpu_count_int: int
+    :type vnf_count_int: int
     :returns: List of CPUs allocated to VM.
     :rtype: list
     """
@@ -51,7 +53,7 @@ def get_affinity_vm(nodes, node, nf_chains=1, nf_nodes=1, nf_chain=1, nf_node=1,
     dut_mc = 1
     dut_dc = cpu_count_int
     skip_cnt = sut_sc + dut_mc + dut_dc
-    dtc = cpu_count_int
+    dtc = vnf_count_int
 
     interface_list = []
     interface_list.append(
@@ -95,7 +97,8 @@ class QemuManager(object):
         nf_chains = int(kwargs['nf_chains'])
         nf_nodes = int(kwargs['nf_nodes'])
         queues = kwargs['rxq_count_int'] if kwargs['auto_scale'] else 1
-        cpu_count_int = kwargs['cpu_count_int'] if kwargs['auto_scale'] else 1
+        cpu_count_int = kwargs['cpu_count_int']
+        vnf_count_int = kwargs['cpu_count_int'] if kwargs['auto_scale'] else 1
 
         img = Constants.QEMU_PERF_VM_KERNEL
 
@@ -113,7 +116,7 @@ class QemuManager(object):
                 self.machines_affinity[name] = get_affinity_vm(
                     nodes=self.nodes, node=node, nf_chains=nf_chains,
                     nf_nodes=nf_nodes, nf_chain=nf_chain, nf_node=nf_node,
-                    cpu_count_int=cpu_count_int)
+                    cpu_count_int=cpu_count_int, vnf_count_int=vnf_count_int)
 
                 self.machines[name] = QemuUtils(
                     node=self.nodes[node], qemu_id=qemu_id,
@@ -123,7 +126,9 @@ class QemuManager(object):
                     mac1='52:54:00:00:{id:02x}:01'.format(id=qemu_id),
                     mac2='52:54:00:00:{id:02x}:02'.format(id=qemu_id),
                     vif1_mac=vif1_mac,
-                    vif2_mac=vif2_mac)
+                    vif2_mac=vif2_mac,
+                    queues=queues,
+                    jumbo_frames=kwargs['jumbo'])
                 self.machines[name].qemu_add_vhost_user_if(
                     sock1, jumbo_frames=kwargs['jumbo'], queues=queues,
                     queue_size=1024)
diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index e4a1256fea..193390c46d 100644
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -24,6 +24,7 @@ from distutils.version import StrictVersion
 from robot.api import logger
 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
 from resources.libraries.python.Constants import Constants
+from resources.libraries.python.DpdkUtil import DpdkUtil
 from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.topology import NodeType, Topology
 from resources.libraries.python.VppConfigGenerator import VppConfigGenerator
@@ -81,7 +82,7 @@ class QemuUtils(object):
     # Use one instance of class per tests.
     ROBOT_LIBRARY_SCOPE = 'TEST CASE'
 
-    def __init__(self, node=None, qemu_id=1, smp=1, mem=512, vnf=None,
+    def __init__(self, node, qemu_id=1, smp=1, mem=512, vnf=None,
                  img='/var/lib/vm/vhost-nested.img', bin_path='/usr/bin'):
         """Initialize QemuUtil class.
 
@@ -245,8 +246,9 @@ class QemuUtils(object):
                                             format(smp=self._opt.get('smp')-1))
         vpp_config.add_dpdk_dev('0000:00:06.0', '0000:00:07.0')
         vpp_config.add_dpdk_log_level('debug')
-        vpp_config.add_dpdk_no_tx_checksum_offload()
-        vpp_config.add_dpdk_no_multi_seg()
+        if not kwargs['jumbo_frames']:
+            vpp_config.add_dpdk_no_multi_seg()
+            vpp_config.add_dpdk_no_tx_checksum_offload()
         vpp_config.add_plugin('disable', 'default')
         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
         vpp_config.apply_config(startup, restart_vpp=False)
@@ -263,6 +265,59 @@ class QemuUtils(object):
                               format(out=src.safe_substitute(**kwargs),
                                      running=running))
 
+    def create_kernelvm_config_testpmd_io(self, **kwargs):
+        """Create QEMU testpmd-io command line.
+
+        :param kwargs: Key-value pairs to construct command line parameters.
+        :type kwargs: dict
+        """
+        testpmd_path = ('{path}/{arch}-native-linuxapp-gcc/app'.
+                        format(path=Constants.QEMU_PERF_VM_DPDK,
+                               arch=Topology.get_node_arch(self._node)))
+        testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
+            eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
+            eal_driver=False,
+            eal_in_memory=True,
+            pmd_num_mbufs=16384,
+            pmd_rxq=kwargs['queues'],
+            pmd_txq=kwargs['queues'],
+            pmd_tx_offloads=False,
+            pmd_disable_hw_vlan=False,
+            pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
+            pmd_nb_cores=str(self._opt.get('smp') - 1))
+
+        self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
+                                format(testpmd_path=testpmd_path,
+                                       testpmd_cmd=testpmd_cmd))
+
+    def create_kernelvm_config_testpmd_mac(self, **kwargs):
+        """Create QEMU testpmd-mac command line.
+
+        :param kwargs: Key-value pairs to construct command line parameters.
+        :type kwargs: dict
+        """
+        testpmd_path = ('{path}/{arch}-native-linuxapp-gcc/app'.
+                        format(path=Constants.QEMU_PERF_VM_DPDK,
+                               arch=Topology.get_node_arch(self._node)))
+        testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
+            eal_corelist='0-{smp}'.format(smp=self._opt.get('smp') - 1),
+            eal_driver=False,
+            eal_in_memory=True,
+            pmd_num_mbufs=16384,
+            pmd_fwd_mode='mac',
+            pmd_eth_peer_0='0,{mac}'.format(mac=kwargs['vif1_mac']),
+            pmd_eth_peer_1='1,{mac}'.format(mac=kwargs['vif2_mac']),
+            pmd_rxq=kwargs['queues'],
+            pmd_txq=kwargs['queues'],
+            pmd_tx_offloads=False,
+            pmd_disable_hw_vlan=False,
+            pmd_max_pkt_len=9200 if kwargs['jumbo_frames'] else None,
+            pmd_nb_cores=str(self._opt.get('smp') - 1))
+
+        self._opt['vnf_bin'] = ('{testpmd_path}/{testpmd_cmd}'.
+                                format(testpmd_path=testpmd_path,
+                                       testpmd_cmd=testpmd_cmd))
+
     def create_kernelvm_init(self, **kwargs):
         """Create QEMU init script.
 
@@ -290,6 +345,10 @@ class QemuUtils(object):
         """
         if 'vpp' in self._opt.get('vnf'):
             self.create_kernelvm_config_vpp(**kwargs)
+        elif 'testpmd_io' in self._opt.get('vnf'):
+            self.create_kernelvm_config_testpmd_io(**kwargs)
+        elif 'testpmd_mac' in self._opt.get('vnf'):
+            self.create_kernelvm_config_testpmd_mac(**kwargs)
         else:
             raise RuntimeError('QEMU: Unsupported VNF!')
         self.create_kernelvm_init(vnf_bin=self._opt.get('vnf_bin'))
@@ -302,7 +361,7 @@ class QemuUtils(object):
         """
         command = ("grep -rwl 'CPU' /proc/$(sudo cat {pidfile})/task/*/comm ".
                    format(pidfile=self._temp.get('pidfile')))
-        command += (r"| xargs dirname | sed -e 's/\/.*\///g'")
+        command += (r"| xargs dirname | sed -e 's/\/.*\///g' | uniq")
 
         stdout, _ = exec_cmd_no_error(self._node, command)
         return stdout.splitlines()
@@ -527,6 +586,8 @@ class QemuUtils(object):
         :param retries: Number of retries.
         :type retries: int
         """
+        vpp_ver = VPPUtil.vpp_show_version(self._node)
+
         for _ in range(retries):
             command = ('tail -1 {log}'.format(log=self._temp.get('log')))
             stdout = None
@@ -535,8 +596,11 @@ class QemuUtils(object):
                 sleep(1)
             except RuntimeError:
                 pass
-            if VPPUtil.vpp_show_version(self._node) in stdout:
+            if vpp_ver in stdout or 'Press enter to exit' in stdout:
                 break
+            if 'reboot: Power down' in stdout:
+                raise RuntimeError('QEMU: NF failed to run on {host}!'.
+                                   format(host=self._node['host']))
         else:
             raise RuntimeError('QEMU: Timeout, VM not booted on {host}!'.
                                format(host=self._node['host']))
@@ -599,6 +663,7 @@ class QemuUtils(object):
                  format(pidfile=self._temp.get('pidfile')), sudo=True)
 
         for value in self._temp.values():
+            exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 
     def qemu_kill_all(self):
@@ -606,6 +671,7 @@ class QemuUtils(object):
         exec_cmd(self._node, 'pkill -SIGKILL qemu', sudo=True)
 
         for value in self._temp.values():
+            exec_cmd(self._node, 'cat {value}'.format(value=value), sudo=True)
             exec_cmd(self._node, 'rm -f {value}'.format(value=value), sudo=True)
 
     def qemu_version(self, version=None):
diff --git a/resources/libraries/robot/performance/performance_configuration.robot b/resources/libraries/robot/performance/performance_configuration.robot
index b69980bf1a..edb3a9191e 100644
--- a/resources/libraries/robot/performance/performance_configuration.robot
+++ b/resources/libraries/robot/performance/performance_configuration.robot
@@ -2370,7 +2370,7 @@
 | | ${testpmd_cpus}= | Cpu list per node str | ${nodes['${dut}']} | ${0}
 | | ... | cpu_cnt=${nf_cpus_count}
 | | Dpdk Testpmd Start | ${vm} | eal_corelist=${testpmd_cpus}
-| | ... | eal_mem_channels=4 | pmd_fwd_mode=io | pmd_disable_hw_vlan=${TRUE}
+| | ... | pmd_fwd_mode=io | pmd_disable_hw_vlan=${TRUE}
 | | ... | pmd_rxd=${perf_qemu_qsz} | pmd_txd=${perf_qemu_qsz}
 | | ... | pmd_rxq=${rxq} | pmd_txq=${rxq} | pmd_max_pkt_len=${max_pkt_len}
 | | Return From Keyword | ${vm}
@@ -2509,7 +2509,7 @@
 | | ${testpmd_cpus}= | Cpu list per node str | ${nodes['${dut}']} | ${0}
 | | ... | cpu_cnt=${nf_cpus_count}
 | | Dpdk Testpmd Start | ${vm} | eal_corelist=${testpmd_cpus}
-| | ... | eal_mem_channels=4 | pmd_fwd_mode=mac | pmd_eth_peer_0=0,${eth0_mac}
+| | ... | pmd_fwd_mode=mac | pmd_eth_peer_0=0,${eth0_mac}
 | | ... | pmd_eth_peer_1=1,${eth1_mac} | pmd_disable_hw_vlan=${TRUE}
 | | ... | pmd_rxd=${perf_qemu_qsz} | pmd_txd=${perf_qemu_qsz}
 | | ... | pmd_rxq=${rxq} | pmd_txq=${rxq} | pmd_max_pkt_len=${max_pkt_len}
diff --git a/resources/templates/vm/init.sh b/resources/templates/vm/init.sh
index 6cdfc09f09..53711fd580 100644
--- a/resources/templates/vm/init.sh
+++ b/resources/templates/vm/init.sh
@@ -2,9 +2,11 @@
 mount -t sysfs -o "nodev,noexec,nosuid" sysfs /sys
 mount -t proc -o "nodev,noexec,nosuid" proc /proc
 mkdir /dev/pts
+mkdir /dev/hugepages
 mount -t devpts -o "rw,noexec,nosuid,gid=5,mode=0620" devpts /dev/pts || true
 mount -t tmpfs -o "rw,noexec,nosuid,size=10%,mode=0755" tmpfs /run
 mount -t tmpfs -o "rw,noexec,nosuid,size=10%,mode=0755" tmpfs /tmp
+mount -t hugetlbfs -o "rw,relatime,pagesize=2M" hugetlbfs /dev/hugepages
 echo 0000:00:06.0 > /sys/bus/pci/devices/0000:00:06.0/driver/unbind
 echo 0000:00:07.0 > /sys/bus/pci/devices/0000:00:07.0/driver/unbind
 echo uio_pci_generic > /sys/bus/pci/devices/0000:00:06.0/driver_override