From f27f09b9a3997710844b662c882505f783b56934 Mon Sep 17 00:00:00 2001 From: pmikus Date: Mon, 27 Sep 2021 08:30:45 +0000 Subject: feat(Performance): Add 2M/1G hugepages + Add ability to switch between hugepages. Signed-off-by: pmikus Change-Id: I84d8eae28ed414a32e5ba82e6c9ed10d7f0ef9cb --- resources/libraries/python/Constants.py | 8 ++-- resources/libraries/python/ContainerUtils.py | 5 ++- resources/libraries/python/QemuManager.py | 3 +- resources/libraries/python/QemuUtils.py | 26 ++++++++++--- resources/libraries/python/VppConfigGenerator.py | 47 +++++------------------- 5 files changed, 40 insertions(+), 49 deletions(-) (limited to 'resources/libraries/python') diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index 6de8cb3814..c6c9bcaf05 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -216,9 +216,11 @@ class Constants: TREX_EXTRA_CMDLINE = get_str_from_env( u"TREX_EXTRA_CMDLINE", u"--mbuf-factor 32") - # graph node variant value - GRAPH_NODE_VARIANT = get_str_from_env( - u"GRAPH_NODE_VARIANT", u"") + # Graph node variant value + GRAPH_NODE_VARIANT = get_str_from_env(u"GRAPH_NODE_VARIANT", u"") + + # Default memory page size in case multiple configured in system + DEFAULT_HUGEPAGE_SIZE = get_str_from_env(u"DEFAULT_HUGEPAGE_SIZE", u"2M") # Sysctl kernel.core_pattern KERNEL_CORE_PATTERN = u"/tmp/%p-%u-%g-%s-%t-%h-%e.core" diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py index 75acf0027a..633144e35f 100644 --- a/resources/libraries/python/ContainerUtils.py +++ b/resources/libraries/python/ContainerUtils.py @@ -702,9 +702,10 @@ class ContainerEngine: vpp_config.add_plugin(u"enable", u"memif_plugin.so") vpp_config.add_plugin(u"enable", u"perfmon_plugin.so") vpp_config.add_main_heap_size(u"2G") - vpp_config.add_main_heap_page_size(u"2M") + vpp_config.add_main_heap_page_size(self.container.page_size) + vpp_config.add_default_hugepage_size(self.container.page_size) vpp_config.add_statseg_size(u"2G") - vpp_config.add_statseg_page_size(u"2M") + vpp_config.add_statseg_page_size(self.container.page_size) vpp_config.add_statseg_per_node_counters(u"on") return vpp_config diff --git a/resources/libraries/python/QemuManager.py b/resources/libraries/python/QemuManager.py index 766372ad9c..259b4c6981 100644 --- a/resources/libraries/python/QemuManager.py +++ b/resources/libraries/python/QemuManager.py @@ -154,7 +154,8 @@ class QemuManager: smp=len(self.machines_affinity[name]), mem=4096, vnf=kwargs[u"vnf"], - img=Constants.QEMU_VM_KERNEL + img=Constants.QEMU_VM_KERNEL, + page_size=kwargs[u"page_size"] ) self.machines[name].add_default_params() self.machines[name].add_kernelvm_params() diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py index 573336153a..64fb5a0e87 100644 --- a/resources/libraries/python/QemuUtils.py +++ b/resources/libraries/python/QemuUtils.py @@ -42,7 +42,7 @@ class QemuUtils: def __init__( self, node, qemu_id=1, smp=1, mem=512, vnf=None, - img=Constants.QEMU_VM_IMAGE): + img=Constants.QEMU_VM_IMAGE, page_size=u""): """Initialize QemuUtil class. :param node: Node to run QEMU on. @@ -51,12 +51,14 @@ class QemuUtils: :param mem: Amount of memory. :param vnf: Network function workload. :param img: QEMU disk image or kernel image path. + :param page_size: Hugepage Size. :type node: dict :type qemu_id: int :type smp: int :type mem: int :type vnf: str :type img: str + :type page_size: str """ self._nic_id = 0 self._node = node @@ -91,6 +93,8 @@ class QemuUtils: self._opt[u"smp"] = int(smp) self._opt[u"img"] = img self._opt[u"vnf"] = vnf + self._opt[u"page_size"] = page_size + # Temporary files. self._temp = dict() self._temp[u"log"] = f"/tmp/serial_{qemu_id}.log" @@ -115,6 +119,9 @@ class QemuUtils: def add_default_params(self): """Set default QEMU command line parameters.""" + mem_path = f"/dev/hugepages1G" \ + if self._opt[u"page_size"] == u"1G" else u"/dev/hugepages" + self._params.add(u"daemonize") self._params.add(u"nodefaults") self._params.add_with_value( @@ -133,7 +140,8 @@ class QemuUtils: ) self._params.add_with_value( u"object", f"memory-backend-file,id=mem," - f"size={self._opt.get(u'mem')}M,mem-path=/dev/hugepages,share=on" + f"size={self._opt.get(u'mem')}M," + f"mem-path={mem_path},share=on" ) self._params.add_with_value(u"m", f"{self._opt.get(u'mem')}M") self._params.add_with_value(u"numa", u"node,memdev=mem") @@ -202,6 +210,8 @@ class QemuUtils: def add_kernelvm_params(self): """Set KernelVM QEMU parameters.""" + hugepages = 3 if self._opt[u"page_size"] == u"1G" else 512 + self._params.add_with_value( u"serial", f"file:{self._temp.get(u'log')}" ) @@ -220,7 +230,8 @@ class QemuUtils: self._params.add_with_value( u"append", f"'ro rootfstype=9p rootflags=trans=virtio " f"root=virtioroot console={self._opt.get(u'console')} " - f"tsc=reliable hugepages=512 " + f"tsc=reliable hugepages={hugepages} " + f"hugepagesz={self._opt.get(u'page_size')} " f"init={self._temp.get(u'ini')} fastboot'" ) @@ -313,9 +324,10 @@ class QemuUtils: vpp_config.add_unix_exec(running) vpp_config.add_socksvr() vpp_config.add_main_heap_size(u"512M") - vpp_config.add_main_heap_page_size(u"2M") + vpp_config.add_main_heap_page_size(self._opt[u"page_size"]) + vpp_config.add_default_hugepage_size(self._opt[u"page_size"]) vpp_config.add_statseg_size(u"512M") - vpp_config.add_statseg_page_size(u"2M") + vpp_config.add_statseg_page_size(self._opt[u"page_size"]) vpp_config.add_statseg_per_node_counters(u"on") vpp_config.add_buffers_per_numa(107520) vpp_config.add_cpu_main_core(u"0") @@ -720,7 +732,9 @@ class QemuUtils: message = f"QEMU: Start failed on {self._node[u'host']}!" try: DUTSetup.check_huge_page( - self._node, u"/dev/hugepages", int(self._opt.get(u"mem"))) + self._node, self._opt.get(u"mem-path"), + int(self._opt.get(u"mem")) + ) exec_cmd_no_error( self._node, cmd_opts, timeout=300, sudo=True, message=message diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py index 4943c1aae6..c338368ff4 100644 --- a/resources/libraries/python/VppConfigGenerator.py +++ b/resources/libraries/python/VppConfigGenerator.py @@ -11,12 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""VPP Configuration File Generator library. - -TODO: Support initialization with default values, -so that we do not need to have block of 6 "Add Unix" commands -in 7 various places of CSIT code. -""" +"""VPP Configuration File Generator library.""" import re @@ -67,8 +62,6 @@ class VppConfigGenerator: self._vpp_logfile = u"/tmp/vpe.log" # VPP Startup config location self._vpp_startup_conf = u"/etc/vpp/startup.conf" - # VPP Startup config backup location - self._vpp_startup_conf_backup = None def set_node(self, node, node_key=None): """Set DUT node. @@ -86,22 +79,6 @@ class VppConfigGenerator: self._node = node self._node_key = node_key - def set_vpp_logfile(self, logfile): - """Set VPP logfile location. - - :param logfile: VPP logfile location. - :type logfile: str - """ - self._vpp_logfile = logfile - - def set_vpp_startup_conf_backup(self, backup=u"/etc/vpp/startup.backup"): - """Set VPP startup configuration backup. - - :param backup: VPP logfile location. - :type backup: str - """ - self._vpp_startup_conf_backup = backup - def get_config_str(self): """Get dumped startup configuration in VPP config format. @@ -433,6 +410,15 @@ class VppConfigGenerator: path = [u"memory", u"main-heap-page-size"] self.add_config_item(self._nodeconfig, value, path) + def add_default_hugepage_size(self, value=Constants.DEFAULT_HUGEPAGE_SIZE): + """Add Default Hugepage Size configuration. + + :param value: Hugepage size. + :type value: str + """ + path = [u"memory", u"default-hugepage-size"] + self.add_config_item(self._nodeconfig, value, path) + def add_api_trace(self): """Add API trace configuration.""" path = [u"api-trace", u"on"] @@ -665,12 +651,6 @@ class VppConfigGenerator: if filename is None: filename = self._vpp_startup_conf - if self._vpp_startup_conf_backup is not None: - cmd = f"cp {self._vpp_startup_conf} {self._vpp_startup_conf_backup}" - exec_cmd_no_error( - self._node, cmd, sudo=True, message=u"Copy config file failed!" - ) - cmd = f"echo \"{self._vpp_config}\" | sudo tee {filename}" exec_cmd_no_error( self._node, cmd, message=u"Writing config file failed!" @@ -692,10 +672,3 @@ class VppConfigGenerator: VPPUtil.restart_vpp_service(self._node, self._node_key) if verify_vpp: VPPUtil.verify_vpp(self._node) - - def restore_config(self): - """Restore VPP startup.conf from backup.""" - cmd = f"cp {self._vpp_startup_conf_backup} {self._vpp_startup_conf}" - exec_cmd_no_error( - self._node, cmd, sudo=True, message=u"Copy config file failed!" - ) -- cgit 1.2.3-korg