aboutsummaryrefslogtreecommitdiffstats
path: root/fdio.infra.ansible/roles/performance_tuning
diff options
context:
space:
mode:
authorpmikus <pmikus@cisco.com>2021-04-08 10:44:18 +0000
committerPeter Mikus <pmikus@cisco.com>2021-04-08 11:17:15 +0000
commitdf5672b3d9c29b51397f4770eb992c9f3f3955ce (patch)
treeadb4cf36c9b680ebbc44c953391a0d21b986d6b7 /fdio.infra.ansible/roles/performance_tuning
parent8018da98e0f362bc69fc9600fac222a86fd46b5e (diff)
Ansible git move
+ Better accessibility + Compliant with fdio.infra._function_ - function [pxe|terraform|ansible|vagrant] + dill==0.3.3 also applied on TBs - ci-man to follow today - Docs to be updated in separate patch Signed-off-by: pmikus <pmikus@cisco.com> Change-Id: Iff9eaa29d63044188cc8160db2d9b44b7635782a
Diffstat (limited to 'fdio.infra.ansible/roles/performance_tuning')
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml20
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils1
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service10
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/files/irqbalance25
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py29
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml13
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml189
-rw-r--r--fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml44
8 files changed, 331 insertions, 0 deletions
diff --git a/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml b/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml
new file mode 100644
index 0000000000..2dad931e92
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml
@@ -0,0 +1,20 @@
+---
+# file: roles/performance_tuning/defaults/main.yaml
+
+packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}"
+
+packages_base:
+ - "cpufrequtils"
+
+packages_by_distro:
+ ubuntu:
+ bionic:
+ - []
+ focal:
+ - []
+
+packages_by_arch:
+ aarch64:
+ - []
+ x86_64:
+ - [] \ No newline at end of file
diff --git a/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils b/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils
new file mode 100644
index 0000000000..03070fefe1
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils
@@ -0,0 +1 @@
+GOVERNOR="performance"
diff --git a/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service b/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service
new file mode 100644
index 0000000000..e04729de50
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Disable Turbo Boost on Intel CPU
+
+[Service]
+ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done'
+ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done'
+RemainAfterExit=yes
+
+[Install]
+WantedBy=sysinit.target
diff --git a/fdio.infra.ansible/roles/performance_tuning/files/irqbalance b/fdio.infra.ansible/roles/performance_tuning/files/irqbalance
new file mode 100644
index 0000000000..861be02fb3
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/files/irqbalance
@@ -0,0 +1,25 @@
+# irqbalance is a daemon process that distributes interrupts across
+# CPUS on SMP systems. The default is to rebalance once every 10
+# seconds. This is the environment file that is specified to systemd via the
+# EnvironmentFile key in the service unit file (or via whatever method the init
+# system you're using has.
+#
+# ONESHOT=yes
+# after starting, wait for a minute, then look at the interrupt
+# load and balance it once; after balancing exit and do not change
+# it again.
+#IRQBALANCE_ONESHOT=
+
+#
+# IRQBALANCE_BANNED_CPUS
+# 64 bit bitmask which allows you to indicate which cpu's should
+# be skipped when reblancing irqs. Cpu numbers which have their
+# corresponding bits set to one in this mask will not have any
+# irq's assigned to them on rebalance
+#
+IRQBALANCE_BANNED_CPUS="{{ ansible_processor_cores | irqbalance_banned_cpu_mask(ansible_processor_count, ansible_processor_threads_per_core) }}"
+#
+# IRQBALANCE_ARGS
+# append any args here to the irqbalance daemon as documented in the man page
+#
+#IRQBALANCE_ARGS=
diff --git a/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py b/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py
new file mode 100644
index 0000000000..d76f6fe166
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py
@@ -0,0 +1,29 @@
+"""Extra Ansible filters"""
+
+def irqbalance_banned_cpu_mask(
+ processor_cores, processor_count, processor_threads_per_core):
+ """
+ Return irqbalance CPU mask.
+ Args:
+ processor_cores (int): Physical processor unit.
+ processor_counts (int): Processors per physical unit.
+ processor_threads_per_core (int): Threads per physical unit.
+ Returns:
+ str: irqbalance_banned_cpus.
+ """
+ mask = int("1" * 128, 2)
+
+ for i in range(processor_count * processor_threads_per_core):
+ mask &= ~(1 << i * processor_cores)
+
+ import re
+ return ",".join(re.findall('.{1,8}', str(hex(mask))[2:]))
+
+
+class FilterModule(object):
+ """Return filter plugin"""
+
+ @staticmethod
+ def filters():
+ """Return filter"""
+ return {'irqbalance_banned_cpu_mask': irqbalance_banned_cpu_mask}
diff --git a/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml b/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml
new file mode 100644
index 0000000000..fa2876b7ac
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml
@@ -0,0 +1,13 @@
+---
+# file roles/performance_tuning/handlers/main.yaml
+
+- name: Update GRUB
+ command: update-grub
+ tags:
+ - update-grub
+
+- name: Reboot server
+ reboot:
+ reboot_timeout: 3600
+ tags:
+ - reboot-server
diff --git a/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml b/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml
new file mode 100644
index 0000000000..e9cdd0d819
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml
@@ -0,0 +1,189 @@
+---
+# file: roles/performance_tuning/tasks/main.yaml
+
+- name: Inst - Update Package Cache (APT)
+ apt:
+ update_cache: yes
+ cache_valid_time: 3600
+ when:
+ - ansible_distribution|lower == 'ubuntu'
+ tags:
+ - perf-inst-prerequisites
+
+- name: Inst - Machine Prerequisites
+ package:
+ name: "{{ packages | flatten(levels=1) }}"
+ state: latest
+ tags:
+ - perf-inst-prerequisites
+
+- name: Conf - Turbo Boost
+ import_tasks: turbo_boost.yaml
+ when: >
+ cpu_microarchitecture == "skylake" or
+ cpu_microarchitecture == "cascadelake"
+ tags:
+ - perf-conf-turbo-boost
+
+- name: Conf - Adjust nr_hugepages
+ # change the minimum size of the hugepage pool.
+ # 2G VPP, 4GB per VNF/CNF, 2G reserve
+ sysctl:
+ name: "vm.nr_hugepages"
+ value: "{{ sysctl.vm.nr_hugepages }}"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Adjust max_map_count
+ # this file contains the maximum number of memory map areas a process
+ # may have. memory map areas are used as a side-effect of calling
+ # malloc, directly by mmap and mprotect, and also when loading shared
+ # libraries.
+ #
+ # while most applications need less than a thousand maps, certain
+ # programs, particularly malloc debuggers, may consume lots of them,
+ # e.g., up to one or two maps per allocation.
+ # must be greater than or equal to (2 * vm.nr_hugepages).
+ sysctl:
+ name: "vm.max_map_count"
+ value: "{{ sysctl.vm.nr_hugepages * 4 }}"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Adjust hugetlb_shm_group
+ # hugetlb_shm_group contains group id that is allowed to create sysv
+ # shared memory segment using hugetlb page.
+ sysctl:
+ name: "vm.hugetlb_shm_group"
+ value: "1000"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Adjust swappiness
+ # this control is used to define how aggressive the kernel will swap
+ # memory pages. higher values will increase agressiveness, lower values
+ # decrease the amount of swap. a value of 0 instructs the kernel not to
+ # initiate swap until the amount of free and file-backed pages is less
+ # than the high water mark in a zone.
+ sysctl:
+ name: "vm.swappiness"
+ value: "0"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Adjust shmmax
+ # shared memory max must be greator or equal to the total size of hugepages.
+ # for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024
+ # if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax)
+ # is greater than the calculated totalhugepagesize then set this parameter
+ # to current shmmax value.
+ sysctl:
+ name: "kernel.shmmax"
+ value: "{{ sysctl.vm.nr_hugepages * 2 * 1024 * 1024 }}"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Adjust watchdog_cpumask
+ # this value can be used to control on which cpus the watchdog may run.
+ # the default cpumask is all possible cores, but if no_hz_full is
+ # enabled in the kernel config, and cores are specified with the
+ # nohz_full= boot argument, those cores are excluded by default.
+ # offline cores can be included in this mask, and if the core is later
+ # brought online, the watchdog will be started based on the mask value.
+ #
+ # typically this value would only be touched in the nohz_full case
+ # to re-enable cores that by default were not running the watchdog,
+ # if a kernel lockup was suspected on those cores.
+ sysctl:
+ name: "kernel.watchdog_cpumask"
+ value: "{{ sysctl.kernel.watchdog_cpumask }}"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Adjust randomize_va_space
+ # this option can be used to select the type of process address
+ # space randomization that is used in the system, for architectures
+ # that support this feature.
+ # 0 - turn the process address space randomization off. this is the
+ # default for architectures that do not support this feature anyways,
+ # and kernels that are booted with the "norandmaps" parameter.
+ sysctl:
+ name: "kernel.randomize_va_space"
+ value: "0"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - perf-conf-sysctl
+
+- name: Conf - Cpufrequtils
+ copy:
+ src: "files/cpufrequtils"
+ dest: "/etc/default/cpufrequtils"
+ owner: "root"
+ group: "root"
+ mode: 0644
+ tags:
+ - perf-conf-cpufrequtils
+
+- name: Conf - Irqbalance
+ template:
+ src: "files/irqbalance"
+ dest: "/etc/default/irqbalance"
+ owner: "root"
+ group: "root"
+ mode: 0644
+ tags:
+ - perf-conf-irqbalance
+
+- name: Conf - Set Ondemand Service To Disable
+ service:
+ name: "ondemand"
+ enabled: "no"
+ tags:
+ - perf-conf-ondemand
+
+- name: Conf - Kernel Parameters
+ lineinfile:
+ path: "/etc/default/grub"
+ state: "present"
+ regexp: "^GRUB_CMDLINE_LINUX="
+ line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value is sameas true %}{{key}} {% else %}{{key}}={{value}} {% endif %}{% endfor %}\""
+ notify:
+ - "Update GRUB"
+ tags:
+ - perf-conf-grub
+
+- meta: flush_handlers
+
+- name: Conf - Load Kernel Modules By Default
+ lineinfile:
+ path: "/etc/modules"
+ state: "present"
+ line: "{{ item }}"
+ with_items:
+ - "vfio-pci"
+ notify:
+ - "Reboot Server"
+ tags:
+ - perf-conf-load-kernel-modules
+
+- meta: flush_handlers \ No newline at end of file
diff --git a/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml
new file mode 100644
index 0000000000..7f69365a2d
--- /dev/null
+++ b/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml
@@ -0,0 +1,44 @@
+---
+# file: roles/performance_tuning/tasks/turbo_boost.yaml
+
+- name: Inst - Update Package Cache (APT)
+ apt:
+ update_cache: yes
+ cache_valid_time: 3600
+ when:
+ - ansible_distribution|lower == 'ubuntu'
+ tags:
+ - turbo-inst-prerequisites
+
+- name: Inst - msr-tools
+ package:
+ name:
+ - "msr-tools"
+ state: latest
+ tags:
+ - turbo-inst-prerequisites
+
+- name: Conf - Load msr By Default
+ lineinfile:
+ path: "/etc/modules"
+ state: "present"
+ line: "msr"
+ tags:
+ - turbo-conf-msr
+
+- name: Conf - Custom Startup Service Hook
+ copy:
+ src: "files/disable-turbo-boost.service"
+ dest: "/etc/systemd/system/disable-turbo-boost.service"
+ owner: "root"
+ group: "root"
+ mode: 0644
+ tags:
+ - turbo-conf-msr
+
+- name: Conf - Custom Startup Service Hook Enable
+ service:
+ name: "disable-turbo-boost"
+ enabled: yes
+ tags:
+ - turbo-conf-msr