diff options
Diffstat (limited to 'fdio.infra.ansible/roles/performance_tuning')
8 files changed, 331 insertions, 0 deletions
diff --git a/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml b/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml new file mode 100644 index 0000000000..2dad931e92 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml @@ -0,0 +1,20 @@ +--- +# file: roles/performance_tuning/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - "cpufrequtils" + +packages_by_distro: + ubuntu: + bionic: + - [] + focal: + - [] + +packages_by_arch: + aarch64: + - [] + x86_64: + - []
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils b/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils new file mode 100644 index 0000000000..03070fefe1 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils @@ -0,0 +1 @@ +GOVERNOR="performance" diff --git a/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service b/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service new file mode 100644 index 0000000000..e04729de50 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service @@ -0,0 +1,10 @@ +[Unit] +Description=Disable Turbo Boost on Intel CPU + +[Service] +ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done' +ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done' +RemainAfterExit=yes + +[Install] +WantedBy=sysinit.target diff --git a/fdio.infra.ansible/roles/performance_tuning/files/irqbalance b/fdio.infra.ansible/roles/performance_tuning/files/irqbalance new file mode 100644 index 0000000000..861be02fb3 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/files/irqbalance @@ -0,0 +1,25 @@ +# irqbalance is a daemon process that distributes interrupts across +# CPUS on SMP systems. The default is to rebalance once every 10 +# seconds. This is the environment file that is specified to systemd via the +# EnvironmentFile key in the service unit file (or via whatever method the init +# system you're using has. +# +# ONESHOT=yes +# after starting, wait for a minute, then look at the interrupt +# load and balance it once; after balancing exit and do not change +# it again. +#IRQBALANCE_ONESHOT= + +# +# IRQBALANCE_BANNED_CPUS +# 64 bit bitmask which allows you to indicate which cpu's should +# be skipped when reblancing irqs. Cpu numbers which have their +# corresponding bits set to one in this mask will not have any +# irq's assigned to them on rebalance +# +IRQBALANCE_BANNED_CPUS="{{ ansible_processor_cores | irqbalance_banned_cpu_mask(ansible_processor_count, ansible_processor_threads_per_core) }}" +# +# IRQBALANCE_ARGS +# append any args here to the irqbalance daemon as documented in the man page +# +#IRQBALANCE_ARGS= diff --git a/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py b/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py new file mode 100644 index 0000000000..d76f6fe166 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py @@ -0,0 +1,29 @@ +"""Extra Ansible filters""" + +def irqbalance_banned_cpu_mask( + processor_cores, processor_count, processor_threads_per_core): + """ + Return irqbalance CPU mask. + Args: + processor_cores (int): Physical processor unit. + processor_counts (int): Processors per physical unit. + processor_threads_per_core (int): Threads per physical unit. + Returns: + str: irqbalance_banned_cpus. + """ + mask = int("1" * 128, 2) + + for i in range(processor_count * processor_threads_per_core): + mask &= ~(1 << i * processor_cores) + + import re + return ",".join(re.findall('.{1,8}', str(hex(mask))[2:])) + + +class FilterModule(object): + """Return filter plugin""" + + @staticmethod + def filters(): + """Return filter""" + return {'irqbalance_banned_cpu_mask': irqbalance_banned_cpu_mask} diff --git a/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml b/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml new file mode 100644 index 0000000000..fa2876b7ac --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml @@ -0,0 +1,13 @@ +--- +# file roles/performance_tuning/handlers/main.yaml + +- name: Update GRUB + command: update-grub + tags: + - update-grub + +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml b/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml new file mode 100644 index 0000000000..e9cdd0d819 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml @@ -0,0 +1,189 @@ +--- +# file: roles/performance_tuning/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + apt: + update_cache: yes + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - perf-inst-prerequisites + +- name: Inst - Machine Prerequisites + package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - perf-inst-prerequisites + +- name: Conf - Turbo Boost + import_tasks: turbo_boost.yaml + when: > + cpu_microarchitecture == "skylake" or + cpu_microarchitecture == "cascadelake" + tags: + - perf-conf-turbo-boost + +- name: Conf - Adjust nr_hugepages + # change the minimum size of the hugepage pool. + # 2G VPP, 4GB per VNF/CNF, 2G reserve + sysctl: + name: "vm.nr_hugepages" + value: "{{ sysctl.vm.nr_hugepages }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust max_map_count + # this file contains the maximum number of memory map areas a process + # may have. memory map areas are used as a side-effect of calling + # malloc, directly by mmap and mprotect, and also when loading shared + # libraries. + # + # while most applications need less than a thousand maps, certain + # programs, particularly malloc debuggers, may consume lots of them, + # e.g., up to one or two maps per allocation. + # must be greater than or equal to (2 * vm.nr_hugepages). + sysctl: + name: "vm.max_map_count" + value: "{{ sysctl.vm.nr_hugepages * 4 }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust hugetlb_shm_group + # hugetlb_shm_group contains group id that is allowed to create sysv + # shared memory segment using hugetlb page. + sysctl: + name: "vm.hugetlb_shm_group" + value: "1000" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust swappiness + # this control is used to define how aggressive the kernel will swap + # memory pages. higher values will increase agressiveness, lower values + # decrease the amount of swap. a value of 0 instructs the kernel not to + # initiate swap until the amount of free and file-backed pages is less + # than the high water mark in a zone. + sysctl: + name: "vm.swappiness" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust shmmax + # shared memory max must be greator or equal to the total size of hugepages. + # for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 + # if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) + # is greater than the calculated totalhugepagesize then set this parameter + # to current shmmax value. + sysctl: + name: "kernel.shmmax" + value: "{{ sysctl.vm.nr_hugepages * 2 * 1024 * 1024 }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust watchdog_cpumask + # this value can be used to control on which cpus the watchdog may run. + # the default cpumask is all possible cores, but if no_hz_full is + # enabled in the kernel config, and cores are specified with the + # nohz_full= boot argument, those cores are excluded by default. + # offline cores can be included in this mask, and if the core is later + # brought online, the watchdog will be started based on the mask value. + # + # typically this value would only be touched in the nohz_full case + # to re-enable cores that by default were not running the watchdog, + # if a kernel lockup was suspected on those cores. + sysctl: + name: "kernel.watchdog_cpumask" + value: "{{ sysctl.kernel.watchdog_cpumask }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust randomize_va_space + # this option can be used to select the type of process address + # space randomization that is used in the system, for architectures + # that support this feature. + # 0 - turn the process address space randomization off. this is the + # default for architectures that do not support this feature anyways, + # and kernels that are booted with the "norandmaps" parameter. + sysctl: + name: "kernel.randomize_va_space" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Cpufrequtils + copy: + src: "files/cpufrequtils" + dest: "/etc/default/cpufrequtils" + owner: "root" + group: "root" + mode: 0644 + tags: + - perf-conf-cpufrequtils + +- name: Conf - Irqbalance + template: + src: "files/irqbalance" + dest: "/etc/default/irqbalance" + owner: "root" + group: "root" + mode: 0644 + tags: + - perf-conf-irqbalance + +- name: Conf - Set Ondemand Service To Disable + service: + name: "ondemand" + enabled: "no" + tags: + - perf-conf-ondemand + +- name: Conf - Kernel Parameters + lineinfile: + path: "/etc/default/grub" + state: "present" + regexp: "^GRUB_CMDLINE_LINUX=" + line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value is sameas true %}{{key}} {% else %}{{key}}={{value}} {% endif %}{% endfor %}\"" + notify: + - "Update GRUB" + tags: + - perf-conf-grub + +- meta: flush_handlers + +- name: Conf - Load Kernel Modules By Default + lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "vfio-pci" + notify: + - "Reboot Server" + tags: + - perf-conf-load-kernel-modules + +- meta: flush_handlers
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml new file mode 100644 index 0000000000..7f69365a2d --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml @@ -0,0 +1,44 @@ +--- +# file: roles/performance_tuning/tasks/turbo_boost.yaml + +- name: Inst - Update Package Cache (APT) + apt: + update_cache: yes + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - turbo-inst-prerequisites + +- name: Inst - msr-tools + package: + name: + - "msr-tools" + state: latest + tags: + - turbo-inst-prerequisites + +- name: Conf - Load msr By Default + lineinfile: + path: "/etc/modules" + state: "present" + line: "msr" + tags: + - turbo-conf-msr + +- name: Conf - Custom Startup Service Hook + copy: + src: "files/disable-turbo-boost.service" + dest: "/etc/systemd/system/disable-turbo-boost.service" + owner: "root" + group: "root" + mode: 0644 + tags: + - turbo-conf-msr + +- name: Conf - Custom Startup Service Hook Enable + service: + name: "disable-turbo-boost" + enabled: yes + tags: + - turbo-conf-msr |