From d3e0654a57aa958aaef1906e4a759e521655a86b Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Wed, 16 Oct 2019 16:25:41 +0000 Subject: Ansible: Role cleanup + Separate and reuse role (better naming) + Cleanup in naming + Introduce defaults that can be overriden Signed-off-by: Peter Mikus Change-Id: Iedaf7caba1d05a35a6f61cd21a483cdbc0073d32 --- .../ansible/roles/performance_tuning/files/90-csit | 80 ++++++++++++++++++++++ .../roles/performance_tuning/files/cpufrequtils | 1 + .../files/disable-turbo-boost.service | 10 +++ .../roles/performance_tuning/files/irqbalance | 6 ++ .../roles/performance_tuning/handlers/main.yaml | 16 +++++ .../roles/performance_tuning/tasks/aarch64.yaml | 12 ++++ .../roles/performance_tuning/tasks/main.yaml | 57 +++++++++++++++ .../performance_tuning/tasks/turbo_boost.yaml | 33 +++++++++ .../performance_tuning/tasks/ubuntu_bionic.yaml | 11 +++ .../roles/performance_tuning/tasks/x86_64.yaml | 13 ++++ 10 files changed, 239 insertions(+) create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml (limited to 'resources/tools/testbed-setup/ansible/roles/performance_tuning') diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit new file mode 100644 index 0000000000..0c9036e936 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit @@ -0,0 +1,80 @@ +# change the minimum size of the hugepage pool. +# 2G VPP, 16x2G for Fullbox VM, 2G reserve +vm.nr_hugepages={{ sysctl.vm.nr_hugepages }} + +# this file contains the maximum number of memory map areas a process +# may have. memory map areas are used as a side-effect of calling +# malloc, directly by mmap and mprotect, and also when loading shared +# libraries. +# +# while most applications need less than a thousand maps, certain +# programs, particularly malloc debuggers, may consume lots of them, +# e.g., up to one or two maps per allocation. +# must be greater than or equal to (2 * vm.nr_hugepages). +vm.max_map_count={{ sysctl.vm.max_map_count }} + +# hugetlb_shm_group contains group id that is allowed to create sysv +# shared memory segment using hugetlb page. +vm.hugetlb_shm_group=0 + +# this control is used to define how aggressive the kernel will swap +# memory pages. higher values will increase agressiveness, lower values +# decrease the amount of swap. a value of 0 instructs the kernel not to +# initiate swap until the amount of free and file-backed pages is less +# than the high water mark in a zone. +vm.swappiness=0 + +# this parameter can be used to control the nmi watchdog +# (i.e. the hard lockup detector) on x86 systems. +# +# 0 - disable the hard lockup detector +# 1 - enable the hard lockup detector +# +# the hard lockup detector monitors each cpu for its ability to respond to +# timer interrupts. the mechanism utilizes cpu performance counter registers +# that are programmed to generate non-maskable interrupts (nmis) periodically +# while a cpu is busy. hence, the alternative name 'nmi watchdog'. +# +# the nmi watchdog is disabled by default if the kernel is running as a guest +# in a kvm virtual machine. this default can be overridden by adding +#kernel. nmi_watchdog=1 + +# shared memory max must be greator or equal to the total size of hugepages. +# for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 +# if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) +# is greater than the calculated totalhugepagesize then set this parameter +# to current shmmax value. +kernel.shmmax=8589934592 + +# this option can be used to select the type of process address +# space randomization that is used in the system, for architectures +# that support this feature. +# 0 - turn the process address space randomization off. this is the +# default for architectures that do not support this feature anyways, +# and kernels that are booted with the "norandmaps" parameter. +kernel.randomize_va_space=0 + +# this parameter can be used to control the soft lockup detector. +# +# 0 - disable the soft lockup detector +# 1 - enable the soft lockup detector +# +# the soft lockup detector monitors cpus for threads that are hogging the cpus +# without rescheduling voluntarily, and thus prevent the 'watchdog/n' threads +# from running. the mechanism depends on the cpus ability to respond to timer +# interrupts which are needed for the 'watchdog/n' threads to be woken up by +# the watchdog timer function, otherwise the nmi watchdog - if enabled - can +# detect a hard lockup condition. +#kernel.soft_watchdog=0 + +# this value can be used to control on which cpus the watchdog may run. +# the default cpumask is all possible cores, but if no_hz_full is +# enabled in the kernel config, and cores are specified with the +# nohz_full= boot argument, those cores are excluded by default. +# offline cores can be included in this mask, and if the core is later +# brought online, the watchdog will be started based on the mask value. +# +# typically this value would only be touched in the nohz_full case +# to re-enable cores that by default were not running the watchdog, +# if a kernel lockup was suspected on those cores. +kernel.watchdog_cpumask={{ sysctl.kernel.watchdog_cpumask }} diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils new file mode 100644 index 0000000000..03070fefe1 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils @@ -0,0 +1 @@ +GOVERNOR="performance" diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service new file mode 100644 index 0000000000..e04729de50 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service @@ -0,0 +1,10 @@ +[Unit] +Description=Disable Turbo Boost on Intel CPU + +[Service] +ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done' +ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done' +RemainAfterExit=yes + +[Install] +WantedBy=sysinit.target diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance new file mode 100644 index 0000000000..84fb5f17e2 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance @@ -0,0 +1,6 @@ +#Configuration for the irqbalance daemon + +#Should irqbalance be enabled? +ENABLED="0" +#Balance the IRQs only once? +ONESHOT="0" diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml new file mode 100644 index 0000000000..ed758d8dfc --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml @@ -0,0 +1,16 @@ +--- +# file roles/performance_tuning/handlers/main.yaml + +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: reboot-server + +- name: Wait for server to restart + wait_for: + host: '{{ inventory_hostname }}' + search_regex: OpenSSH + port: 22 + delay: 60 + timeout: 3600 + tags: reboot-server diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml new file mode 100644 index 0000000000..b5ff02a9ca --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml @@ -0,0 +1,12 @@ +--- +# file: roles/performance_tuning/tasks/aarch64.yaml + +- name: Configure {{ ansible_machine }} kernel parameters + lineinfile: + path: '/etc/default/grub' + state: 'present' + regexp: '^GRUB_CMDLINE_LINUX=' + line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} intel_iommu=on nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1"' + notify: + - 'Update GRUB' + tags: set-grub diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml new file mode 100644 index 0000000000..348bccbb33 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml @@ -0,0 +1,57 @@ +--- +# file: roles/performance_tuning/tasks/main.yaml + +- name: Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' + tags: install-csit-dependencies + +- name: Machine specifics + include_tasks: '{{ ansible_machine }}.yaml' + tags: set-grub + +- name: Turbo boost + import_tasks: turbo_boost.yaml + when: > + cpu_microarchitecture == "skylake" or + cpu_microarchitecture == "cascadelake" + tags: turbo-boost + +- name: Copy CSIT sysctl file + template: + src: 'files/90-csit' + dest: '/etc/sysctl.d/90-csit.conf' + owner: 'root' + group: 'root' + mode: '0644' + tags: copy-90-csit + +- name: Copy IRQ load balancing file + copy: + src: 'files/irqbalance' + dest: '/etc/default/irqbalance' + owner: 'root' + group: 'root' + mode: '0644' + tags: copy-irq + +- name: Copy cpufrequtils file + copy: + src: 'files/cpufrequtils' + dest: '/etc/default/cpufrequtils' + owner: 'root' + group: 'root' + mode: '0644' + tags: copy-cpufrequtils + +- name: Set ondemand service to disable + service: + name: 'ondemand' + enabled: 'no' + tags: set-ondemand + +- name: Load vfio-pci by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'vfio-pci' + tags: load-vfio-pci diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml new file mode 100644 index 0000000000..a4872091a5 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml @@ -0,0 +1,33 @@ +--- +# file: roles/performance_tuning/tasks/turbo_boost.yaml + +- name: Turbo boost - Install msr-tools + apt: + name: + - 'msr-tools' + state: 'present' + cache_valid_time: 3600 + install_recommends: False + tags: turbo-boost + +- name: Turbo boost - Load msr by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'msr' + tags: turbo-boost + +- name: Turbo boost - Custom startup service hook + copy: + src: 'files/disable-turbo-boost.service' + dest: '/etc/systemd/system/disable-turbo-boost.service' + owner: 'root' + group: 'root' + mode: '0644' + tags: turbo-boost + +- name: Turbo boost - Custom startup service hook enable + service: + name: 'disable-turbo-boost' + enabled: yes + tags: turbo-boost diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml new file mode 100644 index 0000000000..62b64be469 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml @@ -0,0 +1,11 @@ +--- +# file: roles/performance_tuning/tasks/ubuntu_bionic.yaml + +- name: Install CSIT dependencies + apt: + name: + - 'cpufrequtils' + state: 'present' + cache_valid_time: 3600 + install_recommends: False + tags: install-csit-dependencies diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml new file mode 100644 index 0000000000..7b66300bc4 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml @@ -0,0 +1,13 @@ +--- +# file: roles/performance_tuning/tasks/x86_64.yaml + +- name: Configure {{ ansible_machine }} kernel parameters + lineinfile: + path: '/etc/default/grub' + state: 'present' + regexp: '^GRUB_CMDLINE_LINUX=' + line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"' + notify: + - 'Update GRUB' + - 'Reboot server' + tags: set-grub -- cgit 1.2.3-korg