Ansible: Role cleanup

+ Separate and reuse role (better naming)
+ Cleanup in naming
+ Introduce defaults that can be overriden

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: Iedaf7caba1d05a35a6f61cd21a483cdbc0073d32

10 files changed, 239 insertions, 0 deletions

diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit
new file mode 100644
index 0000000000..0c9036e936
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit
@@ -0,0 +1,80 @@
+# change the minimum size of the hugepage pool.
+# 2G VPP, 16x2G for Fullbox VM, 2G reserve
+vm.nr_hugepages={{ sysctl.vm.nr_hugepages }}
+
+# this file contains the maximum number of memory map areas a process
+# may have. memory map areas are used as a side-effect of calling
+# malloc, directly by mmap and mprotect, and also when loading shared
+# libraries.
+#
+# while most applications need less than a thousand maps, certain
+# programs, particularly malloc debuggers, may consume lots of them,
+# e.g., up to one or two maps per allocation.
+# must be greater than or equal to (2 * vm.nr_hugepages).
+vm.max_map_count={{ sysctl.vm.max_map_count }}
+
+# hugetlb_shm_group contains group id that is allowed to create sysv
+# shared memory segment using hugetlb page.
+vm.hugetlb_shm_group=0
+
+# this control is used to define how aggressive the kernel will swap
+# memory pages.  higher values will increase agressiveness, lower values
+# decrease the amount of swap.  a value of 0 instructs the kernel not to
+# initiate swap until the amount of free and file-backed pages is less
+# than the high water mark in a zone.
+vm.swappiness=0
+
+# this parameter can be used to control the nmi watchdog
+# (i.e. the hard lockup detector) on x86 systems.
+#
+#    0 - disable the hard lockup detector
+#    1 - enable the hard lockup detector
+#
+# the hard lockup detector monitors each cpu for its ability to respond to
+# timer interrupts. the mechanism utilizes cpu performance counter registers
+# that are programmed to generate non-maskable interrupts (nmis) periodically
+# while a cpu is busy. hence, the alternative name 'nmi watchdog'.
+#
+# the nmi watchdog is disabled by default if the kernel is running as a guest
+# in a kvm virtual machine. this default can be overridden by adding
+#kernel. nmi_watchdog=1
+
+# shared memory max must be greator or equal to the total size of hugepages.
+# for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024
+# if the existing kernel.shmmax setting  (cat /sys/proc/kernel/shmmax)
+# is greater than the calculated totalhugepagesize then set this parameter
+# to current shmmax value.
+kernel.shmmax=8589934592
+
+# this option can be used to select the type of process address
+# space randomization that is used in the system, for architectures
+# that support this feature.
+# 0 - turn the process address space randomization off.  this is the
+#     default for architectures that do not support this feature anyways,
+#     and kernels that are booted with the "norandmaps" parameter.
+kernel.randomize_va_space=0
+
+# this parameter can be used to control the soft lockup detector.
+#
+#    0 - disable the soft lockup detector
+#    1 - enable the soft lockup detector
+#
+# the soft lockup detector monitors cpus for threads that are hogging the cpus
+# without rescheduling voluntarily, and thus prevent the 'watchdog/n' threads
+# from running. the mechanism depends on the cpus ability to respond to timer
+# interrupts which are needed for the 'watchdog/n' threads to be woken up by
+# the watchdog timer function, otherwise the nmi watchdog - if enabled - can
+# detect a hard lockup condition.
+#kernel.soft_watchdog=0
+
+# this value can be used to control on which cpus the watchdog may run.
+# the default cpumask is all possible cores, but if no_hz_full is
+# enabled in the kernel config, and cores are specified with the
+# nohz_full= boot argument, those cores are excluded by default.
+# offline cores can be included in this mask, and if the core is later
+# brought online, the watchdog will be started based on the mask value.
+#
+# typically this value would only be touched in the nohz_full case
+# to re-enable cores that by default were not running the watchdog,
+# if a kernel lockup was suspected on those cores.
+kernel.watchdog_cpumask={{ sysctl.kernel.watchdog_cpumask }}
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils
new file mode 100644
index 0000000000..03070fefe1
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils
@@ -0,0 +1 @@
+GOVERNOR="performance"
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service
new file mode 100644
index 0000000000..e04729de50
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Disable Turbo Boost on Intel CPU
+
+[Service]
+ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done'
+ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done'
+RemainAfterExit=yes
+
+[Install]
+WantedBy=sysinit.target
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance
new file mode 100644
index 0000000000..84fb5f17e2
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance
@@ -0,0 +1,6 @@
+#Configuration for the irqbalance daemon
+ 
+#Should irqbalance be enabled?
+ENABLED="0"
+#Balance the IRQs only once?
+ONESHOT="0"
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml
new file mode 100644
index 0000000000..ed758d8dfc
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml
@@ -0,0 +1,16 @@
+---
+# file roles/performance_tuning/handlers/main.yaml
+
+- name: Reboot server
+  reboot:
+    reboot_timeout: 3600
+  tags: reboot-server
+
+- name: Wait for server to restart
+  wait_for:
+    host: '{{ inventory_hostname }}'
+    search_regex: OpenSSH
+    port: 22
+    delay: 60
+    timeout: 3600
+  tags: reboot-server
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml
new file mode 100644
index 0000000000..b5ff02a9ca
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml
@@ -0,0 +1,12 @@
+---
+# file: roles/performance_tuning/tasks/aarch64.yaml
+
+- name: Configure {{ ansible_machine }} kernel parameters
+  lineinfile:
+    path: '/etc/default/grub'
+    state: 'present'
+    regexp: '^GRUB_CMDLINE_LINUX='
+    line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} intel_iommu=on nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1"'
+  notify:
+    - 'Update GRUB'
+  tags: set-grub
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml
new file mode 100644
index 0000000000..348bccbb33
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml
@@ -0,0 +1,57 @@
+---
+# file: roles/performance_tuning/tasks/main.yaml
+
+- name: Install distribution - release - machine prerequisites
+  include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
+  tags: install-csit-dependencies
+
+- name: Machine specifics
+  include_tasks: '{{ ansible_machine }}.yaml'
+  tags: set-grub
+
+- name: Turbo boost
+  import_tasks: turbo_boost.yaml
+  when: >
+    cpu_microarchitecture == "skylake" or
+    cpu_microarchitecture == "cascadelake"
+  tags: turbo-boost
+
+- name: Copy CSIT sysctl file
+  template:
+    src: 'files/90-csit'
+    dest: '/etc/sysctl.d/90-csit.conf'
+    owner: 'root'
+    group: 'root'
+    mode: '0644'
+  tags: copy-90-csit
+
+- name: Copy IRQ load balancing file
+  copy:
+    src: 'files/irqbalance'
+    dest: '/etc/default/irqbalance'
+    owner: 'root'
+    group: 'root'
+    mode: '0644'
+  tags: copy-irq
+
+- name: Copy cpufrequtils file
+  copy:
+    src: 'files/cpufrequtils'
+    dest: '/etc/default/cpufrequtils'
+    owner: 'root'
+    group: 'root'
+    mode: '0644'
+  tags: copy-cpufrequtils
+
+- name: Set ondemand service to disable
+  service:
+    name: 'ondemand'
+    enabled: 'no'
+  tags: set-ondemand
+
+- name: Load vfio-pci by default
+  lineinfile:
+    path: '/etc/modules'
+    state: 'present'
+    line: 'vfio-pci'
+  tags: load-vfio-pci
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml
new file mode 100644
index 0000000000..a4872091a5
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml
@@ -0,0 +1,33 @@
+---
+# file: roles/performance_tuning/tasks/turbo_boost.yaml
+
+- name: Turbo boost - Install msr-tools
+  apt:
+    name:
+      - 'msr-tools'
+    state: 'present'
+    cache_valid_time: 3600
+    install_recommends: False
+  tags: turbo-boost
+
+- name: Turbo boost - Load msr by default
+  lineinfile:
+    path: '/etc/modules'
+    state: 'present'
+    line: 'msr'
+  tags: turbo-boost
+
+- name: Turbo boost - Custom startup service hook
+  copy:
+    src: 'files/disable-turbo-boost.service'
+    dest: '/etc/systemd/system/disable-turbo-boost.service'
+    owner: 'root'
+    group: 'root'
+    mode: '0644'
+  tags: turbo-boost
+
+- name: Turbo boost - Custom startup service hook enable
+  service:
+    name: 'disable-turbo-boost'
+    enabled: yes
+  tags: turbo-boost
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml
new file mode 100644
index 0000000000..62b64be469
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml
@@ -0,0 +1,11 @@
+---
+# file: roles/performance_tuning/tasks/ubuntu_bionic.yaml
+
+- name: Install CSIT dependencies
+  apt:
+    name:
+      - 'cpufrequtils'
+    state: 'present'
+    cache_valid_time: 3600
+    install_recommends: False
+  tags: install-csit-dependencies
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml
new file mode 100644
index 0000000000..7b66300bc4
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml
@@ -0,0 +1,13 @@
+---
+# file: roles/performance_tuning/tasks/x86_64.yaml
+
+- name: Configure {{ ansible_machine }} kernel parameters
+  lineinfile:
+    path: '/etc/default/grub'
+    state: 'present'
+    regexp: '^GRUB_CMDLINE_LINUX='
+    line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"'
+  notify:
+    - 'Update GRUB'
+    - 'Reboot server'
+  tags: set-grub