From d3e0654a57aa958aaef1906e4a759e521655a86b Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Wed, 16 Oct 2019 16:25:41 +0000 Subject: Ansible: Role cleanup + Separate and reuse role (better naming) + Cleanup in naming + Introduce defaults that can be overriden Signed-off-by: Peter Mikus Change-Id: Iedaf7caba1d05a35a6f61cd21a483cdbc0073d32 --- resources/tools/testbed-setup/ansible/cobbler.yaml | 2 + .../inventories/lf_inventory/group_vars/all.yaml | 19 ---- .../ansible/roles/docker/defaults/main.yaml | 27 +++++ .../ansible/roles/docker/handlers/main.yaml | 8 ++ .../ansible/roles/docker/tasks/main.yaml | 54 ++++++++++ .../ansible/roles/docker/tasks/ubuntu_bionic.yaml | 33 +++++++ .../docker/templates/docker.service.proxy.http | 4 + .../docker/templates/docker.service.proxy.https | 4 + .../ansible/roles/kubernetes/defaults/main.yaml | 12 +++ .../ansible/roles/kubernetes/tasks/main.yaml | 14 +++ .../roles/kubernetes/tasks/ubuntu_bionic.yaml | 37 +++++++ .../ansible/roles/performance_tuning/files/90-csit | 80 +++++++++++++++ .../roles/performance_tuning/files/cpufrequtils | 1 + .../files/disable-turbo-boost.service | 10 ++ .../roles/performance_tuning/files/irqbalance | 6 ++ .../roles/performance_tuning/handlers/main.yaml | 16 +++ .../roles/performance_tuning/tasks/aarch64.yaml | 12 +++ .../roles/performance_tuning/tasks/main.yaml | 57 +++++++++++ .../performance_tuning/tasks/turbo_boost.yaml | 33 +++++++ .../performance_tuning/tasks/ubuntu_bionic.yaml | 11 +++ .../roles/performance_tuning/tasks/x86_64.yaml | 13 +++ .../ansible/roles/sut/tasks/ubuntu_bionic.yaml | 1 - .../ansible/roles/tg_sut/files/90-csit | 80 --------------- .../ansible/roles/tg_sut/files/cpufrequtils | 1 - .../roles/tg_sut/files/disable-turbo-boost.service | 10 -- .../ansible/roles/tg_sut/files/irqbalance | 6 -- .../ansible/roles/tg_sut/handlers/main.yaml | 22 ----- .../ansible/roles/tg_sut/tasks/aarch64.yaml | 12 --- .../ansible/roles/tg_sut/tasks/main.yaml | 57 ----------- .../ansible/roles/tg_sut/tasks/turbo_boost.yaml | 33 ------- .../ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml | 109 --------------------- .../ansible/roles/tg_sut/tasks/x86_64.yaml | 13 --- .../tg_sut/templates/docker.service.proxy.http | 4 - .../tg_sut/templates/docker.service.proxy.https | 4 - resources/tools/testbed-setup/ansible/sut.yaml | 8 +- resources/tools/testbed-setup/ansible/tg.yaml | 6 +- .../tools/testbed-setup/ansible/vpp_device.yaml | 2 + 37 files changed, 446 insertions(+), 375 deletions(-) create mode 100644 resources/tools/testbed-setup/ansible/roles/docker/defaults/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/docker/handlers/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/docker/tasks/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/docker/tasks/ubuntu_bionic.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.http create mode 100644 resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.https create mode 100644 resources/tools/testbed-setup/ansible/roles/kubernetes/defaults/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/ubuntu_bionic.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/files/cpufrequtils delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/files/disable-turbo-boost.service delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/files/irqbalance delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/turbo_boost.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.http delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.https diff --git a/resources/tools/testbed-setup/ansible/cobbler.yaml b/resources/tools/testbed-setup/ansible/cobbler.yaml index 06e6238e18..a738fd2d65 100644 --- a/resources/tools/testbed-setup/ansible/cobbler.yaml +++ b/resources/tools/testbed-setup/ansible/cobbler.yaml @@ -8,3 +8,5 @@ roles: - role: cobbler tags: cobbler + - role: docker + tags: docker diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml index 68994ec36b..f53584a939 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml @@ -22,25 +22,6 @@ name_servers: "1.1.1.1, 8.8.8.8" # no_proxy: localhost,127.0.0.1,{{ ansible_default_ipv4.address }} # NO_PROXY: localhost,127.0.0.1,{{ ansible_default_ipv4.address }} -# Docker settings. -docker_edition: 'ce' -docker_channel: 'edge' -docker_version: '18.05.0' -docker_users: ['testuser'] -docker_repository: 'deb https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} {{ docker_channel }}' -docker_apt_package_name: '{{ docker_version }}~{{ docker_edition }}~3-0~{{ ansible_distribution | lower }}' -docker_daemon_environment_http: - - 'HTTP_PROXY={{ proxy_env.http_proxy }}' - - 'NO_PROXY={{ proxy_env.no_proxy }}' -docker_daemon_environment_https: - - 'HTTPS_PROXY={{ proxy_env.https_proxy }}' - - 'NO_PROXY={{ proxy_env.no_proxy }}' - -# Kubernetes settings. -kubernetes: - version: '1.11.0-00' - repository: 'deb http://apt.kubernetes.io/ kubernetes-xenial main' - # TRex settings. trex: target_dir: '/opt' diff --git a/resources/tools/testbed-setup/ansible/roles/docker/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/docker/defaults/main.yaml new file mode 100644 index 0000000000..30958c9fa4 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/docker/defaults/main.yaml @@ -0,0 +1,27 @@ +# Version options. +docker_edition: 'ce' +docker_version: '19.03.3' +docker_apt_package: '5:{{ docker_version }}~3-0~{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}' +docker_apt_package_state: present + +# Service options. +docker_service_state: started +docker_service_enabled: true +docker_restart_handler_state: restarted + +# Used only for Debian/Ubuntu. Switch 'stable' to 'edge' if needed. +docker_apt_release_channel: 'edge' +docker_apt_repository: 'deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} {{ docker_apt_release_channel }}' +docker_apt_repository_state: present + +# A list of users who will be added to the docker group. +docker_users: + - 'testuser' + +# Proxy settings. +docker_daemon_environment_http: + - 'HTTP_PROXY={{ proxy_env.http_proxy }}' + - 'NO_PROXY={{ proxy_env.no_proxy }}' +docker_daemon_environment_https: + - 'HTTPS_PROXY={{ proxy_env.https_proxy }}' + - 'NO_PROXY={{ proxy_env.no_proxy }}' diff --git a/resources/tools/testbed-setup/ansible/roles/docker/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/docker/handlers/main.yaml new file mode 100644 index 0000000000..d33c63fae8 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/docker/handlers/main.yaml @@ -0,0 +1,8 @@ +--- +# file roles/docker/handlers/main.yaml + +- name: Restart Docker + service: + name: 'docker' + state: '{{ docker_restart_handler_state }}' + tags: restart-docker diff --git a/resources/tools/testbed-setup/ansible/roles/docker/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/docker/tasks/main.yaml new file mode 100644 index 0000000000..368ddf5d1a --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/docker/tasks/main.yaml @@ -0,0 +1,54 @@ +--- +# file: roles/docker/tasks/main.yaml + +- name: Docker - Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' + tags: install-docker + +- name: Docker - Create Service Directory + file: + path: '/etc/systemd/system/docker.service.d' + state: 'directory' + tags: install-docker + +- name: Docker - Setup HTTP Proxy + template: + src: 'templates/docker.service.proxy.http' + dest: '/etc/systemd/system/docker.service.d/http-proxy.conf' + owner: 'root' + group: 'root' + mode: '0644' + when: > + proxy_env is defined and + proxy_env.http_proxy is defined + tags: install-docker + +- name: Docker - Setup HTTPS Proxy + template: + src: 'templates/docker.service.proxy.https' + dest: '/etc/systemd/system/docker.service.d/https-proxy.conf' + owner: 'root' + group: 'root' + mode: '0644' + when: > + proxy_env is defined and + proxy_env.https_proxy is defined + tags: install-docker + +- name: Docker - Reload systemd daemon and restart service + command: 'systemctl daemon-reload' + notify: + - 'Restart Docker' + when: > + proxy_env is defined and + proxy_env.http_proxy is defined + tags: install-docker + +- name: Docker - Set specific users to docker group + user: + name: '{{ item }}' + groups: 'docker' + append: True + with_items: '{{ docker_users }}' + when: docker_users + tags: install-docker diff --git a/resources/tools/testbed-setup/ansible/roles/docker/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/docker/tasks/ubuntu_bionic.yaml new file mode 100644 index 0000000000..32643b1119 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/docker/tasks/ubuntu_bionic.yaml @@ -0,0 +1,33 @@ +--- +# file: roles/docker/tasks/ubuntu_bionic.yaml + +- name: Docker repository - Dependencies + apt: + name: + - 'apt-transport-https' + - 'ca-certificates' + - 'software-properties-common' + state: 'present' + cache_valid_time: 3600 + install_recommends: False + tags: install-docker + +- name: Docker repository - Add an Apt signing key + apt_key: + url: 'https://download.docker.com/linux/ubuntu/gpg' + state: 'present' + tags: install-docker + +- name: Docker repository - Install APT repository + apt_repository: + repo: '{{ docker_apt_repository }}' + state: '{{ docker_apt_repository_state }}' + update_cache: True + tags: install-docker + +- name: Install Docker + apt: + name: 'docker-{{ docker_edition }}={{ docker_apt_package }}' + state: '{{ docker_apt_package_state }}' + force: True + tags: install-docker diff --git a/resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.http b/resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.http new file mode 100644 index 0000000000..73ceba3870 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.http @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +[Service] +Environment="{{ docker_daemon_environment_http | join('" "') }}" diff --git a/resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.https b/resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.https new file mode 100644 index 0000000000..1c2097eb9d --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/docker/templates/docker.service.proxy.https @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +[Service] +Environment="{{ docker_daemon_environment_https | join('" "') }}" diff --git a/resources/tools/testbed-setup/ansible/roles/kubernetes/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/kubernetes/defaults/main.yaml new file mode 100644 index 0000000000..ed936e5da5 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/kubernetes/defaults/main.yaml @@ -0,0 +1,12 @@ +# Version options. +kubernetes_version: '1.11.0-00' +kubernetes_apt_package_state: present + +# Service options. +kubernetes_service_state: started +kubernetes_service_enabled: true +kubernetes_restart_handler_state: restarted + +# APT options. +kubernetes_apt_repository: 'deb http://apt.kubernetes.io/ kubernetes-xenial main' +kubernetes_apt_repository_state: present diff --git a/resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/main.yaml new file mode 100644 index 0000000000..160ffb8c06 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/main.yaml @@ -0,0 +1,14 @@ +--- +# file: roles/kubernetes/tasks/main.yaml + +- name: Kubernetes - Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' + tags: install-kubernetes + +- name: Kubernetes - Apply kubelet parameter + lineinfile: + path: '/etc/default/kubelet' + state: 'present' + regexp: '^KUBELET_EXTRA_ARGS=*' + line: 'KUBELET_EXTRA_ARGS=--feature-gates HugePages=false' + tags: install-kubernetes diff --git a/resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/ubuntu_bionic.yaml new file mode 100644 index 0000000000..ddb885f6ee --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/kubernetes/tasks/ubuntu_bionic.yaml @@ -0,0 +1,37 @@ +--- +# file: roles/kubernetes/tasks/ubuntu_bionic.yaml + +- name: Kubernetes repository - Dependencies + apt: + name: + - 'apt-transport-https' + - 'ca-certificates' + - 'software-properties-common' + state: 'present' + cache_valid_time: 3600 + install_recommends: False + tags: install-kubernetes + +- name: Kubernetes repository - Add an Apt signing key + apt_key: + url: 'https://packages.cloud.google.com/apt/doc/apt-key.gpg' + state: 'present' + tags: install-kubernetes + +- name: Kubernetes repository - Install APT repository + apt_repository: + repo: '{{ kubernetes_apt_repository }}' + state: '{{ kubernetes_apt_repository_state }}' + update_cache: True + tags: install-kubernetes + +- name: Kubernetes - Install + apt: + name: + - 'kubernetes-cni=0.6.0-00' + - 'kubeadm={{ kubernetes_version }}' + - 'kubectl={{ kubernetes_version }}' + - 'kubelet={{ kubernetes_version }}' + state: '{{ kubernetes_apt_package_state }}' + force: True + tags: install-kubernetes diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit new file mode 100644 index 0000000000..0c9036e936 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/90-csit @@ -0,0 +1,80 @@ +# change the minimum size of the hugepage pool. +# 2G VPP, 16x2G for Fullbox VM, 2G reserve +vm.nr_hugepages={{ sysctl.vm.nr_hugepages }} + +# this file contains the maximum number of memory map areas a process +# may have. memory map areas are used as a side-effect of calling +# malloc, directly by mmap and mprotect, and also when loading shared +# libraries. +# +# while most applications need less than a thousand maps, certain +# programs, particularly malloc debuggers, may consume lots of them, +# e.g., up to one or two maps per allocation. +# must be greater than or equal to (2 * vm.nr_hugepages). +vm.max_map_count={{ sysctl.vm.max_map_count }} + +# hugetlb_shm_group contains group id that is allowed to create sysv +# shared memory segment using hugetlb page. +vm.hugetlb_shm_group=0 + +# this control is used to define how aggressive the kernel will swap +# memory pages. higher values will increase agressiveness, lower values +# decrease the amount of swap. a value of 0 instructs the kernel not to +# initiate swap until the amount of free and file-backed pages is less +# than the high water mark in a zone. +vm.swappiness=0 + +# this parameter can be used to control the nmi watchdog +# (i.e. the hard lockup detector) on x86 systems. +# +# 0 - disable the hard lockup detector +# 1 - enable the hard lockup detector +# +# the hard lockup detector monitors each cpu for its ability to respond to +# timer interrupts. the mechanism utilizes cpu performance counter registers +# that are programmed to generate non-maskable interrupts (nmis) periodically +# while a cpu is busy. hence, the alternative name 'nmi watchdog'. +# +# the nmi watchdog is disabled by default if the kernel is running as a guest +# in a kvm virtual machine. this default can be overridden by adding +#kernel. nmi_watchdog=1 + +# shared memory max must be greator or equal to the total size of hugepages. +# for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 +# if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) +# is greater than the calculated totalhugepagesize then set this parameter +# to current shmmax value. +kernel.shmmax=8589934592 + +# this option can be used to select the type of process address +# space randomization that is used in the system, for architectures +# that support this feature. +# 0 - turn the process address space randomization off. this is the +# default for architectures that do not support this feature anyways, +# and kernels that are booted with the "norandmaps" parameter. +kernel.randomize_va_space=0 + +# this parameter can be used to control the soft lockup detector. +# +# 0 - disable the soft lockup detector +# 1 - enable the soft lockup detector +# +# the soft lockup detector monitors cpus for threads that are hogging the cpus +# without rescheduling voluntarily, and thus prevent the 'watchdog/n' threads +# from running. the mechanism depends on the cpus ability to respond to timer +# interrupts which are needed for the 'watchdog/n' threads to be woken up by +# the watchdog timer function, otherwise the nmi watchdog - if enabled - can +# detect a hard lockup condition. +#kernel.soft_watchdog=0 + +# this value can be used to control on which cpus the watchdog may run. +# the default cpumask is all possible cores, but if no_hz_full is +# enabled in the kernel config, and cores are specified with the +# nohz_full= boot argument, those cores are excluded by default. +# offline cores can be included in this mask, and if the core is later +# brought online, the watchdog will be started based on the mask value. +# +# typically this value would only be touched in the nohz_full case +# to re-enable cores that by default were not running the watchdog, +# if a kernel lockup was suspected on those cores. +kernel.watchdog_cpumask={{ sysctl.kernel.watchdog_cpumask }} diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils new file mode 100644 index 0000000000..03070fefe1 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/cpufrequtils @@ -0,0 +1 @@ +GOVERNOR="performance" diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service new file mode 100644 index 0000000000..e04729de50 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/disable-turbo-boost.service @@ -0,0 +1,10 @@ +[Unit] +Description=Disable Turbo Boost on Intel CPU + +[Service] +ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done' +ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done' +RemainAfterExit=yes + +[Install] +WantedBy=sysinit.target diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance new file mode 100644 index 0000000000..84fb5f17e2 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/files/irqbalance @@ -0,0 +1,6 @@ +#Configuration for the irqbalance daemon + +#Should irqbalance be enabled? +ENABLED="0" +#Balance the IRQs only once? +ONESHOT="0" diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml new file mode 100644 index 0000000000..ed758d8dfc --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/handlers/main.yaml @@ -0,0 +1,16 @@ +--- +# file roles/performance_tuning/handlers/main.yaml + +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: reboot-server + +- name: Wait for server to restart + wait_for: + host: '{{ inventory_hostname }}' + search_regex: OpenSSH + port: 22 + delay: 60 + timeout: 3600 + tags: reboot-server diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml new file mode 100644 index 0000000000..b5ff02a9ca --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/aarch64.yaml @@ -0,0 +1,12 @@ +--- +# file: roles/performance_tuning/tasks/aarch64.yaml + +- name: Configure {{ ansible_machine }} kernel parameters + lineinfile: + path: '/etc/default/grub' + state: 'present' + regexp: '^GRUB_CMDLINE_LINUX=' + line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} intel_iommu=on nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1"' + notify: + - 'Update GRUB' + tags: set-grub diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml new file mode 100644 index 0000000000..348bccbb33 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml @@ -0,0 +1,57 @@ +--- +# file: roles/performance_tuning/tasks/main.yaml + +- name: Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' + tags: install-csit-dependencies + +- name: Machine specifics + include_tasks: '{{ ansible_machine }}.yaml' + tags: set-grub + +- name: Turbo boost + import_tasks: turbo_boost.yaml + when: > + cpu_microarchitecture == "skylake" or + cpu_microarchitecture == "cascadelake" + tags: turbo-boost + +- name: Copy CSIT sysctl file + template: + src: 'files/90-csit' + dest: '/etc/sysctl.d/90-csit.conf' + owner: 'root' + group: 'root' + mode: '0644' + tags: copy-90-csit + +- name: Copy IRQ load balancing file + copy: + src: 'files/irqbalance' + dest: '/etc/default/irqbalance' + owner: 'root' + group: 'root' + mode: '0644' + tags: copy-irq + +- name: Copy cpufrequtils file + copy: + src: 'files/cpufrequtils' + dest: '/etc/default/cpufrequtils' + owner: 'root' + group: 'root' + mode: '0644' + tags: copy-cpufrequtils + +- name: Set ondemand service to disable + service: + name: 'ondemand' + enabled: 'no' + tags: set-ondemand + +- name: Load vfio-pci by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'vfio-pci' + tags: load-vfio-pci diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml new file mode 100644 index 0000000000..a4872091a5 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml @@ -0,0 +1,33 @@ +--- +# file: roles/performance_tuning/tasks/turbo_boost.yaml + +- name: Turbo boost - Install msr-tools + apt: + name: + - 'msr-tools' + state: 'present' + cache_valid_time: 3600 + install_recommends: False + tags: turbo-boost + +- name: Turbo boost - Load msr by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'msr' + tags: turbo-boost + +- name: Turbo boost - Custom startup service hook + copy: + src: 'files/disable-turbo-boost.service' + dest: '/etc/systemd/system/disable-turbo-boost.service' + owner: 'root' + group: 'root' + mode: '0644' + tags: turbo-boost + +- name: Turbo boost - Custom startup service hook enable + service: + name: 'disable-turbo-boost' + enabled: yes + tags: turbo-boost diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml new file mode 100644 index 0000000000..62b64be469 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/ubuntu_bionic.yaml @@ -0,0 +1,11 @@ +--- +# file: roles/performance_tuning/tasks/ubuntu_bionic.yaml + +- name: Install CSIT dependencies + apt: + name: + - 'cpufrequtils' + state: 'present' + cache_valid_time: 3600 + install_recommends: False + tags: install-csit-dependencies diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml new file mode 100644 index 0000000000..7b66300bc4 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/x86_64.yaml @@ -0,0 +1,13 @@ +--- +# file: roles/performance_tuning/tasks/x86_64.yaml + +- name: Configure {{ ansible_machine }} kernel parameters + lineinfile: + path: '/etc/default/grub' + state: 'present' + regexp: '^GRUB_CMDLINE_LINUX=' + line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"' + notify: + - 'Update GRUB' + - 'Reboot server' + tags: set-grub diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml index 087b97f121..c109695db0 100644 --- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml @@ -4,7 +4,6 @@ - name: Install CSIT dependencies apt: name: - - 'cpufrequtils' - 'gdb' - 'libglib2.0-dev' - 'libmbedcrypto1' diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit deleted file mode 100644 index 0c9036e936..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit +++ /dev/null @@ -1,80 +0,0 @@ -# change the minimum size of the hugepage pool. -# 2G VPP, 16x2G for Fullbox VM, 2G reserve -vm.nr_hugepages={{ sysctl.vm.nr_hugepages }} - -# this file contains the maximum number of memory map areas a process -# may have. memory map areas are used as a side-effect of calling -# malloc, directly by mmap and mprotect, and also when loading shared -# libraries. -# -# while most applications need less than a thousand maps, certain -# programs, particularly malloc debuggers, may consume lots of them, -# e.g., up to one or two maps per allocation. -# must be greater than or equal to (2 * vm.nr_hugepages). -vm.max_map_count={{ sysctl.vm.max_map_count }} - -# hugetlb_shm_group contains group id that is allowed to create sysv -# shared memory segment using hugetlb page. -vm.hugetlb_shm_group=0 - -# this control is used to define how aggressive the kernel will swap -# memory pages. higher values will increase agressiveness, lower values -# decrease the amount of swap. a value of 0 instructs the kernel not to -# initiate swap until the amount of free and file-backed pages is less -# than the high water mark in a zone. -vm.swappiness=0 - -# this parameter can be used to control the nmi watchdog -# (i.e. the hard lockup detector) on x86 systems. -# -# 0 - disable the hard lockup detector -# 1 - enable the hard lockup detector -# -# the hard lockup detector monitors each cpu for its ability to respond to -# timer interrupts. the mechanism utilizes cpu performance counter registers -# that are programmed to generate non-maskable interrupts (nmis) periodically -# while a cpu is busy. hence, the alternative name 'nmi watchdog'. -# -# the nmi watchdog is disabled by default if the kernel is running as a guest -# in a kvm virtual machine. this default can be overridden by adding -#kernel. nmi_watchdog=1 - -# shared memory max must be greator or equal to the total size of hugepages. -# for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 -# if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) -# is greater than the calculated totalhugepagesize then set this parameter -# to current shmmax value. -kernel.shmmax=8589934592 - -# this option can be used to select the type of process address -# space randomization that is used in the system, for architectures -# that support this feature. -# 0 - turn the process address space randomization off. this is the -# default for architectures that do not support this feature anyways, -# and kernels that are booted with the "norandmaps" parameter. -kernel.randomize_va_space=0 - -# this parameter can be used to control the soft lockup detector. -# -# 0 - disable the soft lockup detector -# 1 - enable the soft lockup detector -# -# the soft lockup detector monitors cpus for threads that are hogging the cpus -# without rescheduling voluntarily, and thus prevent the 'watchdog/n' threads -# from running. the mechanism depends on the cpus ability to respond to timer -# interrupts which are needed for the 'watchdog/n' threads to be woken up by -# the watchdog timer function, otherwise the nmi watchdog - if enabled - can -# detect a hard lockup condition. -#kernel.soft_watchdog=0 - -# this value can be used to control on which cpus the watchdog may run. -# the default cpumask is all possible cores, but if no_hz_full is -# enabled in the kernel config, and cores are specified with the -# nohz_full= boot argument, those cores are excluded by default. -# offline cores can be included in this mask, and if the core is later -# brought online, the watchdog will be started based on the mask value. -# -# typically this value would only be touched in the nohz_full case -# to re-enable cores that by default were not running the watchdog, -# if a kernel lockup was suspected on those cores. -kernel.watchdog_cpumask={{ sysctl.kernel.watchdog_cpumask }} diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/cpufrequtils b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/cpufrequtils deleted file mode 100644 index 03070fefe1..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/cpufrequtils +++ /dev/null @@ -1 +0,0 @@ -GOVERNOR="performance" diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/disable-turbo-boost.service b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/disable-turbo-boost.service deleted file mode 100644 index e04729de50..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/disable-turbo-boost.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Disable Turbo Boost on Intel CPU - -[Service] -ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done' -ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done' -RemainAfterExit=yes - -[Install] -WantedBy=sysinit.target diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/irqbalance b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/irqbalance deleted file mode 100644 index 84fb5f17e2..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/irqbalance +++ /dev/null @@ -1,6 +0,0 @@ -#Configuration for the irqbalance daemon - -#Should irqbalance be enabled? -ENABLED="0" -#Balance the IRQs only once? -ONESHOT="0" diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml deleted file mode 100644 index 8fbcb29de0..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# file roles/tg_sut/handlers/main.yaml - -- name: Restart Docker - service: - name: 'docker' - state: 'restarted' - tags: restart-docker - -- name: Reboot server - reboot: - reboot_timeout: 3600 - tags: reboot-server - -- name: Wait for server to restart - wait_for: - host: '{{ inventory_hostname }}' - search_regex: OpenSSH - port: 22 - delay: 60 - timeout: 3600 - tags: reboot-server diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml deleted file mode 100644 index 433ccc0eb6..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# file: roles/tg_sut/tasks/aarch64.yaml - -- name: Configure {{ ansible_machine }} kernel parameters - lineinfile: - path: '/etc/default/grub' - state: 'present' - regexp: '^GRUB_CMDLINE_LINUX=' - line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} intel_iommu=on nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1"' - notify: - - 'Update GRUB' - tags: set-grub diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml deleted file mode 100644 index 41985a367c..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml +++ /dev/null @@ -1,57 +0,0 @@ ---- -# file: roles/tg_sut/tasks/main.yaml - -- name: Install distribution - release - machine prerequisites - include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' - tags: [ install-csit-dependencies, install-docker, install-kubernetes ] - -- name: Machine specifics - include_tasks: '{{ ansible_machine }}.yaml' - tags: set-grub - -- name: Turbo boost - import_tasks: turbo_boost.yaml - when: > - cpu_microarchitecture == "skylake" or - cpu_microarchitecture == "cascadelake" - tags: turbo-boost - -- name: Copy CSIT sysctl file - template: - src: 'files/90-csit' - dest: '/etc/sysctl.d/90-csit.conf' - owner: 'root' - group: 'root' - mode: '0644' - tags: copy-90-csit - -- name: Copy IRQ load balancing file - copy: - src: 'files/irqbalance' - dest: '/etc/default/irqbalance' - owner: 'root' - group: 'root' - mode: '0644' - tags: copy-irq - -- name: Copy cpufrequtils file - copy: - src: 'files/cpufrequtils' - dest: '/etc/default/cpufrequtils' - owner: 'root' - group: 'root' - mode: '0644' - tags: copy-cpufrequtils - -- name: Set ondemand service to disable - service: - name: 'ondemand' - enabled: 'no' - tags: set-ondemand - -- name: Load vfio-pci by default - lineinfile: - path: '/etc/modules' - state: 'present' - line: 'vfio-pci' - tags: load-vfio-pci diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/turbo_boost.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/turbo_boost.yaml deleted file mode 100644 index 2fa704cdb0..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/turbo_boost.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# file: roles/tg_sut/tasks/turbo_boost.yaml - -- name: Turbo boost - Install msr-tools - apt: - name: - - 'msr-tools' - state: 'present' - cache_valid_time: 3600 - install_recommends: False - tags: turbo-boost - -- name: Turbo boost - Load msr by default - lineinfile: - path: '/etc/modules' - state: 'present' - line: 'msr' - tags: turbo-boost - -- name: Turbo boost - Custom startup service hook - copy: - src: 'files/disable-turbo-boost.service' - dest: '/etc/systemd/system/disable-turbo-boost.service' - owner: 'root' - group: 'root' - mode: '0644' - tags: turbo-boost - -- name: Turbo boost - Custom startup service hook enable - service: - name: 'disable-turbo-boost' - enabled: yes - tags: turbo-boost diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml deleted file mode 100644 index fed589c432..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml +++ /dev/null @@ -1,109 +0,0 @@ ---- -# file: roles/tg_sut/tasks/ubuntu_bionic.yaml - -- name: Install CSIT dependencies - apt: - name: - - 'apt-transport-https' - - 'ca-certificates' - - 'software-properties-common' - state: 'present' - cache_valid_time: 3600 - install_recommends: False - tags: install-csit-dependencies - -- name: Add an Apt signing key, for docker-ce repository - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: 'present' - tags: install-docker - -- name: Install Docker APT repository - apt_repository: - repo: '{{ docker_repository }}' - state: 'present' - update_cache: True - tags: install-docker - -- name: Install Docker - apt: - name: 'docker-{{ docker_edition }}={{ docker_apt_package_name }}' - state: 'present' - force: yes - tags: install-docker - -- name: Creates Docker service directory - file: - path: '/etc/systemd/system/docker.service.d' - state: 'directory' - tags: install-docker - -- name: Setup Docker http proxy - template: - src: 'templates/docker.service.proxy.http' - dest: '/etc/systemd/system/docker.service.d/http-proxy.conf' - owner: 'root' - group: 'root' - mode: '0644' - register: docker_register_systemd_service - when: proxy_env is defined and proxy_env.http_proxy is defined - tags: install-docker - -- name: Setup Docker https proxy - template: - src: 'templates/docker.service.proxy.https' - dest: '/etc/systemd/system/docker.service.d/https-proxy.conf' - owner: 'root' - group: 'root' - mode: '0644' - register: docker_register_systemd_service - when: proxy_env is defined and proxy_env.https_proxy is defined - tags: install-docker - -- name: Reload systemd daemon - command: 'systemctl daemon-reload' - notify: ['Restart Docker'] - when: (docker_register_systemd_service and - docker_register_systemd_service is changed) - tags: install-docker - -- name: Set specific users to docker group - user: - name: '{{ item }}' - groups: 'docker' - append: True - with_items: '{{ docker_users }}' - when: docker_users - tags: install-docker - -- name: Add an Apt signing key, for Kubernetes repository - apt_key: - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - state: 'present' - tags: install-kubernetes - -- name: Install kubernetes APT repository - apt_repository: - repo: '{{ kubernetes.repository }}' - state: 'present' - update_cache: True - tags: install-kubernetes - -- name: Install Kubernetes - apt: - name: - - 'kubernetes-cni=0.6.0-00' - - 'kubeadm={{ kubernetes.version }}' - - 'kubectl={{ kubernetes.version }}' - - 'kubelet={{ kubernetes.version }}' - state: 'present' - force: yes - tags: install-kubernetes - -- name: Apply kubelet parameter - lineinfile: - path: '/etc/default/kubelet' - state: 'present' - regexp: '^KUBELET_EXTRA_ARGS=*' - line: 'KUBELET_EXTRA_ARGS=--feature-gates HugePages=false' - tags: install-kubernetes diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml deleted file mode 100644 index 5b705eacb8..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# file: roles/tg_sut/tasks/x86_64.yaml - -- name: Configure {{ ansible_machine }} kernel parameters - lineinfile: - path: '/etc/default/grub' - state: 'present' - regexp: '^GRUB_CMDLINE_LINUX=' - line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"' - notify: - - 'Update GRUB' - - 'Reboot server' - tags: set-grub diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.http b/resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.http deleted file mode 100644 index 73ceba3870..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.http +++ /dev/null @@ -1,4 +0,0 @@ -# {{ ansible_managed }} - -[Service] -Environment="{{ docker_daemon_environment_http | join('" "') }}" diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.https b/resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.https deleted file mode 100644 index 1c2097eb9d..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/templates/docker.service.proxy.https +++ /dev/null @@ -1,4 +0,0 @@ -# {{ ansible_managed }} - -[Service] -Environment="{{ docker_daemon_environment_https | join('" "') }}" diff --git a/resources/tools/testbed-setup/ansible/sut.yaml b/resources/tools/testbed-setup/ansible/sut.yaml index a8dbb38331..d93d0b07cf 100644 --- a/resources/tools/testbed-setup/ansible/sut.yaml +++ b/resources/tools/testbed-setup/ansible/sut.yaml @@ -10,7 +10,11 @@ tags: common - role: sut tags: sut - - role: tg_sut - tags: tg_sut + - role: docker + tags: docker + - role: kubernetes + tags: kubernetes + - role: performance_tuning + tags: performance_tuning - role: calibration tags: calibration diff --git a/resources/tools/testbed-setup/ansible/tg.yaml b/resources/tools/testbed-setup/ansible/tg.yaml index 16d2e0fd40..4b5106652b 100644 --- a/resources/tools/testbed-setup/ansible/tg.yaml +++ b/resources/tools/testbed-setup/ansible/tg.yaml @@ -10,7 +10,9 @@ tags: common - role: tg tags: tg - - role: tg_sut - tags: tg_sut + - role: docker + tags: docker + - role: performance_tuning + tags: performance_tuning - role: calibration tags: calibration diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml index 3d4894136f..8353be03c2 100644 --- a/resources/tools/testbed-setup/ansible/vpp_device.yaml +++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml @@ -8,5 +8,7 @@ roles: - role: common tags: common + - role: docker + tags: docker - role: vpp_device tags: vpp_device -- cgit 1.2.3-korg