diff options
Diffstat (limited to 'resources/tools/testbed-setup/ansible/roles')
16 files changed, 197 insertions, 86 deletions
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list new file mode 100644 index 0000000000..08da1ecf96 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list @@ -0,0 +1,47 @@ +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic main restricted +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic universe +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic universe +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates universe +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic multiverse +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic multiverse +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates multiverse +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted universe multiverse +deb-src http://ca.ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted universe multiverse + +deb http://ports.ubuntu.com/ubuntu-ports bionic-security main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports bionic-security main restricted +deb http://ports.ubuntu.com/ubuntu-ports bionic-security universe +deb-src http://ports.ubuntu.com/ubuntu-ports bionic-security universe +deb http://ports.ubuntu.com/ubuntu-ports bionic-security multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports bionic-security multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu bionic partner +# deb-src http://archive.canonical.com/ubuntu bionic partner diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml index 06058a427b..38d8188be8 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml @@ -2,8 +2,8 @@ # file: roles/common/tasks/main.yaml - name: Ubuntu specific - import_tasks: ubuntu_x86_64.yaml - when: ansible_distribution|lower == 'ubuntu' and ansible_machine == 'x86_64' + import_tasks: ubuntu.yaml + when: ansible_distribution|lower == 'ubuntu' - name: Set hostname hostname: @@ -17,7 +17,7 @@ line: '{{ ansible_default_ipv4.address }} {{ hostname }}.linuxfoundation.org' tags: set-hostname -- name: Set sudoers +- name: Set sudoers admin lineinfile: path: '/etc/sudoers' state: 'present' @@ -26,7 +26,7 @@ validate: '/usr/sbin/visudo -cf %s' tags: set-sudoers -- name: Set sudoers +- name: Set sudoers sudo lineinfile: path: '/etc/sudoers' state: 'present' diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml index b6116c3690..ff47ffc656 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_x86_64.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml @@ -1,11 +1,19 @@ --- -# file: roles/common/tasks/ubuntu_x86_64.yaml +# file: roles/common/tasks/ubuntu.yaml -- name: Copy apt sources file +- name: Copy aarch64 apt sources file + template: + src: 'files/apt-sources-arm.list' + dest: '/etc/apt/sources.list' + tags: copy-apt-sources + when: ansible_machine == 'aarch64' + +- name: Copy x86_64 apt sources file template: src: 'files/apt-sources.list' dest: '/etc/apt/sources.list' tags: copy-apt-sources + when: ansible_machine == 'x86_64' - name: Install python-apt apt: @@ -44,7 +52,7 @@ - name: Install qemu apt: - name: 'qemu-system-x86' + name: 'qemu-system' state: 'present' update_cache: True tags: install-qemu diff --git a/resources/tools/testbed-setup/ansible/roles/sut/files/taishan_workaround.service b/resources/tools/testbed-setup/ansible/roles/sut/files/taishan_workaround.service new file mode 100644 index 0000000000..05f865f407 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/sut/files/taishan_workaround.service @@ -0,0 +1,11 @@ +[Unit] +Description=Numa node workaround configuration + +[Service] +ExecStart=/bin/sh -c 'echo 1 > /sys/bus/pci/devices/000a:11:00.0/numa_node' +ExecStart=/bin/sh -c 'echo 1 > /sys/bus/pci/devices/000a:11:00.1/numa_node' +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml index 0ecdcdba3b..f5a2dd86ad 100644 --- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml @@ -2,8 +2,12 @@ # file: roles/sut/tasks/main.yaml - name: Ubuntu specific - import_tasks: ubuntu_x86_64.yaml - when: ansible_distribution|lower == 'ubuntu' and ansible_machine == 'x86_64' + import_tasks: ubuntu.yaml + when: ansible_distribution|lower == 'ubuntu' + +- name: Taishan aarch64 specific + import_tasks: taishan_workaround.yaml + when: taishan_workaround is defined - name: Copy 80-vpp.conf file: diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/taishan_workaround.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/taishan_workaround.yaml new file mode 100644 index 0000000000..e262e7288b --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/taishan_workaround.yaml @@ -0,0 +1,30 @@ +--- +# file: roles/sut/tasks/taishan_workaround.yaml + +- name: Ensure systemd directory exists + file: + path: "/etc/systemd/system" + state: "directory" + owner: "root" + group: "root" + mode: "0755" + tags: taishan-workaround + +- name: Copy systemd numa config unit file + template: + src: "files/taishan_workaround.service" + dest: "/etc/systemd/system/nic-numa-config.service" + owner: "root" + group: "root" + mode: "0644" + register: numa_config_service + tags: taishan-workaround + +- name: Reload systemd daemon + command: "systemctl daemon-reload" + when: (numa_config_service and numa_config_service is changed) + tags: taishan-workaround + +- name: Enable numa config service + command: "systemctl enable nic-numa-config.service" + tags: taishan-workaround diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml index 38946b6bbb..824f0926b5 100644 --- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_x86_64.yaml +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml @@ -1,5 +1,5 @@ --- -# file: roles/sut/tasks/ubuntu_x86_64.yaml +# file: roles/sut/tasks/ubuntu.yaml - name: Install DKMS apt: diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/defaults/main.yaml index 174dc78cb0..3a6f68fb8b 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/defaults/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/defaults/main.yaml @@ -16,8 +16,7 @@ docker_daemon_options: [] # - "NO_PROXY=localhost,127.0.0.1" docker_daemon_environment: [] -docker_apt_key: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88" -docker_repository: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} {{ docker_channel }}" +docker_repository: "deb https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} {{ docker_channel }}" docker_apt_package_name: "{{ docker_version }}~{{ docker_edition }}~3-0~{{ ansible_distribution | lower }}" apt_cache_time: 86400 @@ -25,6 +24,5 @@ apt_cache_time: 86400 kubernetes_channel: "main" kubernetes_version: "1.11.0-00" -kubernetes_apt_key: "54A647F9048D5688D7DA2ABE6A030B21BA07F4FB" kubernetes_repository: "deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_channel }}" kubernetes_apt_package_name: "{{ kubernetes_version }}" diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit index 4a57a7c131..e96547e4d7 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/90-csit @@ -77,4 +77,4 @@ kernel.randomize_va_space=0 # typically this value would only be touched in the nohz_full case # to re-enable cores that by default were not running the watchdog, # if a kernel lockup was suspected on those cores. -kernel.watchdog_cpumask=0,18 +kernel.watchdog_cpumask={{ cfs_cpus }} diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/interfaces_physical b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/interfaces_physical deleted file mode 100644 index 734d8cd18f..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/interfaces_physical +++ /dev/null @@ -1,14 +0,0 @@ -# This file describes the network interfaces available on your system -# and how to activate them. For more information, see interfaces(5). - -# The loopback network interface -auto lo -iface lo inet loopback - -# The primary network interface -auto {{ ansible_default_ipv4["interface"] }} -iface {{ ansible_default_ipv4["interface"] }} inet static - address {{ ansible_default_ipv4["address"] }} - netmask {{ ansible_default_ipv4["netmask"] }} - gateway {{ ansible_default_ipv4["gateway"] }} - dns-nameservers 199.204.44.24 199.204.47.54 diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config new file mode 100644 index 0000000000..2f34cfbaca --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config @@ -0,0 +1,11 @@ +# This file describes the network interfaces available on your system +# For more information, see netplan(5). +network: + version: 2 + renderer: networkd + ethernets: + {{ ansible_default_ipv4["interface"] }}: + addresses: [ {{ (ansible_default_ipv4.address + '/' + ansible_default_ipv4.netmask) | ipaddr('host/prefix') }} ] + gateway4: {{ ansible_default_ipv4["gateway"] }} + nameservers: + addresses: [ 199.204.44.24, 199.204.47.54 ] diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml new file mode 100644 index 0000000000..afaf5cd06b --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/aarch64.yaml @@ -0,0 +1,12 @@ +--- +# file: roles/tg_sut/tasks/aarch64.yaml + +- name: Configure aarch64 kernel parameters + lineinfile: + path: '/etc/default/grub' + state: 'present' + regexp: '^GRUB_CMDLINE_LINUX=' + line: 'GRUB_CMDLINE_LINUX="isolcpus={{ isolcpus }} nohz_full={{ isolcpus }} rcu_nocbs={{ isolcpus }} intel_iommu=on nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1"' + notify: ['Update GRUB'] + tags: set-grub + diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml index ea779c07da..e5da2fc72d 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml @@ -2,19 +2,31 @@ # file: roles/tg_sut/tasks/main.yaml - name: Ubuntu specific - import_tasks: ubuntu_x86_64.yaml - when: ansible_distribution|lower == 'ubuntu' and ansible_machine == 'x86_64' + import_tasks: ubuntu.yaml + when: ansible_distribution|lower == 'ubuntu' -- name: Copy interfaces file +- name: x86 specific + import_tasks: x86_64.yaml + when: ansible_machine == 'x86_64' + +- name: skylake specific + import_tasks: skylake.yaml + when: ("skylake" in groups) and inventory_hostname in groups['skylake'] + +- name: aarch specific + import_tasks: aarch64.yaml + when: ansible_machine == 'aarch64' + +- name: Copy netplan network config file template: - src: 'files/interfaces_physical' - dest: '/etc/network/interfaces' + src: 'files/netplan_config' + dest: '/etc/netplan/01-netcfg.yaml' owner: 'root' group: 'root' mode: '0644' tags: copy-interface-file -- name: Copy sysctl file +- name: Copy CSIT sysctl file template: src: 'files/90-csit' dest: '/etc/sysctl.d/90-csit.conf' @@ -47,15 +59,6 @@ dest: '/tmp/requirements.txt' tags: copy-pip -- name: Set isolcpus and pstate parameter - lineinfile: - path: '/etc/default/grub' - state: 'present' - regexp: '^GRUB_CMDLINE_LINUX=' - line: 'GRUB_CMDLINE_LINUX="isolcpus={{ isolcpus }} nohz_full={{ isolcpus }} rcu_nocbs={{ isolcpus }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"' - notify: ['Update GRUB'] - tags: set-grub - - name: Set ondemand service to disable service: name: 'ondemand' @@ -73,29 +76,3 @@ state: 'present' line: 'vfio-pci' tags: load-vfio-pci - -- name: Load msr by default - lineinfile: - path: '/etc/modules' - state: 'present' - line: 'msr' - tags: disable-turbo-boost - -- name: Disable Turbo-Boost by service - copy: - src: 'files/disable-turbo-boost.service' - dest: '/etc/systemd/system/disable-turbo-boost.service' - owner: 'root' - group: 'root' - mode: '0644' - when: > - inventory_hostname in groups['skylake'] - tags: disable-turbo-boost - -- name: Disable Turbo-Boost by service on startup - service: - name: disable-turbo-boost - enabled: yes - when: > - inventory_hostname in groups['skylake'] - tags: disable-turbo-boost diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/skylake.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/skylake.yaml new file mode 100644 index 0000000000..03b4714d3e --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/skylake.yaml @@ -0,0 +1,17 @@ +--- +# file: roles/tg_sut/tasks/skylake.yaml + +- name: Disable Skylake Turbo-Boost by service + copy: + src: 'files/disable-turbo-boost.service' + dest: '/etc/systemd/system/disable-turbo-boost.service' + owner: 'root' + group: 'root' + mode: '0644' + tags: disable-turbo-boost + +- name: Disable Skylake Turbo-Boost by service on startup + service: + name: disable-turbo-boost + enabled: yes + tags: disable-turbo-boost diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml index 76af5be7fd..8604185aa2 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_x86_64.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml @@ -1,5 +1,5 @@ --- -# file: roles/tg_sut/tasks/ubuntu_x86_64.yaml +# file: roles/tg_sut/tasks/ubuntu.yaml - name: Install python-dev apt: @@ -69,17 +69,13 @@ - "cron" tags: install-docker -- name: Install upstream APT GPG key +- name: Install Docker APT GPG key apt_key: - id: "{{ docker_apt_key }}" - keyserver: "{{ ansible_local.core.keyserver - if (ansible_local|d() and ansible_local.core|d() and - ansible_local.core.keyserver) - else 'hkp://pool.sks-keyservers.net' }}" + url: https://download.docker.com/linux/ubuntu/gpg state: "present" tags: install-docker -- name: Install upstream APT repository +- name: Install Docker APT repository apt_repository: repo: "{{ docker_repository }}" state: "present" @@ -95,7 +91,7 @@ cache_valid_time: "{{ apt_cache_time }}" tags: install-docker -- name: Remove Upstart config file +- name: Remove Upstart docker config file file: path: "/etc/default/docker" state: "absent" @@ -110,7 +106,7 @@ mode: "0755" tags: ensure-docker -- name: Copy systemd unit file +- name: Copy systemd docker unit file template: src: "templates/docker.service.j2" dest: "/etc/systemd/system/docker.service" @@ -136,17 +132,13 @@ when: docker_users tags: set-docker -- name: Install upstream APT GPG key +- name: Install kubernetes APT GPG key apt_key: - id: "{{ kubernetes_apt_key }}" - keyserver: "{{ ansible_local.core.keyserver - if (ansible_local|d() and ansible_local.core|d() and - ansible_local.core.keyserver) - else 'hkp://pool.sks-keyservers.net' }}" + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg state: "present" tags: install-kubernetes -- name: Install upstream APT repository +- name: Install kubernetes APT repository apt_repository: repo: "{{ kubernetes_repository }}" state: "present" diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml new file mode 100644 index 0000000000..5c7110e289 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml @@ -0,0 +1,18 @@ +--- +# file: roles/tg_sut/tasks/x86_64.yaml + +- name: Configure x86_64 kernel parameters + lineinfile: + path: '/etc/default/grub' + state: 'present' + regexp: '^GRUB_CMDLINE_LINUX=' + line: 'GRUB_CMDLINE_LINUX="isolcpus={{ isolcpus }} nohz_full={{ isolcpus }} rcu_nocbs={{ isolcpus }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"' + notify: ['Update GRUB'] + tags: set-grub + +- name: Load msr by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'msr' + tags: disable-turbo-boost |