diff options
Diffstat (limited to 'fdio.infra.ansible/roles/aws')
| -rw-r--r-- | fdio.infra.ansible/roles/aws/defaults/main.yaml | 26 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh | 203 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/handlers/main.yaml | 19 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/tasks/main.yaml | 106 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/tasks/ubuntu_focal.yaml | 10 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml | 31 |
6 files changed, 334 insertions, 61 deletions
diff --git a/fdio.infra.ansible/roles/aws/defaults/main.yaml b/fdio.infra.ansible/roles/aws/defaults/main.yaml index d4ea91afd4..5b6978da51 100644 --- a/fdio.infra.ansible/roles/aws/defaults/main.yaml +++ b/fdio.infra.ansible/roles/aws/defaults/main.yaml @@ -1,2 +1,26 @@ --- -# file: roles/aws/defaults/main.yaml +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower] + packages_by_arch[ansible_machine] }}" + +packages_repo: + ubuntu: + aarch64: + "http://ports.ubuntu.com/" + x86_64: + "http://archive.ubuntu.com/ubuntu" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "linux-image-5.4.0-1009-aws" + - "linux-headers-5.4.0-1009-aws" + - "linux-tools-5.4.0-1009-aws" + +packages_by_arch: + aarch64: + - [] + x86_64: + - []
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh new file mode 100644 index 0000000000..02a3139b66 --- /dev/null +++ b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# Enable WC in VFIO-PCI driver +# Tested on: +# * Amazon Linux 2 AMI (HVM), SSD Volume Type - ami-0bb3fad3c0286ebd5 +# * Amazon Linux AMI 2018.03.0 (HVM), SSD Volume Type - ami-015232c01a82b847b +# * Red Hat Enterprise Linux 8 (HVM), SSD Volume Type - ami-08f4717d06813bf00 +# * Ubuntu Server 20.04 LTS (HVM), SSD Volume Type - ami-06fd8a495a537da8b +# * Ubuntu Server 18.04 LTS (HVM), SSD Volume Type - ami-0823c236601fef765 + +set -e + +TMP_DIR="tmp" + +# Kernel modules location: +P1="/usr/lib/modules/`uname -r`/kernel/drivers/vfio" +P2="/lib/modules/`uname -r`/kernel/drivers/vfio" + +# This may return an error if executed from inside the script +set +e +RED="$(tput setaf 1)" +GREEN="$(tput setaf 2)" + +BOLD="$(tput bold)" +NORMAL="$(tput sgr0)" +set -e + +function bold { + echo -e "${BOLD}${@}${NORMAL}" +} + +function err { + bold "${RED}ERROR: ${@}" +} + +function green { + bold "${GREEN}${@}" +} + +function get_kernel_version { + local ver=$(uname -r | cut -f 1 -d '-') + local ver_major=$(echo $ver | cut -f1 -d '.') + local ver_minor=$(echo $ver | cut -f2 -d '.') + local ver_subminor=$(echo $ver | cut -f3 -d '.') + + printf "%d%02d%04d" "${ver_major}" "${ver_minor}" "${ver_subminor}" +} + +function download_kernel_src_yum { + echo "Use yum to get the kernel sources" + + bold "\nInstall required applications and kernel headers" + yum install -y gcc "kernel-$(uname -r)" "kernel-devel-$(uname -r)" \ + git make elfutils-libelf-devel patch yum-utils + green Done + + # Download kernel source + bold "\nDownload kernel source with vfio" + yumdownloader --source "kernel-devel-$(uname -r)" + rpm2cpio kernel*.src.rpm | cpio -idmv + green Done + + rm -f *patches.tar + tar xf linux-*.tar* + rm -f linux-*.tar* linux-*.patch +} + +function download_kernel_src_apt { + echo "Use apt-get to get the kernel sources" + apt-get -q -y update + green Done + + bold "\nInstall required applications" + apt-get -q -y install dpkg-dev build-essential git + green Done + + bold "\nDownload Linux kernel source with vfio" + if ! apt-get -q -y source -t focal linux-image-$(uname -r); then + err "Cannot download Linux kernel source.\nPlease uncomment appropriate 'deb-src' line in the /etc/apt/sources.list file" + exit 1 + fi + green Done + + rm -f linux-*.dsc linux-*.gz +} + +function download_kernel_src { + bold "[1] Downloading prerequisites..." + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + cd "${TMP_DIR}" + + if apt-get -v >/dev/null 2>/dev/null; then + download_kernel_src_apt + else + download_kernel_src_yum + fi + cd linux-* +} + +function apply_wc_patch { + echo "Using patch for kernel version 4.10" + local wc_patch="${BASE_PATH}/patches/linux-4.10-vfio-wc.patch" + + if ! patch --ignore-whitespace -p1 < "${wc_patch}"; then + err "Cannot apply patch: ${wc_patch}!" + exit 1 + fi +} + +function compile_vfio_driver { + bold "\n[2] Patch and build the vfio driver" + # Adjust VFIO-PCI driver + + bold "Apply patch for the write combining to the vfio-pci" + apply_wc_patch + green Done + + cd drivers/vfio + # Configure Makefile - build VFIO with support for NOIOMMU mode + bold "\nConfigure Makefile for standalone vfio build and noiommu mode support" + echo "ccflags-y := -DCONFIG_VFIO_NOIOMMU=1" >> Makefile + echo 'all:' >> Makefile + echo ' make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules' >> Makefile + green Done + + bold "\nBuild the driver" + if ! make; then + err "Compilation error." + exit 1 + fi + green Done +} + +function get_module_location { + for p in ${P1} ${P2}; do + if find "${p}" -name "vfio.*" >/dev/null 2>/dev/null; then + MOD_PATH="${p}" + break + fi + done + + if [ -z "${MOD_PATH}" ]; then + err "Cannot find kernel modules location..." + exit + fi +} + +function get_module_compression { + if ls "${MOD_PATH}/vfio.ko.xz" >/dev/null 2>/dev/null; then + XZ=".xz" + else + XZ="" + fi +} + +function replace_module { + local installed=0 + + bold "\n[3] Install module" + get_module_location + get_module_compression + + for name in "pci/vfio-pci.ko" "pci/vfio-pci-core.ko" "vfio.ko"; do + if test -e "${MOD_PATH}/${name}${XZ}"; then + if [ -n "${XZ}" ]; then + xz "${name}" -c > "${name}${XZ}" + fi + mv "${MOD_PATH}/${name}${XZ}" "${MOD_PATH}/${name}${XZ}_no_wc" + cp "${name}${XZ}" "${MOD_PATH}/${name}${XZ}" + bold "Installing: ${MOD_PATH}/${name}${XZ}" + installed=1 + fi + done + if [ "${installed}" -eq 1 ]; then + green "Module installed at: ${MOD_PATH}" + else + err "Failure during vfio-pci module installation. Prehaps it's not provided as a kernel module!" + exit 1 + fi +} + +############################################### +# Main script code +############################################### + +if [ "$(id -u)" -ne 0 ]; then + err 'Please execute script as a root' + exit 1 +fi + +cd $(dirname ${0}) +BASE_PATH=$(pwd) + +KERNEL_VERSION=$(get_kernel_version) + +if [ "${KERNEL_VERSION}" -lt 4100000 ]; then + err "Kernel version: $(uname -r) is not supported by the script. Please upgrade kernel to at least v4.10." + exit 1 +fi + +download_kernel_src +compile_vfio_driver +replace_module diff --git a/fdio.infra.ansible/roles/aws/handlers/main.yaml b/fdio.infra.ansible/roles/aws/handlers/main.yaml index 7363dc2c34..d55db1c22f 100644 --- a/fdio.infra.ansible/roles/aws/handlers/main.yaml +++ b/fdio.infra.ansible/roles/aws/handlers/main.yaml @@ -1,15 +1,20 @@ --- # file: roles/aws/handlers/main.yaml -- name: Reboot server - reboot: - reboot_timeout: 3600 - tags: - - reboot-server - -- name: AWS - Reload systemd-modules +- name: Reload systemd-modules systemd: name: "systemd-modules-load" state: "restarted" tags: - reload-systemd-modules + +- name: Update GRUB + ansible.builtin.command: update-grub + tags: + - update-grub + +- name: Reboot Server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/aws/tasks/main.yaml b/fdio.infra.ansible/roles/aws/tasks/main.yaml index 2b8c22ccce..b5132c1909 100644 --- a/fdio.infra.ansible/roles/aws/tasks/main.yaml +++ b/fdio.infra.ansible/roles/aws/tasks/main.yaml @@ -1,60 +1,43 @@ --- -# file: roles/aws/tasks/main.yaml +# file: tasks/main.yaml - name: Edit repositories include_tasks: "{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml" tags: - aws-edit-repo -- name: Get vfio-pci With WC Patcher - get_url: - url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh" - dest: "/opt/get-vfio-with-wc.sh" - mode: "744" +- name: Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: "latest" tags: - - aws-vfio-patch - -- name: Create vfio-pci Patch Directory - file: - path: "/opt/patches/" - state: "directory" - tags: - - aws-vfio-patch - -- name: Get vfio-pci WC Patch >=4.10 - get_url: - url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/linux-4.10-vfio-wc.patch" - dest: "/opt/patches/linux-4.10-vfio-wc.patch" - mode: "744" - tags: - - aws-vfio-patch + - aws-inst-prerequisites -- name: Get vfio-pci WC Patch >=5.8 - get_url: - url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/linux-5.8-vfio-wc.patch" - dest: "/opt/patches/linux-5.8-vfio-wc.patch" - mode: "744" +- name: Switch Kernel At Boot + ansible.builtin.lineinfile: + path: "/etc/default/grub" + state: "present" + line: "GRUB_DEFAULT=\"1>2\"" + notify: + - "Update GRUB" tags: - - aws-vfio-patch + - perf-conf-grub -- name: Compile vfio-pci With WC Patch - shell: "/bin/bash /opt/get-vfio-with-wc.sh" - tags: - - aws-vfio-patch +- meta: flush_handlers - name: Load Kernel Modules By Default - lineinfile: + ansible.builtin.lineinfile: path: "/etc/modules" state: "present" line: "{{ item }}" with_items: - - "vfio-pci" - "igb_uio" + - "vfio-pci" tags: - aws-load-kernel-modules - name: Add Kernel Modules Options (igb_uio) - lineinfile: + ansible.builtin.lineinfile: path: "/etc/modprobe.d/igb_uio.conf" state: "present" line: "{{ item }}" @@ -65,7 +48,7 @@ - aws-load-kernel-modules - name: Add Kernel Modules Options (vfio-pci) - lineinfile: + ansible.builtin.lineinfile: path: "/etc/modprobe.d/vfio-noiommu.conf" state: "present" line: "{{ item }}" @@ -75,24 +58,67 @@ tags: - aws-load-kernel-modules +#- name: Get vfio-pci With WC Patcher +# ansible.builtin.get_url: +# url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh" +# dest: "/opt/get-vfio-with-wc.sh" +# mode: 0744 +# tags: +# - aws-vfio-patch + +- name: Create vfio-pci Patch Directory + ansible.builtin.file: + path: "/opt/patches/" + state: "directory" + tags: + - aws-vfio-patch + +- name: Get vfio-pci WC Patch + ansible.builtin.get_url: + url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/{{ item }}" + dest: "/opt/patches/{{ item }}" + mode: 0744 + with_items: + - "linux-4.10-vfio-wc.patch" + - "linux-5.8-vfio-wc.patch" + - "linux-5.15-vfio-wc.patch" + tags: + - aws-vfio-patch + +- name: Copy vfio-pci WC Patch + ansible.builtin.copy: + src: "files/get-vfio-with-wc.sh" + dest: "/opt" + mode: 0744 + tags: + - aws-vfio-patch + +- name: Compile vfio-pci With WC Patch + ansible.builtin.shell: "/bin/bash /opt/get-vfio-with-wc.sh" + environment: + DEBIAN_FRONTEND: "noninteractive" + TERM: "vt100" + tags: + - aws-vfio-patch + - name: Reload systemd-modules - systemd: + ansible.builtin.systemd: name: "systemd-modules-load" state: "restarted" tags: - aws-reload-systemd-modules - name: Adjust nr_hugepages - sysctl: + ansible.builtin.sysctl: name: "vm.nr_hugepages" value: "8192" state: "present" sysctl_file: "/etc/sysctl.d/90-csit.conf" - reload: "yes" + reload: true tags: - aws-set-hugepages - name: Shutdown host with delay - command: "/sbin/shutdown -P +720" + ansible.builtin.command: "/sbin/shutdown -P +720" tags: - aws-set-self-terminate diff --git a/fdio.infra.ansible/roles/aws/tasks/ubuntu_focal.yaml b/fdio.infra.ansible/roles/aws/tasks/ubuntu_focal.yaml deleted file mode 100644 index 60302f2309..0000000000 --- a/fdio.infra.ansible/roles/aws/tasks/ubuntu_focal.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -# file: roles/aws/tasks/ubuntu_focal.yaml.yaml - -- name: Enable deb-src APT Repository - apt_repository: - repo: "deb-src http://archive.ubuntu.com/ubuntu focal main" - state: "present" - update_cache: true - tags: - - aws-enable-src-repo diff --git a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml index 4ee1545baf..c589239f61 100644 --- a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml +++ b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml @@ -1,10 +1,35 @@ --- -# file: roles/aws/tasks/ubuntu_jammy.yaml.yaml +# file: tasks/ubuntu_jammy.yaml - name: Enable deb-src APT Repository - apt_repository: - repo: "deb-src http://archive.ubuntu.com/ubuntu jammy main" + ansible.builtin.apt_repository: + repo: "deb-src {{ packages_repo[ansible_distribution|lower][ansible_machine] }} jammy main" state: "present" update_cache: true tags: - aws-enable-src-repo + +- name: Enable deb APT Repository Focal + ansible.builtin.apt_repository: + repo: "deb {{ packages_repo[ansible_distribution|lower][ansible_machine] }} focal main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Enable deb-src APT Repository Focal Src + ansible.builtin.apt_repository: + repo: "deb-src {{ packages_repo[ansible_distribution|lower][ansible_machine] }} focal main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution == 'Ubuntu' + tags: + - aws-enable-src-repo
\ No newline at end of file |