diff options
Diffstat (limited to 'fdio.infra.ansible')
| -rw-r--r-- | fdio.infra.ansible/roles/aws/defaults/main.yaml | 17 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh | 203 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/handlers/main.yaml | 19 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/tasks/main.yaml | 109 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml | 25 | ||||
| -rw-r--r-- | fdio.infra.ansible/roles/calibration/defaults/main.yaml | 3 | ||||
| -rw-r--r-- | fdio.infra.ansible/tg.yaml | 4 |
7 files changed, 326 insertions, 54 deletions
diff --git a/fdio.infra.ansible/roles/aws/defaults/main.yaml b/fdio.infra.ansible/roles/aws/defaults/main.yaml index d4ea91afd4..e9701cd5ab 100644 --- a/fdio.infra.ansible/roles/aws/defaults/main.yaml +++ b/fdio.infra.ansible/roles/aws/defaults/main.yaml @@ -1,2 +1,19 @@ --- # file: roles/aws/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "linux-image-5.4.0-1009-aws" + - "linux-headers-5.4.0-1009-aws" + - "linux-tools-5.4.0-1009-aws" + +packages_by_arch: + aarch64: + - [] + x86_64: + - []
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh new file mode 100644 index 0000000000..02a3139b66 --- /dev/null +++ b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# Enable WC in VFIO-PCI driver +# Tested on: +# * Amazon Linux 2 AMI (HVM), SSD Volume Type - ami-0bb3fad3c0286ebd5 +# * Amazon Linux AMI 2018.03.0 (HVM), SSD Volume Type - ami-015232c01a82b847b +# * Red Hat Enterprise Linux 8 (HVM), SSD Volume Type - ami-08f4717d06813bf00 +# * Ubuntu Server 20.04 LTS (HVM), SSD Volume Type - ami-06fd8a495a537da8b +# * Ubuntu Server 18.04 LTS (HVM), SSD Volume Type - ami-0823c236601fef765 + +set -e + +TMP_DIR="tmp" + +# Kernel modules location: +P1="/usr/lib/modules/`uname -r`/kernel/drivers/vfio" +P2="/lib/modules/`uname -r`/kernel/drivers/vfio" + +# This may return an error if executed from inside the script +set +e +RED="$(tput setaf 1)" +GREEN="$(tput setaf 2)" + +BOLD="$(tput bold)" +NORMAL="$(tput sgr0)" +set -e + +function bold { + echo -e "${BOLD}${@}${NORMAL}" +} + +function err { + bold "${RED}ERROR: ${@}" +} + +function green { + bold "${GREEN}${@}" +} + +function get_kernel_version { + local ver=$(uname -r | cut -f 1 -d '-') + local ver_major=$(echo $ver | cut -f1 -d '.') + local ver_minor=$(echo $ver | cut -f2 -d '.') + local ver_subminor=$(echo $ver | cut -f3 -d '.') + + printf "%d%02d%04d" "${ver_major}" "${ver_minor}" "${ver_subminor}" +} + +function download_kernel_src_yum { + echo "Use yum to get the kernel sources" + + bold "\nInstall required applications and kernel headers" + yum install -y gcc "kernel-$(uname -r)" "kernel-devel-$(uname -r)" \ + git make elfutils-libelf-devel patch yum-utils + green Done + + # Download kernel source + bold "\nDownload kernel source with vfio" + yumdownloader --source "kernel-devel-$(uname -r)" + rpm2cpio kernel*.src.rpm | cpio -idmv + green Done + + rm -f *patches.tar + tar xf linux-*.tar* + rm -f linux-*.tar* linux-*.patch +} + +function download_kernel_src_apt { + echo "Use apt-get to get the kernel sources" + apt-get -q -y update + green Done + + bold "\nInstall required applications" + apt-get -q -y install dpkg-dev build-essential git + green Done + + bold "\nDownload Linux kernel source with vfio" + if ! apt-get -q -y source -t focal linux-image-$(uname -r); then + err "Cannot download Linux kernel source.\nPlease uncomment appropriate 'deb-src' line in the /etc/apt/sources.list file" + exit 1 + fi + green Done + + rm -f linux-*.dsc linux-*.gz +} + +function download_kernel_src { + bold "[1] Downloading prerequisites..." + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + cd "${TMP_DIR}" + + if apt-get -v >/dev/null 2>/dev/null; then + download_kernel_src_apt + else + download_kernel_src_yum + fi + cd linux-* +} + +function apply_wc_patch { + echo "Using patch for kernel version 4.10" + local wc_patch="${BASE_PATH}/patches/linux-4.10-vfio-wc.patch" + + if ! patch --ignore-whitespace -p1 < "${wc_patch}"; then + err "Cannot apply patch: ${wc_patch}!" + exit 1 + fi +} + +function compile_vfio_driver { + bold "\n[2] Patch and build the vfio driver" + # Adjust VFIO-PCI driver + + bold "Apply patch for the write combining to the vfio-pci" + apply_wc_patch + green Done + + cd drivers/vfio + # Configure Makefile - build VFIO with support for NOIOMMU mode + bold "\nConfigure Makefile for standalone vfio build and noiommu mode support" + echo "ccflags-y := -DCONFIG_VFIO_NOIOMMU=1" >> Makefile + echo 'all:' >> Makefile + echo ' make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules' >> Makefile + green Done + + bold "\nBuild the driver" + if ! make; then + err "Compilation error." + exit 1 + fi + green Done +} + +function get_module_location { + for p in ${P1} ${P2}; do + if find "${p}" -name "vfio.*" >/dev/null 2>/dev/null; then + MOD_PATH="${p}" + break + fi + done + + if [ -z "${MOD_PATH}" ]; then + err "Cannot find kernel modules location..." + exit + fi +} + +function get_module_compression { + if ls "${MOD_PATH}/vfio.ko.xz" >/dev/null 2>/dev/null; then + XZ=".xz" + else + XZ="" + fi +} + +function replace_module { + local installed=0 + + bold "\n[3] Install module" + get_module_location + get_module_compression + + for name in "pci/vfio-pci.ko" "pci/vfio-pci-core.ko" "vfio.ko"; do + if test -e "${MOD_PATH}/${name}${XZ}"; then + if [ -n "${XZ}" ]; then + xz "${name}" -c > "${name}${XZ}" + fi + mv "${MOD_PATH}/${name}${XZ}" "${MOD_PATH}/${name}${XZ}_no_wc" + cp "${name}${XZ}" "${MOD_PATH}/${name}${XZ}" + bold "Installing: ${MOD_PATH}/${name}${XZ}" + installed=1 + fi + done + if [ "${installed}" -eq 1 ]; then + green "Module installed at: ${MOD_PATH}" + else + err "Failure during vfio-pci module installation. Prehaps it's not provided as a kernel module!" + exit 1 + fi +} + +############################################### +# Main script code +############################################### + +if [ "$(id -u)" -ne 0 ]; then + err 'Please execute script as a root' + exit 1 +fi + +cd $(dirname ${0}) +BASE_PATH=$(pwd) + +KERNEL_VERSION=$(get_kernel_version) + +if [ "${KERNEL_VERSION}" -lt 4100000 ]; then + err "Kernel version: $(uname -r) is not supported by the script. Please upgrade kernel to at least v4.10." + exit 1 +fi + +download_kernel_src +compile_vfio_driver +replace_module diff --git a/fdio.infra.ansible/roles/aws/handlers/main.yaml b/fdio.infra.ansible/roles/aws/handlers/main.yaml index 7363dc2c34..d55db1c22f 100644 --- a/fdio.infra.ansible/roles/aws/handlers/main.yaml +++ b/fdio.infra.ansible/roles/aws/handlers/main.yaml @@ -1,15 +1,20 @@ --- # file: roles/aws/handlers/main.yaml -- name: Reboot server - reboot: - reboot_timeout: 3600 - tags: - - reboot-server - -- name: AWS - Reload systemd-modules +- name: Reload systemd-modules systemd: name: "systemd-modules-load" state: "restarted" tags: - reload-systemd-modules + +- name: Update GRUB + ansible.builtin.command: update-grub + tags: + - update-grub + +- name: Reboot Server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/aws/tasks/main.yaml b/fdio.infra.ansible/roles/aws/tasks/main.yaml index 321f2f64f2..e2256880c0 100644 --- a/fdio.infra.ansible/roles/aws/tasks/main.yaml +++ b/fdio.infra.ansible/roles/aws/tasks/main.yaml @@ -6,52 +6,24 @@ tags: - aws-edit-repo -- name: Get vfio-pci With WC Patcher - ansible.builtin.get_url: - url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh" - dest: "/opt/get-vfio-with-wc.sh" - mode: "744" - tags: - - aws-vfio-patch - -- name: Create vfio-pci Patch Directory - ansible.builtin.file: - path: "/opt/patches/" - state: "directory" - tags: - - aws-vfio-patch - -- name: Get vfio-pci WC Patch >=5.15 - ansible.builtin.get_url: - url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/linux-5.15-vfio-wc.patch" - dest: "/opt/patches/linux-5.15-vfio-wc.patch" - mode: "744" +- name: Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: "latest" tags: - - aws-vfio-patch + - aws-inst-prerequisites -- name: Patch WC Patch Script +- name: Switch Kernel At Boot ansible.builtin.lineinfile: - path: "/opt/get-vfio-with-wc.sh" - regexp: '^ rm -f linux-' - line: " rm -f linux-*.dsc linux-*.gz linux-*.xz" + path: "/etc/default/grub" + state: "present" + line: "GRUB_DEFAULT=\"1>2\"" + notify: + - "Update GRUB" tags: - - aws-vfio-patch + - perf-conf-grub -- name: Patch WC Patch Script II - ansible.builtin.replace: - path: "/opt/get-vfio-with-wc.sh" - regexp: 'linux-image-' - replace: 'linux-image-unsigned-' - tags: - - aws-vfio-patch - -#- name: Compile vfio-pci With WC Patch -# ansible.builtin.shell: "/bin/bash /opt/get-vfio-with-wc.sh" -# environment: -# DEBIAN_FRONTEND: "noninteractive" -# TERM: "vt100" -# tags: -# - aws-vfio-patch +- meta: flush_handlers - name: Load Kernel Modules By Default ansible.builtin.lineinfile: @@ -59,8 +31,10 @@ state: "present" line: "{{ item }}" with_items: - - "vfio-pci" - "igb_uio" + - "vfio-pci" + notify: + - "Reboot Server" tags: - aws-load-kernel-modules @@ -72,6 +46,8 @@ create: "yes" with_items: - "options igb_uio wc_activate=1" + notify: + - "Reboot Server" tags: - aws-load-kernel-modules @@ -83,9 +59,56 @@ create: "yes" with_items: - "options vfio enable_unsafe_noiommu_mode=1" + notify: + - "Reboot Server" tags: - aws-load-kernel-modules +- meta: flush_handlers + +#- name: Get vfio-pci With WC Patcher +# ansible.builtin.get_url: +# url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh" +# dest: "/opt/get-vfio-with-wc.sh" +# mode: 0744 +# tags: +# - aws-vfio-patch + +- name: Create vfio-pci Patch Directory + ansible.builtin.file: + path: "/opt/patches/" + state: "directory" + tags: + - aws-vfio-patch + +- name: Get vfio-pci WC Patch + ansible.builtin.get_url: + url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/{{ item }}" + dest: "/opt/patches/{{ item }}" + mode: 0744 + with_items: + - "linux-4.10-vfio-wc.patch" + - "linux-5.8-vfio-wc.patch" + - "linux-5.15-vfio-wc.patch" + tags: + - aws-vfio-patch + +- name: Copy vfio-pci WC Patch + ansible.builtin.copy: + src: "files/get-vfio-with-wc.sh" + dest: "/opt" + mode: 0744 + tags: + - aws-vfio-patch + +- name: Compile vfio-pci With WC Patch + ansible.builtin.shell: "/bin/bash /opt/get-vfio-with-wc.sh" + environment: + DEBIAN_FRONTEND: "noninteractive" + TERM: "vt100" + tags: + - aws-vfio-patch + - name: Reload systemd-modules ansible.builtin.systemd: name: "systemd-modules-load" @@ -99,7 +122,7 @@ value: "8192" state: "present" sysctl_file: "/etc/sysctl.d/90-csit.conf" - reload: "yes" + reload: true tags: - aws-set-hugepages diff --git a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml index 28e852476a..5cc9fd3acf 100644 --- a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml +++ b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml @@ -8,3 +8,28 @@ update_cache: true tags: - aws-enable-src-repo + +- name: Enable deb-src APT Repository Focal + ansible.builtin.apt_repository: + repo: "deb http://archive.ubuntu.com/ubuntu focal main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Enable deb-src APT Repository Focal Src + ansible.builtin.apt_repository: + repo: "deb-src http://archive.ubuntu.com/ubuntu focal main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution == 'Ubuntu' + tags: + - aws-enable-src-repo
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/calibration/defaults/main.yaml b/fdio.infra.ansible/roles/calibration/defaults/main.yaml index 6219094894..da34e97b9d 100644 --- a/fdio.infra.ansible/roles/calibration/defaults/main.yaml +++ b/fdio.infra.ansible/roles/calibration/defaults/main.yaml @@ -30,15 +30,14 @@ kernel_version_by_distro_by_arch: focal: x86_64: - "5.4.0-65-generic" - - "5.3.0-1020-azure" - "5.4.0-1035-aws" aarch64: - "5.4.0-65-generic" jammy: x86_64: - "5.15.0-46-generic" # Placeholder - - "5.3.0-1020-azure" # Placeholder - "5.15.0-1000-aws" # Placeholder + - "5.4.0-1009-aws" # Placeholder aarch64: - "5.15.0-46-generic" # Placeholder diff --git a/fdio.infra.ansible/tg.yaml b/fdio.infra.ansible/tg.yaml index 86184e10a7..b6f1f06d7a 100644 --- a/fdio.infra.ansible/tg.yaml +++ b/fdio.infra.ansible/tg.yaml @@ -68,12 +68,12 @@ tags: tg - role: iperf tags: iperf + - role: aws + tags: aws - role: trex tags: trex - role: ab tags: ab - - role: aws - tags: aws - role: cleanup tags: cleanup - role: calibration |