fix(packer): AWS images

Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: Ia725b895eab531f106ebf4919a0fc455447b5d9a

7 files changed, 326 insertions, 54 deletions

diff --git a/fdio.infra.ansible/roles/aws/defaults/main.yaml b/fdio.infra.ansible/roles/aws/defaults/main.yaml
index d4ea91afd4..e9701cd5ab 100644
--- a/fdio.infra.ansible/roles/aws/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/aws/defaults/main.yaml
@@ -1,2 +1,19 @@
 ---
 # file: roles/aws/defaults/main.yaml
+
+packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower] + packages_by_arch[ansible_machine] }}"
+
+packages_base:
+  - []
+
+packages_by_distro:
+  ubuntu:
+    - "linux-image-5.4.0-1009-aws"
+    - "linux-headers-5.4.0-1009-aws"
+    - "linux-tools-5.4.0-1009-aws"
+
+packages_by_arch:
+  aarch64:
+    - []
+  x86_64:
+    - []
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh
new file mode 100644
index 0000000000..02a3139b66
--- /dev/null
+++ b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh
@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+# Enable WC in VFIO-PCI driver
+# Tested on:
+#  * Amazon Linux 2 AMI (HVM), SSD Volume Type - ami-0bb3fad3c0286ebd5
+#  * Amazon Linux AMI 2018.03.0 (HVM), SSD Volume Type - ami-015232c01a82b847b
+#  * Red Hat Enterprise Linux 8 (HVM), SSD Volume Type - ami-08f4717d06813bf00
+#  * Ubuntu Server 20.04 LTS (HVM), SSD Volume Type - ami-06fd8a495a537da8b
+#  * Ubuntu Server 18.04 LTS (HVM), SSD Volume Type - ami-0823c236601fef765
+
+set -e
+
+TMP_DIR="tmp"
+
+# Kernel modules location:
+P1="/usr/lib/modules/`uname -r`/kernel/drivers/vfio"
+P2="/lib/modules/`uname -r`/kernel/drivers/vfio"
+
+# This may return an error if executed from inside the script
+set +e
+RED="$(tput setaf 1)"
+GREEN="$(tput setaf 2)"
+
+BOLD="$(tput bold)"
+NORMAL="$(tput sgr0)"
+set -e
+
+function bold {
+	echo -e "${BOLD}${@}${NORMAL}"
+}
+
+function err {
+	bold "${RED}ERROR: ${@}"
+}
+
+function green {
+	bold "${GREEN}${@}"
+}
+
+function get_kernel_version {
+	local ver=$(uname -r | cut -f 1 -d '-')
+	local ver_major=$(echo $ver | cut -f1 -d '.')
+	local ver_minor=$(echo $ver | cut -f2 -d '.')
+	local ver_subminor=$(echo $ver | cut -f3 -d '.')
+
+	printf "%d%02d%04d" "${ver_major}" "${ver_minor}" "${ver_subminor}"
+}
+
+function download_kernel_src_yum {
+	echo "Use yum to get the kernel sources"
+
+	bold "\nInstall required applications and kernel headers"
+	yum install -y gcc "kernel-$(uname -r)" "kernel-devel-$(uname -r)" \
+	    git make elfutils-libelf-devel patch yum-utils
+	green Done
+
+	# Download kernel source
+	bold "\nDownload kernel source with vfio"
+	yumdownloader --source "kernel-devel-$(uname -r)"
+	rpm2cpio kernel*.src.rpm | cpio -idmv
+	green Done
+
+	rm -f *patches.tar
+	tar xf linux-*.tar*
+	rm -f linux-*.tar* linux-*.patch
+}
+
+function download_kernel_src_apt {
+	echo "Use apt-get to get the kernel sources"
+	apt-get -q -y update
+	green Done
+
+	bold "\nInstall required applications"
+	apt-get -q -y install dpkg-dev build-essential git
+	green Done
+
+	bold "\nDownload Linux kernel source with vfio"
+	if ! apt-get -q -y source -t focal linux-image-$(uname -r); then
+		err "Cannot download Linux kernel source.\nPlease uncomment appropriate 'deb-src' line in the /etc/apt/sources.list file"
+		exit 1
+	fi
+	green Done
+
+	rm -f linux-*.dsc linux-*.gz
+}
+
+function download_kernel_src {
+	bold "[1] Downloading prerequisites..."
+	rm -rf "${TMP_DIR}"
+	mkdir -p "${TMP_DIR}"
+	cd "${TMP_DIR}"
+
+	if apt-get -v >/dev/null 2>/dev/null; then
+		download_kernel_src_apt
+	else
+		download_kernel_src_yum
+	fi
+	cd linux-*
+}
+
+function apply_wc_patch {
+	echo "Using patch for kernel version 4.10"
+	local wc_patch="${BASE_PATH}/patches/linux-4.10-vfio-wc.patch"
+
+	if ! patch --ignore-whitespace -p1 < "${wc_patch}"; then
+		err "Cannot apply patch: ${wc_patch}!"
+		exit 1
+	fi
+}
+
+function compile_vfio_driver {
+	bold "\n[2] Patch and build the vfio driver"
+	# Adjust VFIO-PCI driver
+
+	bold "Apply patch for the write combining to the vfio-pci"
+	apply_wc_patch
+	green Done
+
+	cd drivers/vfio
+	# Configure Makefile - build VFIO with support for NOIOMMU mode
+	bold "\nConfigure Makefile for standalone vfio build and noiommu mode support"
+	echo "ccflags-y := -DCONFIG_VFIO_NOIOMMU=1" >> Makefile
+	echo 'all:' >> Makefile
+	echo '	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules' >> Makefile
+	green Done
+
+	bold "\nBuild the driver"
+	if ! make; then
+		err "Compilation error."
+		exit 1
+	fi
+	green Done
+}
+
+function get_module_location {
+	for p in ${P1} ${P2}; do
+		if find "${p}" -name "vfio.*" >/dev/null 2>/dev/null; then
+			MOD_PATH="${p}"
+			break
+		fi
+	done
+
+	if [ -z "${MOD_PATH}" ]; then
+		err "Cannot find kernel modules location..."
+		exit
+	fi
+}
+
+function get_module_compression {
+	if ls "${MOD_PATH}/vfio.ko.xz" >/dev/null 2>/dev/null; then
+		XZ=".xz"
+	else
+		XZ=""
+	fi
+}
+
+function replace_module {
+	local installed=0
+
+	bold "\n[3] Install module"
+	get_module_location
+	get_module_compression
+
+	for name in "pci/vfio-pci.ko" "pci/vfio-pci-core.ko" "vfio.ko"; do
+		if test -e "${MOD_PATH}/${name}${XZ}"; then
+			if [ -n "${XZ}" ]; then
+				xz "${name}" -c > "${name}${XZ}"
+			fi
+			mv "${MOD_PATH}/${name}${XZ}" "${MOD_PATH}/${name}${XZ}_no_wc"
+			cp "${name}${XZ}" "${MOD_PATH}/${name}${XZ}"
+			bold "Installing: ${MOD_PATH}/${name}${XZ}"
+			installed=1
+		fi
+	done
+	if [ "${installed}" -eq 1 ]; then
+		green "Module installed at: ${MOD_PATH}"
+	else
+		err "Failure during vfio-pci module installation. Prehaps it's not provided as a kernel module!"
+		exit 1
+	fi
+}
+
+###############################################
+# Main script code
+###############################################
+
+if [ "$(id -u)" -ne 0 ]; then
+	err 'Please execute script as a root'
+	exit 1
+fi
+
+cd $(dirname ${0})
+BASE_PATH=$(pwd)
+
+KERNEL_VERSION=$(get_kernel_version)
+
+if [ "${KERNEL_VERSION}" -lt 4100000 ]; then
+	err "Kernel version: $(uname -r) is not supported by the script. Please upgrade kernel to at least v4.10."
+	exit 1
+fi
+
+download_kernel_src
+compile_vfio_driver
+replace_module
diff --git a/fdio.infra.ansible/roles/aws/handlers/main.yaml b/fdio.infra.ansible/roles/aws/handlers/main.yaml
index 7363dc2c34..d55db1c22f 100644
--- a/fdio.infra.ansible/roles/aws/handlers/main.yaml
+++ b/fdio.infra.ansible/roles/aws/handlers/main.yaml
@@ -1,15 +1,20 @@
 ---
 # file: roles/aws/handlers/main.yaml
 
-- name: Reboot server
-  reboot:
-    reboot_timeout: 3600
-  tags:
-    - reboot-server
-
-- name: AWS - Reload systemd-modules
+- name: Reload systemd-modules
   systemd:
     name: "systemd-modules-load"
     state: "restarted"
   tags:
     - reload-systemd-modules
+
+- name: Update GRUB
+  ansible.builtin.command: update-grub
+  tags:
+    - update-grub
+
+- name: Reboot Server
+  ansible.builtin.reboot:
+    reboot_timeout: 3600
+  tags:
+    - reboot-server
diff --git a/fdio.infra.ansible/roles/aws/tasks/main.yaml b/fdio.infra.ansible/roles/aws/tasks/main.yaml
index 321f2f64f2..e2256880c0 100644
--- a/fdio.infra.ansible/roles/aws/tasks/main.yaml
+++ b/fdio.infra.ansible/roles/aws/tasks/main.yaml
@@ -6,52 +6,24 @@
   tags:
     - aws-edit-repo
 
-- name: Get vfio-pci With WC Patcher
-  ansible.builtin.get_url:
-    url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh"
-    dest: "/opt/get-vfio-with-wc.sh"
-    mode: "744"
-  tags:
-    - aws-vfio-patch
-
-- name: Create vfio-pci Patch Directory
-  ansible.builtin.file:
-    path: "/opt/patches/"
-    state: "directory"
-  tags:
-    - aws-vfio-patch
-
-- name: Get vfio-pci WC Patch >=5.15
-  ansible.builtin.get_url:
-    url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/linux-5.15-vfio-wc.patch"
-    dest: "/opt/patches/linux-5.15-vfio-wc.patch"
-    mode: "744"
+- name: Prerequisites
+  ansible.builtin.package:
+    name: "{{ packages | flatten(levels=1) }}"
+    state: "latest"
   tags:
-    - aws-vfio-patch
+    - aws-inst-prerequisites
 
-- name: Patch WC Patch Script
+- name: Switch Kernel At Boot
   ansible.builtin.lineinfile:
-    path: "/opt/get-vfio-with-wc.sh"
-    regexp: '^	rm -f linux-'
-    line: "	rm -f linux-*.dsc linux-*.gz linux-*.xz"
+    path: "/etc/default/grub"
+    state: "present"
+    line: "GRUB_DEFAULT=\"1>2\""
+  notify:
+    - "Update GRUB"
   tags:
-    - aws-vfio-patch
+    - perf-conf-grub
 
-- name: Patch WC Patch Script II
-  ansible.builtin.replace:
-    path: "/opt/get-vfio-with-wc.sh"
-    regexp: 'linux-image-'
-    replace: 'linux-image-unsigned-'
-  tags:
-    - aws-vfio-patch
-
-#- name: Compile vfio-pci With WC Patch
-#  ansible.builtin.shell: "/bin/bash /opt/get-vfio-with-wc.sh"
-#  environment:
-#    DEBIAN_FRONTEND: "noninteractive"
-#    TERM: "vt100"
-#  tags:
-#    - aws-vfio-patch
+- meta: flush_handlers
 
 - name: Load Kernel Modules By Default
   ansible.builtin.lineinfile:
@@ -59,8 +31,10 @@
     state: "present"
     line: "{{ item }}"
   with_items:
-    - "vfio-pci"
     - "igb_uio"
+    - "vfio-pci"
+  notify:
+    - "Reboot Server"
   tags:
     - aws-load-kernel-modules
 
@@ -72,6 +46,8 @@
     create: "yes"
   with_items:
     - "options igb_uio wc_activate=1"
+  notify:
+    - "Reboot Server"
   tags:
     - aws-load-kernel-modules
 
@@ -83,9 +59,56 @@
     create: "yes"
   with_items:
     - "options vfio enable_unsafe_noiommu_mode=1"
+  notify:
+    - "Reboot Server"
   tags:
     - aws-load-kernel-modules
 
+- meta: flush_handlers
+
+#- name: Get vfio-pci With WC Patcher
+#  ansible.builtin.get_url:
+#    url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh"
+#    dest: "/opt/get-vfio-with-wc.sh"
+#    mode: 0744
+#  tags:
+#    - aws-vfio-patch
+
+- name: Create vfio-pci Patch Directory
+  ansible.builtin.file:
+    path: "/opt/patches/"
+    state: "directory"
+  tags:
+    - aws-vfio-patch
+
+- name: Get vfio-pci WC Patch
+  ansible.builtin.get_url:
+    url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/{{ item }}"
+    dest: "/opt/patches/{{ item }}"
+    mode: 0744
+  with_items:
+    - "linux-4.10-vfio-wc.patch"
+    - "linux-5.8-vfio-wc.patch"
+    - "linux-5.15-vfio-wc.patch"
+  tags:
+    - aws-vfio-patch
+
+- name: Copy vfio-pci WC Patch
+  ansible.builtin.copy:
+    src: "files/get-vfio-with-wc.sh"
+    dest: "/opt"
+    mode: 0744
+  tags:
+    - aws-vfio-patch
+
+- name: Compile vfio-pci With WC Patch
+  ansible.builtin.shell: "/bin/bash /opt/get-vfio-with-wc.sh"
+  environment:
+    DEBIAN_FRONTEND: "noninteractive"
+    TERM: "vt100"
+  tags:
+    - aws-vfio-patch
+
 - name: Reload systemd-modules
   ansible.builtin.systemd:
     name: "systemd-modules-load"
@@ -99,7 +122,7 @@
     value: "8192"
     state: "present"
     sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
+    reload: true
   tags:
     - aws-set-hugepages
 
diff --git a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml
index 28e852476a..5cc9fd3acf 100644
--- a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml
+++ b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml
@@ -8,3 +8,28 @@
     update_cache: true
   tags:
     - aws-enable-src-repo
+
+- name: Enable deb-src APT Repository Focal
+  ansible.builtin.apt_repository:
+    repo: "deb http://archive.ubuntu.com/ubuntu focal main"
+    state: "present"
+    update_cache: true
+  tags:
+    - aws-enable-src-repo
+
+- name: Enable deb-src APT Repository Focal Src
+  ansible.builtin.apt_repository:
+    repo: "deb-src http://archive.ubuntu.com/ubuntu focal main"
+    state: "present"
+    update_cache: true
+  tags:
+    - aws-enable-src-repo
+
+- name: Update Package Cache (APT)
+  ansible.builtin.apt:
+    update_cache: true
+    cache_valid_time: 3600
+  when:
+    - ansible_distribution == 'Ubuntu'
+  tags:
+    - aws-enable-src-repo
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/calibration/defaults/main.yaml b/fdio.infra.ansible/roles/calibration/defaults/main.yaml
index 6219094894..da34e97b9d 100644
--- a/fdio.infra.ansible/roles/calibration/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/calibration/defaults/main.yaml
@@ -30,15 +30,14 @@ kernel_version_by_distro_by_arch:
     focal:
       x86_64:
         - "5.4.0-65-generic"
-        - "5.3.0-1020-azure"
         - "5.4.0-1035-aws"
       aarch64:
         - "5.4.0-65-generic"
     jammy:
       x86_64:
         - "5.15.0-46-generic" # Placeholder
-        - "5.3.0-1020-azure" # Placeholder
         - "5.15.0-1000-aws" # Placeholder
+        - "5.4.0-1009-aws" # Placeholder
       aarch64:
         - "5.15.0-46-generic" # Placeholder
 
diff --git a/fdio.infra.ansible/tg.yaml b/fdio.infra.ansible/tg.yaml
index 86184e10a7..b6f1f06d7a 100644
--- a/fdio.infra.ansible/tg.yaml
+++ b/fdio.infra.ansible/tg.yaml
@@ -68,12 +68,12 @@
       tags: tg
     - role: iperf
       tags: iperf
+    - role: aws
+      tags: aws
     - role: trex
       tags: trex
     - role: ab
       tags: ab
-    - role: aws
-      tags: aws
     - role: cleanup
       tags: cleanup
     - role: calibration