diff options
Diffstat (limited to 'fdio.infra.ansible/roles')
164 files changed, 7454 insertions, 0 deletions
diff --git a/fdio.infra.ansible/roles/ab/defaults/main.yaml b/fdio.infra.ansible/roles/ab/defaults/main.yaml new file mode 100644 index 0000000000..adabf1464c --- /dev/null +++ b/fdio.infra.ansible/roles/ab/defaults/main.yaml @@ -0,0 +1,18 @@ +--- +# file: roles/ab/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + jammy: + - "apache2-utils" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] diff --git a/fdio.infra.ansible/roles/ab/tasks/main.yaml b/fdio.infra.ansible/roles/ab/tasks/main.yaml new file mode 100644 index 0000000000..2a70fd1d1d --- /dev/null +++ b/fdio.infra.ansible/roles/ab/tasks/main.yaml @@ -0,0 +1,18 @@ +--- +# file: roles/ab/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - ab-inst-prerequisites + +- name: Inst - Apache ab tools + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: present + tags: + - ab-inst diff --git a/fdio.infra.ansible/roles/aws/defaults/main.yaml b/fdio.infra.ansible/roles/aws/defaults/main.yaml new file mode 100644 index 0000000000..5b6978da51 --- /dev/null +++ b/fdio.infra.ansible/roles/aws/defaults/main.yaml @@ -0,0 +1,26 @@ +--- +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower] + packages_by_arch[ansible_machine] }}" + +packages_repo: + ubuntu: + aarch64: + "http://ports.ubuntu.com/" + x86_64: + "http://archive.ubuntu.com/ubuntu" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "linux-image-5.4.0-1009-aws" + - "linux-headers-5.4.0-1009-aws" + - "linux-tools-5.4.0-1009-aws" + +packages_by_arch: + aarch64: + - [] + x86_64: + - []
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh new file mode 100644 index 0000000000..02a3139b66 --- /dev/null +++ b/fdio.infra.ansible/roles/aws/files/get-vfio-with-wc.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# Enable WC in VFIO-PCI driver +# Tested on: +# * Amazon Linux 2 AMI (HVM), SSD Volume Type - ami-0bb3fad3c0286ebd5 +# * Amazon Linux AMI 2018.03.0 (HVM), SSD Volume Type - ami-015232c01a82b847b +# * Red Hat Enterprise Linux 8 (HVM), SSD Volume Type - ami-08f4717d06813bf00 +# * Ubuntu Server 20.04 LTS (HVM), SSD Volume Type - ami-06fd8a495a537da8b +# * Ubuntu Server 18.04 LTS (HVM), SSD Volume Type - ami-0823c236601fef765 + +set -e + +TMP_DIR="tmp" + +# Kernel modules location: +P1="/usr/lib/modules/`uname -r`/kernel/drivers/vfio" +P2="/lib/modules/`uname -r`/kernel/drivers/vfio" + +# This may return an error if executed from inside the script +set +e +RED="$(tput setaf 1)" +GREEN="$(tput setaf 2)" + +BOLD="$(tput bold)" +NORMAL="$(tput sgr0)" +set -e + +function bold { + echo -e "${BOLD}${@}${NORMAL}" +} + +function err { + bold "${RED}ERROR: ${@}" +} + +function green { + bold "${GREEN}${@}" +} + +function get_kernel_version { + local ver=$(uname -r | cut -f 1 -d '-') + local ver_major=$(echo $ver | cut -f1 -d '.') + local ver_minor=$(echo $ver | cut -f2 -d '.') + local ver_subminor=$(echo $ver | cut -f3 -d '.') + + printf "%d%02d%04d" "${ver_major}" "${ver_minor}" "${ver_subminor}" +} + +function download_kernel_src_yum { + echo "Use yum to get the kernel sources" + + bold "\nInstall required applications and kernel headers" + yum install -y gcc "kernel-$(uname -r)" "kernel-devel-$(uname -r)" \ + git make elfutils-libelf-devel patch yum-utils + green Done + + # Download kernel source + bold "\nDownload kernel source with vfio" + yumdownloader --source "kernel-devel-$(uname -r)" + rpm2cpio kernel*.src.rpm | cpio -idmv + green Done + + rm -f *patches.tar + tar xf linux-*.tar* + rm -f linux-*.tar* linux-*.patch +} + +function download_kernel_src_apt { + echo "Use apt-get to get the kernel sources" + apt-get -q -y update + green Done + + bold "\nInstall required applications" + apt-get -q -y install dpkg-dev build-essential git + green Done + + bold "\nDownload Linux kernel source with vfio" + if ! apt-get -q -y source -t focal linux-image-$(uname -r); then + err "Cannot download Linux kernel source.\nPlease uncomment appropriate 'deb-src' line in the /etc/apt/sources.list file" + exit 1 + fi + green Done + + rm -f linux-*.dsc linux-*.gz +} + +function download_kernel_src { + bold "[1] Downloading prerequisites..." + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + cd "${TMP_DIR}" + + if apt-get -v >/dev/null 2>/dev/null; then + download_kernel_src_apt + else + download_kernel_src_yum + fi + cd linux-* +} + +function apply_wc_patch { + echo "Using patch for kernel version 4.10" + local wc_patch="${BASE_PATH}/patches/linux-4.10-vfio-wc.patch" + + if ! patch --ignore-whitespace -p1 < "${wc_patch}"; then + err "Cannot apply patch: ${wc_patch}!" + exit 1 + fi +} + +function compile_vfio_driver { + bold "\n[2] Patch and build the vfio driver" + # Adjust VFIO-PCI driver + + bold "Apply patch for the write combining to the vfio-pci" + apply_wc_patch + green Done + + cd drivers/vfio + # Configure Makefile - build VFIO with support for NOIOMMU mode + bold "\nConfigure Makefile for standalone vfio build and noiommu mode support" + echo "ccflags-y := -DCONFIG_VFIO_NOIOMMU=1" >> Makefile + echo 'all:' >> Makefile + echo ' make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules' >> Makefile + green Done + + bold "\nBuild the driver" + if ! make; then + err "Compilation error." + exit 1 + fi + green Done +} + +function get_module_location { + for p in ${P1} ${P2}; do + if find "${p}" -name "vfio.*" >/dev/null 2>/dev/null; then + MOD_PATH="${p}" + break + fi + done + + if [ -z "${MOD_PATH}" ]; then + err "Cannot find kernel modules location..." + exit + fi +} + +function get_module_compression { + if ls "${MOD_PATH}/vfio.ko.xz" >/dev/null 2>/dev/null; then + XZ=".xz" + else + XZ="" + fi +} + +function replace_module { + local installed=0 + + bold "\n[3] Install module" + get_module_location + get_module_compression + + for name in "pci/vfio-pci.ko" "pci/vfio-pci-core.ko" "vfio.ko"; do + if test -e "${MOD_PATH}/${name}${XZ}"; then + if [ -n "${XZ}" ]; then + xz "${name}" -c > "${name}${XZ}" + fi + mv "${MOD_PATH}/${name}${XZ}" "${MOD_PATH}/${name}${XZ}_no_wc" + cp "${name}${XZ}" "${MOD_PATH}/${name}${XZ}" + bold "Installing: ${MOD_PATH}/${name}${XZ}" + installed=1 + fi + done + if [ "${installed}" -eq 1 ]; then + green "Module installed at: ${MOD_PATH}" + else + err "Failure during vfio-pci module installation. Prehaps it's not provided as a kernel module!" + exit 1 + fi +} + +############################################### +# Main script code +############################################### + +if [ "$(id -u)" -ne 0 ]; then + err 'Please execute script as a root' + exit 1 +fi + +cd $(dirname ${0}) +BASE_PATH=$(pwd) + +KERNEL_VERSION=$(get_kernel_version) + +if [ "${KERNEL_VERSION}" -lt 4100000 ]; then + err "Kernel version: $(uname -r) is not supported by the script. Please upgrade kernel to at least v4.10." + exit 1 +fi + +download_kernel_src +compile_vfio_driver +replace_module diff --git a/fdio.infra.ansible/roles/aws/handlers/main.yaml b/fdio.infra.ansible/roles/aws/handlers/main.yaml new file mode 100644 index 0000000000..d55db1c22f --- /dev/null +++ b/fdio.infra.ansible/roles/aws/handlers/main.yaml @@ -0,0 +1,20 @@ +--- +# file: roles/aws/handlers/main.yaml + +- name: Reload systemd-modules + systemd: + name: "systemd-modules-load" + state: "restarted" + tags: + - reload-systemd-modules + +- name: Update GRUB + ansible.builtin.command: update-grub + tags: + - update-grub + +- name: Reboot Server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/aws/tasks/main.yaml b/fdio.infra.ansible/roles/aws/tasks/main.yaml new file mode 100644 index 0000000000..b5132c1909 --- /dev/null +++ b/fdio.infra.ansible/roles/aws/tasks/main.yaml @@ -0,0 +1,124 @@ +--- +# file: tasks/main.yaml + +- name: Edit repositories + include_tasks: "{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml" + tags: + - aws-edit-repo + +- name: Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: "latest" + tags: + - aws-inst-prerequisites + +- name: Switch Kernel At Boot + ansible.builtin.lineinfile: + path: "/etc/default/grub" + state: "present" + line: "GRUB_DEFAULT=\"1>2\"" + notify: + - "Update GRUB" + tags: + - perf-conf-grub + +- meta: flush_handlers + +- name: Load Kernel Modules By Default + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "igb_uio" + - "vfio-pci" + tags: + - aws-load-kernel-modules + +- name: Add Kernel Modules Options (igb_uio) + ansible.builtin.lineinfile: + path: "/etc/modprobe.d/igb_uio.conf" + state: "present" + line: "{{ item }}" + create: "yes" + with_items: + - "options igb_uio wc_activate=1" + tags: + - aws-load-kernel-modules + +- name: Add Kernel Modules Options (vfio-pci) + ansible.builtin.lineinfile: + path: "/etc/modprobe.d/vfio-noiommu.conf" + state: "present" + line: "{{ item }}" + create: "yes" + with_items: + - "options vfio enable_unsafe_noiommu_mode=1" + tags: + - aws-load-kernel-modules + +#- name: Get vfio-pci With WC Patcher +# ansible.builtin.get_url: +# url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/get-vfio-with-wc.sh" +# dest: "/opt/get-vfio-with-wc.sh" +# mode: 0744 +# tags: +# - aws-vfio-patch + +- name: Create vfio-pci Patch Directory + ansible.builtin.file: + path: "/opt/patches/" + state: "directory" + tags: + - aws-vfio-patch + +- name: Get vfio-pci WC Patch + ansible.builtin.get_url: + url: "https://github.com/amzn/amzn-drivers/raw/master/userspace/dpdk/enav2-vfio-patch/patches/{{ item }}" + dest: "/opt/patches/{{ item }}" + mode: 0744 + with_items: + - "linux-4.10-vfio-wc.patch" + - "linux-5.8-vfio-wc.patch" + - "linux-5.15-vfio-wc.patch" + tags: + - aws-vfio-patch + +- name: Copy vfio-pci WC Patch + ansible.builtin.copy: + src: "files/get-vfio-with-wc.sh" + dest: "/opt" + mode: 0744 + tags: + - aws-vfio-patch + +- name: Compile vfio-pci With WC Patch + ansible.builtin.shell: "/bin/bash /opt/get-vfio-with-wc.sh" + environment: + DEBIAN_FRONTEND: "noninteractive" + TERM: "vt100" + tags: + - aws-vfio-patch + +- name: Reload systemd-modules + ansible.builtin.systemd: + name: "systemd-modules-load" + state: "restarted" + tags: + - aws-reload-systemd-modules + +- name: Adjust nr_hugepages + ansible.builtin.sysctl: + name: "vm.nr_hugepages" + value: "8192" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: true + tags: + - aws-set-hugepages + +- name: Shutdown host with delay + ansible.builtin.command: "/sbin/shutdown -P +720" + tags: + - aws-set-self-terminate diff --git a/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml new file mode 100644 index 0000000000..c589239f61 --- /dev/null +++ b/fdio.infra.ansible/roles/aws/tasks/ubuntu_jammy.yaml @@ -0,0 +1,35 @@ +--- +# file: tasks/ubuntu_jammy.yaml + +- name: Enable deb-src APT Repository + ansible.builtin.apt_repository: + repo: "deb-src {{ packages_repo[ansible_distribution|lower][ansible_machine] }} jammy main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Enable deb APT Repository Focal + ansible.builtin.apt_repository: + repo: "deb {{ packages_repo[ansible_distribution|lower][ansible_machine] }} focal main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Enable deb-src APT Repository Focal Src + ansible.builtin.apt_repository: + repo: "deb-src {{ packages_repo[ansible_distribution|lower][ansible_machine] }} focal main" + state: "present" + update_cache: true + tags: + - aws-enable-src-repo + +- name: Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution == 'Ubuntu' + tags: + - aws-enable-src-repo
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/azure/defaults/main.yaml b/fdio.infra.ansible/roles/azure/defaults/main.yaml new file mode 100644 index 0000000000..68f6148413 --- /dev/null +++ b/fdio.infra.ansible/roles/azure/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +# file: roles/azure/defaults/main.yaml diff --git a/fdio.infra.ansible/roles/azure/files/10-dtap.link b/fdio.infra.ansible/roles/azure/files/10-dtap.link new file mode 100644 index 0000000000..a8e0aa10f3 --- /dev/null +++ b/fdio.infra.ansible/roles/azure/files/10-dtap.link @@ -0,0 +1,4 @@ +[Match] +OriginalName=dtap* +[Link] +NamePolicy=kernel diff --git a/fdio.infra.ansible/roles/azure/handlers/main.yaml b/fdio.infra.ansible/roles/azure/handlers/main.yaml new file mode 100644 index 0000000000..f0d46062d9 --- /dev/null +++ b/fdio.infra.ansible/roles/azure/handlers/main.yaml @@ -0,0 +1,15 @@ +--- +# file: roles/azure/handlers/main.yaml + +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: + - reboot-server + +- name: Azure - Reload systemd-modules + systemd: + name: "systemd-modules-load" + state: "restarted" + tags: + - reload-systemd-modules diff --git a/fdio.infra.ansible/roles/azure/tasks/main.yaml b/fdio.infra.ansible/roles/azure/tasks/main.yaml new file mode 100644 index 0000000000..c8d72475d8 --- /dev/null +++ b/fdio.infra.ansible/roles/azure/tasks/main.yaml @@ -0,0 +1,38 @@ +--- +# file: roles/azure/tasks/main.yaml + +- name: Azure - Load Kernel Modules By Default + lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "vfio-pci" + - "ib_uverbs" + - "mlx4_ib" + - "mlx5_ib" + notify: "Azure - Reload systemd-modules" + tags: + - load-kernel-modules + +- name: Azure - Performance Tuning - Adjust nr_hugepages + sysctl: + name: "vm.nr_hugepages" + value: "8192" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: Azure - prevent interface renaming + copy: + src: "files/10-dtap.link" + dest: "/etc/systemd/network/" + owner: "root" + group: "root" + mode: "0644" + notify: + - "Reboot server" + tags: + - prevent-interface-renaming diff --git a/fdio.infra.ansible/roles/baremetal/handlers/cimc.yaml b/fdio.infra.ansible/roles/baremetal/handlers/cimc.yaml new file mode 100644 index 0000000000..3d244d8d19 --- /dev/null +++ b/fdio.infra.ansible/roles/baremetal/handlers/cimc.yaml @@ -0,0 +1,74 @@ +--- +# file: roles/baremeatal/handlers/cimc.yaml + +- name: Boot from network + imc_rest: + hostname: "{{ inventory_cimc_hostname }}" + username: "{{ inventory_cimc_username }}" + password: "{{ inventory_cimc_password }}" + validate_certs: false + content: | + <!-- Configure PXE boot --> + <configConfMo><inConfig> + <lsbootLan dn="sys/rack-unit-1/boot-policy/lan-read-only" access="read-only" order="1" prot="pxe" type="lan"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: + - boot-network + +- name: Boot from storage + imc_rest: + hostname: "{{ inventory_cimc_hostname }}" + username: "{{ inventory_cimc_username }}" + password: "{{ inventory_cimc_password }}" + validate_certs: false + content: | + <configConfMo><inConfig> + <lsbootStorage dn="sys/rack-unit-1/boot-policy/storage-read-write" access="read-write" order="1" type="storage"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: + - boot-storage + +- name: Power up server + imc_rest: + hostname: "{{ inventory_cimc_hostname }}" + username: "{{ inventory_cimc_username }}" + password: "{{ inventory_cimc_password }}" + validate_certs: false + content: | + <configConfMo><inConfig> + <computeRackUnit dn="sys/rack-unit-1" adminPower="up"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: + - power-up + +- name: Power down server + imc_rest: + hostname: "{{ inventory_cimc_hostname }}" + username: "{{ inventory_cimc_username }}" + password: "{{ inventory_cimc_password }}" + validate_certs: false + content: | + <configConfMo><inConfig> + <computeRackUnit dn="sys/rack-unit-1" adminPower="down"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: + - power-down + +- name: Power cycle server + imc_rest: + hostname: "{{ inventory_cimc_hostname }}" + username: "{{ inventory_cimc_username }}" + password: "{{ inventory_cimc_password }}" + validate_certs: false + content: | + <!-- Power cycle server --> + <configConfMo><inConfig> + <computeRackUnit dn="sys/rack-unit-1" adminPower="cycle-immediate"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: + - power-cycle diff --git a/fdio.infra.ansible/roles/baremetal/handlers/ipmi.yaml b/fdio.infra.ansible/roles/baremetal/handlers/ipmi.yaml new file mode 100644 index 0000000000..b3cc3d0a82 --- /dev/null +++ b/fdio.infra.ansible/roles/baremetal/handlers/ipmi.yaml @@ -0,0 +1,52 @@ +--- +# file: roles/baremetal/handlers/ipmi.yaml + +- name: Boot from network + ipmi_boot: + name: "{{ inventory_ipmi_hostname }}" + user: "{{ inventory_ipmi_username }}" + password: "{{ inventory_ipmi_password }}" + bootdev: network + delegate_to: localhost + tags: + - boot-network + +- name: Boot from storage + ipmi_boot: + name: "{{ inventory_ipmi_hostname }}" + user: "{{ inventory_ipmi_username }}" + password: "{{ inventory_ipmi_password }}" + bootdev: hd + delegate_to: localhost + tags: + - boot-storage + +- name: Power up server + ipmi_power: + name: "{{ inventory_ipmi_hostname }}" + user: "{{ inventory_ipmi_username }}" + password: "{{ inventory_ipmi_password }}" + state: true + delegate_to: localhost + tags: + - power-up + +- name: Power down server + ipmi_power: + name: "{{ inventory_ipmi_hostname }}" + user: "{{ inventory_ipmi_username }}" + password: "{{ inventory_ipmi_password }}" + state: false + delegate_to: localhost + tags: + - power-down + +- name: Power cycle server + ipmi_power: + name: "{{ inventory_ipmi_hostname }}" + user: "{{ inventory_ipmi_username }}" + password: "{{ inventory_ipmi_password }}" + state: boot + delegate_to: localhost + tags: + - power-cycle diff --git a/fdio.infra.ansible/roles/baremetal/handlers/main.yaml b/fdio.infra.ansible/roles/baremetal/handlers/main.yaml new file mode 100644 index 0000000000..6e8734eaa9 --- /dev/null +++ b/fdio.infra.ansible/roles/baremetal/handlers/main.yaml @@ -0,0 +1,30 @@ +--- +# file: roles/baremetal/handlers/main.yaml + +- name: IPMI specific + import_tasks: ipmi.yaml + when: inventory_ipmi_hostname is defined + tags: + - ipmi-handlers + +- name: CIMC specific + import_tasks: cimc.yaml + when: inventory_cimc_hostname is defined + tags: + - cimc-handlers + +- name: Reboot server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server + +- name: Wait for server to restart + ansible.builtin.wait_for: + host: "{{ inventory_hostname }}" + search_regex: OpenSSH + port: 22 + delay: 60 + timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/cadvisor/defaults/main.yaml b/fdio.infra.ansible/roles/cadvisor/defaults/main.yaml new file mode 100644 index 0000000000..5dba8c9112 --- /dev/null +++ b/fdio.infra.ansible/roles/cadvisor/defaults/main.yaml @@ -0,0 +1,24 @@ +--- +# file: roles/cadvisor/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "python3-docker" + - "python3-dockerpty" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +image: "{{ image_by_arch[ansible_machine] }}" + +image_by_arch: + aarch64: "zcube/cadvisor:v0.37.0" + x86_64: "gcr.io/cadvisor/cadvisor:v0.38.7" diff --git a/fdio.infra.ansible/roles/cadvisor/tasks/main.yaml b/fdio.infra.ansible/roles/cadvisor/tasks/main.yaml new file mode 100644 index 0000000000..a8c3f70124 --- /dev/null +++ b/fdio.infra.ansible/roles/cadvisor/tasks/main.yaml @@ -0,0 +1,39 @@ +--- +# file: roles/cadvisor/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - cadvisor-inst-prerequisites + +- name: Inst - Prerequisites + package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - cadvisor-inst-prerequisites + +- name: Inst - Start a container + docker_container: + name: "cAdvisor" + image: "{{ image }}" + state: "started" + restart_policy: "unless-stopped" + detach: true + devices: + - "/dev/kmsg" + ports: + - "8080:8080" + privileged: true + volumes: + - "/:/rootfs:ro" + - "/var/run:/var/run:ro" + - "/sys:/sys:ro" + - "/var/lib/docker/:/var/lib/docker:ro" + - "/dev/disk/:/dev/disk:ro" + tags: + - cadvisor-run-container diff --git a/fdio.infra.ansible/roles/calibration/defaults/main.yaml b/fdio.infra.ansible/roles/calibration/defaults/main.yaml new file mode 100644 index 0000000000..5dc3330e08 --- /dev/null +++ b/fdio.infra.ansible/roles/calibration/defaults/main.yaml @@ -0,0 +1,37 @@ +--- +# file: roles/calibration/defaults/main.yaml + +# Packages to install. +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + jammy: + - "build-essential" + - "dmidecode" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +# Kernel version to check. +kernel_version: "{{ kernel_version_by_distro_by_arch[ansible_distribution | lower][ansible_distribution_release][ansible_machine] }}" + +kernel_version_by_distro_by_arch: + ubuntu: + jammy: + x86_64: + - "5.15.0-46-generic" # Placeholder + - "5.15.0-1000-aws" # Placeholder + - "5.4.0-1009-aws" # Placeholder + aarch64: + - "5.15.0-46-generic" # Placeholder + +pma_directory: "/tmp/pma_tools" +jitter_core: 7 +jitter_iterations: 20 diff --git a/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml b/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml new file mode 100644 index 0000000000..ca4e75d268 --- /dev/null +++ b/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml @@ -0,0 +1,2 @@ +--- +# file: roles/calibration/tasks/aarch64.yaml diff --git a/fdio.infra.ansible/roles/calibration/tasks/main.yaml b/fdio.infra.ansible/roles/calibration/tasks/main.yaml new file mode 100644 index 0000000000..5807d7e2a4 --- /dev/null +++ b/fdio.infra.ansible/roles/calibration/tasks/main.yaml @@ -0,0 +1,89 @@ +--- +# file: roles/calibration/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - calibration-inst-prerequisites + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - calibration-inst-prerequisites + +- name: Check CPU Power States + ansible.builtin.shell: "lscpu" + register: current_lscpu + changed_when: false + tags: + - check-cpu-frequency + +- name: Check CPU Power States + ansible.builtin.assert: + that: + - "'CPU min MHz' not in current_lscpu.stdout or 'Intel(R) Xeon(R)' not in ansible_processor" + fail_msg: "CPU configuration!" + success_msg: "CPU configuration match." + tags: + - check-cpu-frequency + +- name: Check Kernel Parameters + ansible.builtin.assert: + that: + - item in ansible_cmdline + fail_msg: "Kernel parameters!" + success_msg: "Kernel parameters match." + loop: "{{ grub.keys()|sort }}" + when: + - grub is defined + tags: + - check-kernel-params + +- name: Check Kernel Version + ansible.builtin.assert: + that: + - ansible_kernel not in kernel_version_by_distro_by_arch + fail_msg: "Kernel version!" + success_msg: "Kernel version match." + tags: + - check-kernel-version + +- name: Spectre Meltdown Checker Status + ansible.builtin.stat: + path: "/opt/spectre-meltdown-checker.sh" + register: spectre_meltdown_status + tags: + - check-spectre-meltdown + +- name: Get Spectre Meltdown Checker + ansible.builtin.get_url: + url: "https://meltdown.ovh" + dest: "/opt/spectre-meltdown-checker.sh" + mode: "744" + when: + - not spectre_meltdown_status.stat.exists + tags: + - check-spectre-meltdown + +- name: Run Spectre Meltdown Checker + ansible.builtin.shell: "/opt/spectre-meltdown-checker.sh --no-color --sysfs-only || true" + ignore_errors: true + register: spectre_meltdown_sync + tags: + - check-spectre-meltdown + +- debug: var=spectre_meltdown_sync.stdout_lines + tags: + - check-spectre-meltdown + +- name: "{{ ansible_machine }} Specific" + include_tasks: "{{ ansible_machine }}.yaml" + tags: + - check-machine-specific + - check-jitter-tool diff --git a/fdio.infra.ansible/roles/calibration/tasks/x86_64.yaml b/fdio.infra.ansible/roles/calibration/tasks/x86_64.yaml new file mode 100644 index 0000000000..2d28f92ae3 --- /dev/null +++ b/fdio.infra.ansible/roles/calibration/tasks/x86_64.yaml @@ -0,0 +1,35 @@ +--- +# file: roles/calibration/tasks/x86_64.yaml + +- name: Calibration - Clone PMA Tool + ansible.builtin.git: + repo: "https://gerrit.fd.io/r/pma_tools" + dest: "{{ pma_directory }}" + tags: + - check-jitter-tool + +- name: Calibration - Compile PMA Tool + ansible.builtin.raw: "cd {{ pma_directory }}/jitter && make" + tags: + - check-jitter-tool + +- name: Calibration - Run Jitter Tool + ansible.builtin.shell: "{{ pma_directory }}/jitter/jitter -c {{ jitter_core }} -i {{ jitter_iterations }} -f" + become: true + async: 60 + poll: 0 + ignore_errors: true + register: jitter_async + tags: + - check-jitter-tool + +- name: Check sync status + ansible.builtin.async_status: + jid: "{{ jitter_async.ansible_job_id }}" + register: "jitter_poll_results" + until: jitter_poll_results.finished + retries: 30 + +- ansible.builtin.debug: var=jitter_poll_results.stdout_lines + tags: + - check-jitter-tool diff --git a/fdio.infra.ansible/roles/cleanup/files/reset_vppdevice.sh b/fdio.infra.ansible/roles/cleanup/files/reset_vppdevice.sh new file mode 100644 index 0000000000..ede2db1273 --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/files/reset_vppdevice.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +set -euo pipefail + +function die () { + # Print the message to standard error end exit with error code specified + # by the second argument. + # + # Hardcoded values: + # - The default error message. + # Arguments: + # - ${1} - The whole error message, be sure to quote. Optional + # - ${2} - the code to exit with, default: 1. + + set +eu + warn "${1:-Unspecified run-time error occurred!}" + exit "${2:-1}" +} + + +function set_eligibility_off { + # Set Nomad eligibility to ineligible for scheduling. Fail otherwise. + + set -euo pipefail + + node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die + node_status="$(nomad node status | grep $(hostname))" || die + + if [[ "${node_status}" != *"ineligible"* ]]; then + nomad node eligibility -disable "${node_id}" || die + node_status="$(nomad node status | grep $(hostname))" || die + if [[ "${node_status}" != *"ineligible"* ]]; then + die "Set eligibility off failed!" + fi + fi +} + + +function set_eligibility_on { + # Set Nomad eligibility to eligible for scheduling. Fail otherwise. + + set -euo pipefail + + node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die + node_status="$(nomad node status | grep $(hostname))" || die + + if [[ "${node_status}" == *"ineligible"* ]]; then + nomad node eligibility -enable "${node_id}" || die + node_status="$(nomad node status | grep $(hostname))" || die + if [[ "${node_status}" == *"ineligible"* ]]; then + die "Set eligibility on failed!" + fi + fi +} + + +function restart_vfs_service { + # Stop and start VF serice. This will reinitialize VFs and driver mappings. + + set -euo pipefail + + warn "Restarting VFs service (this may take few minutes)..." + sudo service csit-initialize-vfs stop || die "Failed to stop VFs service!" + sudo service csit-initialize-vfs start || die "Failed to start VFs service!" +} + + +function wait_for_pending_containers { + # Wait in loop for defined amount of time for pending containers to + # gracefully quit them. If parameter force is specified. Force kill them. + + # Arguments: + # - ${@} - Script parameters. + + set -euo pipefail + + retries=60 + wait_time=60 + containers=(docker ps --quiet --filter name=csit*) + + for i in $(seq 1 ${retries}); do + mapfile -t pending_containers < <( ${containers[@]} ) || die + warn "Waiting for pending containers [${pending_containers[@]}] ..." + if [ ${#pending_containers[@]} -eq 0 ]; then + break + fi + sleep "${wait_time}" || die + done + if [ ${#pending_containers[@]} -ne 0 ]; then + if [[ "${1-}" == "force" ]]; then + warn "Force killing [${pending_containers[@]}] ..." + docker rm --force ${pending_containers[@]} || die + else + die "Still few containers running!" + fi + fi +} + + +function warn () { + # Print the message to standard error. + # + # Arguments: + # - ${@} - The text of the message. + + echo "$@" >&2 +} + + +set_eligibility_off || die +wait_for_pending_containers "${@}" || die +restart_vfs_service || die +set_eligibility_on || die diff --git a/fdio.infra.ansible/roles/cleanup/tasks/clean_images.yaml b/fdio.infra.ansible/roles/cleanup/tasks/clean_images.yaml new file mode 100644 index 0000000000..76704ab50d --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/clean_images.yaml @@ -0,0 +1,36 @@ +--- +# file: tasks/clean_images.yaml + +- name: Clean Docker Images + block: + - name: Clean Images - Prefetch Docker Images + ansible.builtin.cron: + name: "Prefetch docker image {{ item }}" + minute: "10" + hour: "7" + job: "/usr/bin/docker pull {{ item }}" + loop: + "{{ images_to_prefetch_by_arch[ansible_machine] }}" + tags: + - prefetch-docker-images + + - name: Clean Images - Remove Dangling Docker Images + ansible.builtin.cron: + name: "Remove dangling docker images" + minute: "10" + hour: "5" + weekday: "7" + job: "/usr/bin/docker rmi $(/usr/bin/docker images --filter 'dangling=true' -q)" + tags: + - remove-docker-images-dangling + +# TODO: Disabled until all images will be in registry +# - name: Clean Images - Prune Docker Images +# cron: +# name: "Prune docker images" +# minute: "10" +# hour: "6" +# weekday: 7 +# job: "/usr/bin/docker image prune --all --force" +# tags: +# - prune-docker-images diff --git a/fdio.infra.ansible/roles/cleanup/tasks/kill_containers.yaml b/fdio.infra.ansible/roles/cleanup/tasks/kill_containers.yaml new file mode 100644 index 0000000000..dc739eb954 --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/kill_containers.yaml @@ -0,0 +1,42 @@ +--- +# file: tasks/kill_containers.yaml + +- name: Kill Docker Containers + block: + - name: Get Running Docker Containers + ansible.builtin.shell: "docker ps -a --filter name=DUT -q" + register: running_containers + changed_when: false + tags: + - kill-containers + + - name: Remove All Docker Containers + ansible.builtin.shell: "docker rm --force {{ item }}" + with_items: "{{ running_containers.stdout_lines }}" + tags: + - kill-containers + + rescue: + - name: Restart Docker Daemon + ansible.builtin.systemd: + name: "docker" + state: "restarted" + +- name: Kill LXC Containers + block: + - name: Get Running LXC Containers + ansible.builtin.shell: "lxc-ls" + register: running_containers + changed_when: false + tags: + - kill-containers + + - name: Remove All LXC Containers + ansible.builtin.shell: "lxc-destroy --force -n {{ item }}" + with_items: "{{ running_containers.stdout_lines }}" + tags: + - kill-containers + + rescue: + - fail: + msg: "Kill LXC containers failed!" diff --git a/fdio.infra.ansible/roles/cleanup/tasks/kill_process.yaml b/fdio.infra.ansible/roles/cleanup/tasks/kill_process.yaml new file mode 100644 index 0000000000..9ab98a8e57 --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/kill_process.yaml @@ -0,0 +1,38 @@ +--- +# file: tasks/kill_process.yaml + +- name: Kill Process - {{ process }} + block: + - name: Get PID Of {{ process }} + ansible.builtin.shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'" + when: + - process is defined and process != "" + register: running_processes + tags: + - kill-process + + - name: Safe Kill {{ process }} + ansible.builtin.shell: "kill {{ item }}" + with_items: "{{ running_processes.stdout_lines }}" + ignore_errors: true + tags: + - kill-process + + - wait_for: + path: "/proc/{{ item }}/status" + state: "absent" + with_items: "{{ running_processes.stdout_lines }}" + ignore_errors: true + register: killed_processes + tags: + - kill-process + + - name: Kill Process - Force Kill {{ process }} + ansible.builtin.shell: "kill -9 {{ item }}" + with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}" + tags: + - kill-process + + rescue: + - fail: + msg: "Kill process {{ process }} failed!" diff --git a/fdio.infra.ansible/roles/cleanup/tasks/main.yaml b/fdio.infra.ansible/roles/cleanup/tasks/main.yaml new file mode 100644 index 0000000000..c97b9c5d7e --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/main.yaml @@ -0,0 +1,26 @@ +--- +# file: tasks/main.yaml + +- name: tg specific + include_tasks: tg.yaml + when: "'tg' in group_names" + tags: + - cleanup + +- name: sut specific + include_tasks: sut.yaml + when: "'sut' in group_names" + tags: + - cleanup + +- name: vpp_device specific + include_tasks: vpp_device.yaml + when: "'vpp_device' in group_names" + tags: + - cleanup + +- name: nomad specific + include_tasks: nomad.yaml + when: "'nomad' in group_names" + tags: + - cleanup diff --git a/fdio.infra.ansible/roles/cleanup/tasks/nomad.yaml b/fdio.infra.ansible/roles/cleanup/tasks/nomad.yaml new file mode 100644 index 0000000000..086a4eff7d --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/nomad.yaml @@ -0,0 +1,18 @@ +--- +# file: tasks/nomad.yaml + +- name: Host Cleanup + block: + - name: Clean Images + import_tasks: clean_images.yaml + vars: + images_to_prefetch_by_arch: + aarch64: + - "fdiotools/builder-ubuntu2204:prod-aarch64" + - "fdiotools/builder-ubuntu2004:prod-aarch64" + x86_64: + - "fdiotools/builder-ubuntu2204:prod-x86_64" + - "fdiotools/builder-ubuntu2004:prod-x86_64" + - "fdiotools/builder-debian11:prod-x86_64" + tags: + - clean-images diff --git a/fdio.infra.ansible/roles/cleanup/tasks/remove_package.yaml b/fdio.infra.ansible/roles/cleanup/tasks/remove_package.yaml new file mode 100644 index 0000000000..652729bc30 --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/remove_package.yaml @@ -0,0 +1,21 @@ +--- +# file: tasks/remove_package.yaml + +- name: Fix Corrupted APT + ansible.builtin.shell: "dpkg --configure -a" + when: + - ansible_distribution == 'Ubuntu' + tags: + - remove-package + +- name: Remove Package - {{ package }} + ansible.builtin.apt: + name: "{{ package }}" + force: true + purge: true + state: "absent" + failed_when: false + when: + - ansible_distribution == 'Ubuntu' + tags: + - remove-package
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/cleanup/tasks/sut.yaml b/fdio.infra.ansible/roles/cleanup/tasks/sut.yaml new file mode 100644 index 0000000000..22bf596369 --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/sut.yaml @@ -0,0 +1,97 @@ +--- +# file: tasks/sut.yaml + +- name: Host Cleanup + block: + - name: Kill Processes - Qemu + import_tasks: kill_process.yaml + vars: + process: "qemu" + tags: + - kill-process + + - name: Kill Processes - L3fwd + import_tasks: kill_process.yaml + vars: + process: "l3fwd" + tags: + - kill-process + + - name: Kill Processes - Testpmd + import_tasks: kill_process.yaml + vars: + process: "testpmd" + tags: + - kill-process + + - name: Kill Processes - iPerf3 + import_tasks: kill_process.yaml + vars: + process: "iperf3" + tags: + - kill-process + + - name: Kill Processes - nohup + import_tasks: kill_process.yaml + vars: + process: "nohup" + tags: + - kill-process + + - name: Kill Processes - vpp + import_tasks: kill_process.yaml + vars: + process: "vpp" + tags: + - kill-process + + - name: Kill Processes - vpp_echo + import_tasks: kill_process.yaml + vars: + process: "vpp_echo" + tags: + - kill-process + + - name: Find File Or Dir - Core Zip File + ansible.builtin.find: + paths: "/tmp/" + patterns: "*tar.lzo.lrz.xz*" + register: files_to_delete + tags: + - remove-file-dir + + - name: Remove File Or Dir - Core Zip File + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + with_items: "{{ files_to_delete.files }}" + tags: + - remove-file-dir + + - name: Find File Or Dir - Core Dump File + ansible.builtin.find: + paths: "/tmp/" + patterns: "*core*" + register: files_to_delete + tags: + - remove-file-dir + + - name: Remove File Or Dir - Core Dump File + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + with_items: "{{ files_to_delete.files }}" + tags: + - remove-file-dir + + - name: Kill Containers - Remove All Containers + import_tasks: kill_containers.yaml + tags: + - kill-containers + + - name: Remove Packages - Remove VPP + import_tasks: remove_package.yaml + vars: + package: "*vpp*" + tags: + - remove-package diff --git a/fdio.infra.ansible/roles/cleanup/tasks/tg.yaml b/fdio.infra.ansible/roles/cleanup/tasks/tg.yaml new file mode 100644 index 0000000000..8c0162df2c --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/tg.yaml @@ -0,0 +1,13 @@ +--- +# file: tasks/tg.yaml + +- name: Host Cleanup + block: + - name: Kill Processes - TRex + import_tasks: kill_process.yaml + vars: + process: "_t-rex" + when: + - docker_tg is undefined + tags: + - kill-process diff --git a/fdio.infra.ansible/roles/cleanup/tasks/vpp_device.yaml b/fdio.infra.ansible/roles/cleanup/tasks/vpp_device.yaml new file mode 100644 index 0000000000..c97fa0cde5 --- /dev/null +++ b/fdio.infra.ansible/roles/cleanup/tasks/vpp_device.yaml @@ -0,0 +1,32 @@ +--- +# file: tasks/vpp_device.yaml + +- name: Host Cleanup + block: + - name: Reset vpp_device Binary + ansible.builtin.copy: + src: "files/reset_vppdevice.sh" + dest: "/usr/local/bin" + owner: "root" + group: "root" + mode: "744" + tags: + - reset-sriov + + - name: Clean Images + import_tasks: clean_images.yaml + vars: + images_to_prefetch_by_arch: + aarch64: + - "fdiotools/builder-ubuntu2004:prod-aarch64" + - "fdiotools/builder-ubuntu1804:prod-aarch64" + - "fdiotools/builder-centos8:prod-aarch64" + x86_64: + - "fdiotools/builder-ubuntu2004:prod-x86_64" + - "fdiotools/builder-ubuntu1804:prod-x86_64" + - "fdiotools/builder-debian10:prod-x86_64" + - "fdiotools/builder-debian9:prod-x86_64" + - "fdiotools/builder-centos8:prod-x86_64" + - "fdiotools/builder-centos7:prod-x86_64" + tags: + - clean-images diff --git a/fdio.infra.ansible/roles/common/defaults/main.yaml b/fdio.infra.ansible/roles/common/defaults/main.yaml new file mode 100644 index 0000000000..9ded8fcba9 --- /dev/null +++ b/fdio.infra.ansible/roles/common/defaults/main.yaml @@ -0,0 +1,55 @@ +--- +# file: roles/common/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - "autoconf" + - "ca-certificates" + - "cgroup-tools" + - "dkms" + - "iperf3" + - "linux-tools-common" + - "ninja-build" + - "numactl" + - "qemu-system" + - "socat" + - "unzip" + - "virtualenv" + +packages_by_distro: + ubuntu: + jammy: + - "build-essential" + - "libpcap-dev" + - "net-tools" + - "python3-all" + - "python3-apt" + - "python3-cffi" + - "python3-cffi-backend" + - "python3-dev" + - "python3-pip" + - "python3-pyelftools" + - "python3-setuptools" + +packages_by_arch: + aarch64: + - "gfortran" + - "libblas-dev" + - "libffi-dev" + - "liblapack-dev" + - "libssl-dev" + x86_64: + - [] + +# Proxy settings: Uncomment and fill the proper values. These variables will be +# set globally by writing into /etc/environment file on target machine. +# proxy_env: +# http_proxy: http://proxy.com:80 +# HTTP_PROXY: http://proxy.com:80 +# https_proxy: http://proxy.com:80 +# HTTPS_PROXY: http://proxy.com:80 +# ftp_proxy: http://proxy.com:80 +# FTP_PROXY: http://proxy.com:80 +# no_proxy: localhost,127.0.0.1,{{ ansible_default_ipv4.address }} +# NO_PROXY: localhost,127.0.0.1,{{ ansible_default_ipv4.address }} diff --git a/fdio.infra.ansible/roles/common/handlers/main.yaml b/fdio.infra.ansible/roles/common/handlers/main.yaml new file mode 100644 index 0000000000..0a4944b4ca --- /dev/null +++ b/fdio.infra.ansible/roles/common/handlers/main.yaml @@ -0,0 +1,8 @@ +--- +# file: roles/common/handlers/main.yaml + +- name: Reboot Server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/common/tasks/main.yaml b/fdio.infra.ansible/roles/common/tasks/main.yaml new file mode 100644 index 0000000000..e47a1fc7a8 --- /dev/null +++ b/fdio.infra.ansible/roles/common/tasks/main.yaml @@ -0,0 +1,56 @@ +--- +# file: roles/common/tasks/main.yaml + +- name: Conf - Add permanent proxy settings + ansible.builtin.lineinfile: + path: "/etc/environment" + state: "present" + line: "{{ item.key }}={{ item.value }}" + with_dict: "{{ proxy_env }}" + when: proxy_env is defined + tags: + - common-conf-proxy + +- name: Inst - Update package cache (apt) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - common-inst-prerequisites + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: "latest" + tags: + - common-inst-prerequisites + +- name: Inst - Meson (DPDK) + ansible.builtin.pip: + name: + - "meson==0.64.1" + state: "forcereinstall" + tags: + - common-inst-meson + +- name: Conf - sudoers admin + ansible.builtin.lineinfile: + path: "/etc/sudoers" + state: "present" + regexp: "^%admin ALL=" + line: "%admin ALL=(ALL) ALL" + validate: "/usr/sbin/visudo -cf %s" + tags: + - common-conf-sudoers + +- name: Conf - sudoers nopasswd + ansible.builtin.lineinfile: + path: "/etc/sudoers" + state: "present" + regexp: "^%sudo" + line: "%sudo ALL=(ALL:ALL) NOPASSWD: ALL" + validate: "/usr/sbin/visudo -cf %s" + tags: + - common-conf-sudoers diff --git a/fdio.infra.ansible/roles/consul/defaults/main.yaml b/fdio.infra.ansible/roles/consul/defaults/main.yaml new file mode 100644 index 0000000000..9ea38efb56 --- /dev/null +++ b/fdio.infra.ansible/roles/consul/defaults/main.yaml @@ -0,0 +1,87 @@ +--- +# file: defaults/main.yaml + +# Inst - Prerequisites. +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" +packages_base: + - "curl" + - "unzip" +packages_by_distro: + ubuntu: + - [] +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +# Inst - Consul Map. +consul_architecture_map: + amd64: "amd64" + x86_64: "amd64" + armv7l: "arm" + aarch64: "arm64" + 32-bit: "386" + 64-bit: "amd64" +consul_architecture: "{{ consul_architecture_map[ansible_architecture] }}" +consul_version: "1.16.1" +consul_pkg: "consul_{{ consul_version }}_linux_{{ consul_architecture }}.zip" +consul_zip_url: "https://releases.hashicorp.com/consul/{{ consul_version }}/{{ consul_pkg }}" +consul_force_update: false + +# Inst - System paths. +consul_bin_dir: "/usr/local/bin" +consul_config_dir: "/etc/consul.d" +consul_data_dir: "/var/consul" +consul_inst_dir: "/opt" +consul_lockfile: "/var/lock/subsys/consul" +consul_run_dir: "/var/run/consul" +consul_ssl_dir: "/etc/consul.d/ssl" + +# Conf - Service. +consul_node_role: "both" +consul_restart_handler_state: "restarted" +nomad_restart_handler_state: "restarted" +systemd_resolved_state: "stopped" +consul_service_mgr: "" + +# Conf - User and group. +consul_group: "consul" +consul_user: "consul" + +# Conf - base.hcl +consul_allow_tls: true +consul_bind_addr: "{{ ansible_default_ipv4.address }}" +consul_bootstrap_expect: 1 +consul_client_addr: "0.0.0.0" +consul_datacenter: "dc1" +consul_disable_update_check: true +consul_enable_debug: false +consul_enable_syslog: true +consul_encrypt: "" +consul_log_level: "INFO" +consul_node_name: "{{ inventory_hostname }}" +consul_recursors: + - 1.1.1.1 + - 8.8.8.8 +consul_retry_join: false +consul_ui_config: + enabled: true +consul_verify_incoming: true +consul_verify_outgoing: true +consul_vefify_server_hostname: false +consul_ca_file: "{{ consul_ssl_dir }}/ca.pem" +consul_cert_file: "{{ consul_ssl_dir }}/consul.pem" +consul_key_file: "{{ consul_ssl_dir }}/consul-key.pem" + +# Conf - ports.hcl +consul_port_dns: 53 +consul_port_http: 8500 +consul_port_https: 8501 +consul_port_grpc: 8502 +consul_port_serf_lan: 8301 +consul_port_serf_wan: 8302 +consul_port_server: 8300 + +# Conf - services.json +consul_services: false diff --git a/fdio.infra.ansible/roles/consul/handlers/main.yaml b/fdio.infra.ansible/roles/consul/handlers/main.yaml new file mode 100644 index 0000000000..a9de4d1439 --- /dev/null +++ b/fdio.infra.ansible/roles/consul/handlers/main.yaml @@ -0,0 +1,16 @@ +--- +# file handlers/main.yaml + +- name: Restart Nomad + ansible.builtin.systemd: + daemon_reload: true + enabled: true + name: "nomad" + state: "{{ nomad_restart_handler_state }}" + +- name: Restart Consul + ansible.builtin.systemd: + daemon_reload: true + enabled: true + name: "consul" + state: "{{ consul_restart_handler_state }}" diff --git a/fdio.infra.ansible/roles/consul/meta/main.yaml b/fdio.infra.ansible/roles/consul/meta/main.yaml new file mode 100644 index 0000000000..673c3b738d --- /dev/null +++ b/fdio.infra.ansible/roles/consul/meta/main.yaml @@ -0,0 +1,21 @@ +--- +# file: meta/main.yaml + +dependencies: [] + +galaxy_info: + role_name: "consul" + author: "pmikus" + description: "Hashicorp Consul." + company: "none" + license: "license (Apache)" + min_ansible_version: "2.9" + platforms: + - name: "Ubuntu" + versions: + - "focal" + - "jammy" + - "kinetic" + galaxy_tags: + - "consul" + - "hashicorp" diff --git a/fdio.infra.ansible/roles/consul/tasks/main.yaml b/fdio.infra.ansible/roles/consul/tasks/main.yaml new file mode 100644 index 0000000000..6dd430754b --- /dev/null +++ b/fdio.infra.ansible/roles/consul/tasks/main.yaml @@ -0,0 +1,145 @@ +--- +# file: tasks/main.yaml + +- name: Update Repositories Cache + ansible.builtin.apt: + update_cache: true + when: + - ansible_os_family == 'Debian' + tags: + - consul-inst-package + +- name: Dependencies + ansible.builtin.apt: + name: "{{ packages | flatten(levels=1) }}" + state: "present" + cache_valid_time: 3600 + install_recommends: false + when: + - ansible_os_family == 'Debian' + tags: + - consul-inst-dependencies + +- name: Add Consul Group + ansible.builtin.group: + name: "{{ consul_group }}" + state: "present" + tags: + - consul-conf-user + +- name: Add Consul user + ansible.builtin.user: + name: "{{ consul_user }}" + group: "{{ consul_group }}" + state: "present" + system: true + tags: + - consul-conf-user + +- name: Download Consul + ansible.builtin.get_url: + url: "{{ consul_zip_url }}" + dest: "{{ consul_inst_dir }}/{{ consul_pkg }}" + tags: + - consul-inst-package + +- name: Clean Consul + ansible.builtin.file: + path: "{{ consul_inst_dir }}/consul" + state: "absent" + when: + - consul_force_update | bool + tags: + - consul-inst-package + +- name: Unarchive Consul + ansible.builtin.unarchive: + src: "{{ consul_inst_dir }}/{{ consul_pkg }}" + dest: "{{ consul_inst_dir }}/" + remote_src: true + tags: + - consul-inst-package + +- name: Consul + ansible.builtin.copy: + src: "{{ consul_inst_dir }}/consul" + dest: "{{ consul_bin_dir }}" + owner: "{{ consul_user }}" + group: "{{ consul_group }}" + force: true + mode: 0755 + remote_src: true + tags: + - consul-inst-package + +- name: Create Directories + ansible.builtin.file: + dest: "{{ item }}" + state: "directory" + owner: "{{ consul_user }}" + group: "{{ consul_group }}" + mode: 0755 + with_items: + - "{{ consul_config_dir }}" + - "{{ consul_ssl_dir }}" + - "{{ consul_data_dir }}" + - "{{ nomad_config_dir }}" + - "{{ nomad_ssl_dir }}" + tags: + - consul-conf + +- name: Base Configuration + ansible.builtin.template: + src: "{{ item }}.hcl.j2" + dest: "{{ consul_config_dir }}/{{ item }}.hcl" + owner: "{{ consul_user }}" + group: "{{ consul_group }}" + mode: 0644 + with_items: + - "base" + - "ports" + - "telemetry" + tags: + - consul-conf + +- name: Copy Certificates And Keys + ansible.builtin.copy: + content: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ consul_user }}" + group: "{{ consul_group }}" + mode: 0600 + no_log: true + loop: "{{ consul_certificates | flatten(levels=1) }}" + when: + - consul_certificates is defined + tags: + - consul-conf + +- name: Stop Systemd-resolved + ansible.builtin.systemd: + daemon_reload: true + enabled: false + name: "systemd-resolved" + state: "{{ systemd_resolved_state }}" + when: + - consul_service_mgr == "systemd" + tags: + - consul-conf + +- name: System.d Script + ansible.builtin.template: + src: "consul_systemd.service.j2" + dest: "/lib/systemd/system/consul.service" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Restart Consul" + when: + - consul_service_mgr == "systemd" + tags: + - consul-conf + +- name: Flush handlers + ansible.builtin.meta: flush_handlers diff --git a/fdio.infra.ansible/roles/consul/templates/base.hcl.j2 b/fdio.infra.ansible/roles/consul/templates/base.hcl.j2 new file mode 100644 index 0000000000..15104b2710 --- /dev/null +++ b/fdio.infra.ansible/roles/consul/templates/base.hcl.j2 @@ -0,0 +1,56 @@ +node_name = "{{ consul_node_name }}" +datacenter = "{{ consul_datacenter }}" + +bind_addr = "{{ consul_bind_addr }}" +client_addr = "{{ consul_client_addr }}" +data_dir = "{{ consul_data_dir }}" + +enable_syslog = {{ consul_enable_syslog | bool | lower }} +enable_debug = {{ consul_enable_debug | bool | lower }} +disable_update_check = {{ consul_disable_update_check | bool | lower }} +log_level = "{{ consul_log_level }}" + +server = {{ consul_node_server | bool | lower }} +encrypt = "{{ consul_encrypt }}" +{% if consul_node_server | bool == True %} +bootstrap_expect = {{ consul_bootstrap_expect }} +verify_incoming = {{ consul_verify_incoming | bool | lower }} +verify_outgoing = {{ consul_verify_outgoing | bool | lower }} +verify_server_hostname = {{ consul_vefify_server_hostname | bool | lower }} +ca_file = "{{ consul_ca_file }}" +cert_file = "{{ consul_cert_file }}" +key_file = "{{ consul_key_file }}" +auto_encrypt { + allow_tls = {{ consul_allow_tls | bool | lower }} +} +{% else %} +verify_incoming = {{ consul_verify_incoming | bool | lower }} +verify_outgoing = {{ consul_verify_outgoing | bool | lower }} +verify_server_hostname = {{ consul_vefify_server_hostname | bool | lower }} +ca_file = "{{ consul_ca_file }}" +auto_encrypt { + tls = {{ consul_allow_tls | bool | lower }} +} +{% endif %} +{% if consul_retry_join | bool -%} +retry_join = [ {% for ip_port in consul_retry_servers -%} "{{ ip_port }}"{% if not loop.last %}, {% endif %}{%- endfor -%} ] +{%- endif %} + +{% if consul_ui_config -%} +ui_config { +{% for key, value in consul_ui_config.items() %} + {%- if value|bool %} + {{ key }} = {{ value | bool | lower }} + {%- elif value|string or value == "" %} + {{ key }} = "{{ value }}" + {%- else %} + {{ key }} = {{ value }} + {%- endif %} +{% endfor %} + +} +{%- endif %} + +{% if consul_recursors -%} +recursors = [ {% for server in consul_recursors -%} "{{ server }}"{% if not loop.last %}, {% endif %}{%- endfor -%} ] +{%- endif %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2 b/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2 new file mode 100644 index 0000000000..16874f213e --- /dev/null +++ b/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description="HashiCorp Consul - A service mesh solution" +Documentation=https://www.consul.io/ +Requires=network-online.target +After=network-online.target + +[Service] +User=root +Group=root +ExecStart={{ consul_bin_dir }}/consul agent -config-dir={{ consul_config_dir }} +ExecReload=/bin/kill --signal HUP $MAINPID +KillMode=process +KillSignal=SIGTERM +Restart=on-failure +LimitNOFILE=infinity + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/consul/templates/ports.hcl.j2 b/fdio.infra.ansible/roles/consul/templates/ports.hcl.j2 new file mode 100644 index 0000000000..02932bf6dc --- /dev/null +++ b/fdio.infra.ansible/roles/consul/templates/ports.hcl.j2 @@ -0,0 +1,9 @@ +ports { + dns = {{ consul_port_dns }} + http = {{ consul_port_http }} + https = {{ consul_port_https }} + grpc_tls = {{ consul_port_grpc }} + serf_lan = {{ consul_port_serf_lan }} + serf_wan = {{ consul_port_serf_wan }} + server = {{ consul_port_server }} +}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/consul/templates/telemetry.hcl.j2 b/fdio.infra.ansible/roles/consul/templates/telemetry.hcl.j2 new file mode 100644 index 0000000000..ec7fabc9da --- /dev/null +++ b/fdio.infra.ansible/roles/consul/templates/telemetry.hcl.j2 @@ -0,0 +1,3 @@ +telemetry { + prometheus_retention_time = "24h" +}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/consul/vars/main.yaml b/fdio.infra.ansible/roles/consul/vars/main.yaml new file mode 100644 index 0000000000..5d813dffc7 --- /dev/null +++ b/fdio.infra.ansible/roles/consul/vars/main.yaml @@ -0,0 +1,5 @@ +--- +# file: vars/main.yaml + +consul_node_client: "{{ (consul_node_role == 'client') or (consul_node_role == 'both') }}" +consul_node_server: "{{ (consul_node_role == 'server') or (consul_node_role == 'both') }}" diff --git a/fdio.infra.ansible/roles/docker/defaults/main.yaml b/fdio.infra.ansible/roles/docker/defaults/main.yaml new file mode 100644 index 0000000000..bf97b4a192 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/defaults/main.yaml @@ -0,0 +1,35 @@ +--- +# file: roles/docker/defaults/main.yaml + +# Version options. +docker_edition: "ce" +docker_package: "docker-{{ docker_edition }}" +docker_package_state: latest + +# Service options. +docker_service_state: started +docker_service_enabled: true +docker_restart_handler_state: restarted +docker_service_mgr: "systemd" + +# Used only for Debian/Ubuntu. +docker_apt_release_channel: "stable" +docker_apt_repository: "deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable" +docker_apt_repository_state: present +docker_apt_ignore_key_error: true +docker_apt_gpg_key: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg" +docker_apt_gpg_key_state: present + +# Used only for RedHat/CentOS/Fedora. +docker_yum_repo_url: https://download.docker.com/linux/{{ (ansible_distribution == "Fedora") | ternary("fedora","centos") }}/docker-{{ docker_edition }}.repo +docker_yum_repo_enable_edge: "0" +docker_yum_repo_enable_test: "0" +docker_yum_gpg_key: https://download.docker.com/linux/centos/gpg + +# Proxy settings. +docker_daemon_environment_http: + - "HTTP_PROXY={{ proxy_env.http_proxy }}" + - "NO_PROXY={{ proxy_env.no_proxy }}" +docker_daemon_environment_https: + - "HTTPS_PROXY={{ proxy_env.https_proxy }}" + - "NO_PROXY={{ proxy_env.no_proxy }}" diff --git a/fdio.infra.ansible/roles/docker/handlers/main.yaml b/fdio.infra.ansible/roles/docker/handlers/main.yaml new file mode 100644 index 0000000000..53eb8528f6 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/handlers/main.yaml @@ -0,0 +1,9 @@ +--- +# file roles/docker/handlers/main.yaml + +- name: Restart Docker + ansible.builtin.service: + name: "docker" + state: "{{ docker_restart_handler_state }}" + tags: + - docker-restart-service diff --git a/fdio.infra.ansible/roles/docker/meta/main.yaml b/fdio.infra.ansible/roles/docker/meta/main.yaml new file mode 100644 index 0000000000..7bef656eb5 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/meta/main.yaml @@ -0,0 +1,18 @@ +--- +# file: meta/main.yaml + +dependencies: [] + +galaxy_info: + role_name: docker + author: fd.io + description: Docker-CE for Linux. + company: none + license: "license (Apache)" + min_ansible_version: 2.9 + platforms: + - name: Ubuntu + versions: + - jammy + galaxy_tags: + - docker diff --git a/fdio.infra.ansible/roles/docker/tasks/jammy.yaml b/fdio.infra.ansible/roles/docker/tasks/jammy.yaml new file mode 100644 index 0000000000..8ec7a01ee1 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/tasks/jammy.yaml @@ -0,0 +1,30 @@ +--- +# file: roles/docker/tasks/ubuntu_jammy.yaml + +- name: Inst - Dependencies + ansible.builtin.apt: + name: + - "apt-transport-https" + - "ca-certificates" + - "gpg-agent" + - "software-properties-common" + state: "present" + cache_valid_time: 3600 + install_recommends: false + tags: + - docker-inst-dependencies + +- name: Conf - Add APT Key + ansible.builtin.apt_key: + url: "{{ docker_apt_gpg_key }}" + state: "{{ docker_apt_gpg_key_state }}" + tags: + - docker-conf-apt + +- name: Conf - Install APT Repository + ansible.builtin.apt_repository: + repo: "{{ docker_apt_repository }}" + state: "{{ docker_apt_repository_state }}" + update_cache: true + tags: + - docker-conf-apt diff --git a/fdio.infra.ansible/roles/docker/tasks/main.yaml b/fdio.infra.ansible/roles/docker/tasks/main.yaml new file mode 100644 index 0000000000..e07b29e363 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/tasks/main.yaml @@ -0,0 +1,85 @@ +--- +# file: roles/docker/tasks/main.yaml + +- include_tasks: "{{ ansible_distribution_release }}.yaml" + tags: + - docker-inst-dependencies + +- name: Inst - Docker + ansible.builtin.package: + name: + - "{{ docker_package }}" + - "{{ docker_package }}-cli" + state: "{{ docker_package_state }}" + tags: + - docker-inst-package + +- name: Conf - Docker Service + ansible.builtin.service: + name: docker + state: "{{ docker_service_state }}" + enabled: "{{ docker_service_enabled }}" + when: + - docker_service_mgr == "systemd" + tags: + - docker-conf-service + +- name: Conf - Docker Service Directory + ansible.builtin.file: + path: "/etc/systemd/system/docker.service.d" + state: "directory" + mode: "0755" + when: + - docker_service_mgr == "systemd" + tags: + - docker-conf-service + +- name: Conf - Docker Daemon + ansible.builtin.template: + src: "templates/daemon.json.j2" + dest: "/etc/docker/daemon.json" + owner: "root" + group: "root" + mode: "0644" + notify: + - "Restart Docker" + when: > + docker_daemon is defined and + docker_service_mgr == "systemd" + tags: + - docker-conf-daemon + +- name: Conf - Docker HTTP Proxy + ansible.builtin.template: + src: "templates/docker.service.proxy.http" + dest: "/etc/systemd/system/docker.service.d/http-proxy.conf" + owner: "root" + group: "root" + mode: "0644" + notify: + - "Restart Docker" + when: > + proxy_env is defined and + proxy_env.http_proxy is defined and + docker_service_mgr == "systemd" + tags: + - docker-conf-service + +- name: Conf - Docker HTTPS Proxy + ansible.builtin.template: + src: "templates/docker.service.proxy.https" + dest: "/etc/systemd/system/docker.service.d/https-proxy.conf" + owner: "root" + group: "root" + mode: "0644" + notify: + - "Restart Docker" + when: > + proxy_env is defined and + proxy_env.https_proxy is defined and + docker_service_mgr == "systemd" + tags: + - docker-conf-service + +- name: Meta - Flush handlers + ansible.builtin.meta: flush_handlers diff --git a/fdio.infra.ansible/roles/docker/templates/daemon.json.j2 b/fdio.infra.ansible/roles/docker/templates/daemon.json.j2 new file mode 100644 index 0000000000..becc2b1af7 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/templates/daemon.json.j2 @@ -0,0 +1 @@ +{{ docker_daemon | to_nice_json }}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker/templates/docker.service.proxy.http b/fdio.infra.ansible/roles/docker/templates/docker.service.proxy.http new file mode 100644 index 0000000000..73ceba3870 --- /dev/null +++ b/fdio.infra.ansible/roles/docker/templates/docker.service.proxy.http @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +[Service] +Environment="{{ docker_daemon_environment_http | join('" "') }}" diff --git a/fdio.infra.ansible/roles/docker/templates/docker.service.proxy.https b/fdio.infra.ansible/roles/docker/templates/docker.service.proxy.https new file mode 100644 index 0000000000..1c2097eb9d --- /dev/null +++ b/fdio.infra.ansible/roles/docker/templates/docker.service.proxy.https @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +[Service] +Environment="{{ docker_daemon_environment_https | join('" "') }}" diff --git a/fdio.infra.ansible/roles/docker_images/files/base/Dockerfile b/fdio.infra.ansible/roles/docker_images/files/base/Dockerfile new file mode 100644 index 0000000000..88af96bfa8 --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/files/base/Dockerfile @@ -0,0 +1,140 @@ +FROM ubuntu:22.04 + +# Setup the environment +ENV DEBIAN_FRONTEND=noninteractive + +# Configure locales +RUN apt-get update -qq \ + && apt-get install -y \ + apt-utils \ + locales \ + && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \ + && locale-gen en_US.UTF-8 \ + && dpkg-reconfigure --frontend=noninteractive locales \ + && update-locale LANG=en_US.UTF-8 \ + && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone \ + && rm -r /var/lib/apt/lists/* +ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8" + +# Install packages and Docker +RUN apt-get -q update \ + && apt-get install -y -qq \ + apt-transport-https \ + bridge-utils \ + ca-certificates \ + cgroup-tools \ + cloud-init \ + cmake \ + curl \ + dkms \ + ethtool \ + gdb \ + gfortran \ + libapr1 \ + libblas-dev \ + libffi-dev \ + libibverbs-dev \ + liblapack-dev \ + libmbedcrypto7 \ + libmbedtls14 \ + libmbedx509-1 \ + libnuma1 \ + libnuma-dev \ + libpcap-dev \ + libpixman-1-dev \ + libsctp-dev \ + libssl-dev \ + net-tools \ + ninja-build \ + openssh-server \ + pciutils \ + python3-all \ + python3-apt \ + python3-cffi \ + python3-cffi-backend \ + python3-dev \ + python3-pip \ + python3-pyelftools \ + python3-setuptools \ + python3-virtualenv \ + qemu-system \ + rdma-core \ + rsyslog \ + screen \ + socat \ + software-properties-common \ + strace \ + strongswan \ + ssh \ + sshpass \ + sudo \ + supervisor \ + tar \ + tcpdump \ + unzip \ + vim \ + wget \ + zlib1g-dev \ + && ln -s -f /usr/lib/x86_64-linux-gnu/libc.a /usr/lib/x86_64-linux-gnu/liblibc.a \ + && curl -fsSL https://get.docker.com | sh \ + && rm -rf /var/lib/apt/lists/* + +# Fix permissions +RUN chown root:syslog /var/log \ + && chmod 755 /etc/default + +# Create directory structure +RUN mkdir -p /tmp/dumps \ + && mkdir -p /var/cache/vpp/python \ + && mkdir -p /var/run/sshd \ + && mkdir -p /var/log/vpp + +# CSIT PIP pre-cache +RUN pip3 install \ + ecdsa==0.18.0 \ + paramiko==3.3.1 \ + pycrypto==2.6.1 \ + python-dateutil==2.8.2 \ + PyYAML==6.0.1 \ + requests==2.31.0 \ + robotframework==6.1.1 \ + scapy==2.4.5 \ + scp==0.14.5 \ + ansible==8.2.0 \ + ansible-core==2.15.2 \ + dill==0.3.7 \ + numpy==1.25.2 \ + scipy==1.11.1 \ + ply==3.11 \ + jsonschema==4.18.4 \ + rfc3339-validator==0.1.4 \ + rfc3987==1.3.8 \ + attrs==23.1.0 \ + bcrypt==4.0.1 \ + certifi==2023.7.22 \ + cffi==1.15.1 \ + charset-normalizer==3.2.0 \ + cryptography==41.0.3 \ + idna==3.4 \ + Jinja2==3.1.2 \ + jsonschema-specifications==2023.7.1 \ + MarkupSafe==2.1.3 \ + packaging==23.1 \ + pycparser==2.21 \ + PyNaCl==1.5.0 \ + referencing==0.30.0 \ + resolvelib==1.0.1 \ + rpds-py==0.9.2 \ + six==1.16.0 \ + urllib3==2.0.4 \ + meson==0.64.1 + +RUN groupadd -g 1000 testuser \ + && useradd -rm -d /home/testuser -s /bin/bash -g testuser -G sudo -u 1000 testuser \ + && echo 'testuser:Csit1234' | chpasswd + +RUN echo 'root:Csit1234' | chpasswd \ + && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \ + && echo "export VISIBLE=now" >> /etc/profile + +RUN service ssh start
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/files/csit-initialize-docker-sut.service b/fdio.infra.ansible/roles/docker_images/files/csit-initialize-docker-sut.service new file mode 100644 index 0000000000..431387c95c --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/files/csit-initialize-docker-sut.service @@ -0,0 +1,12 @@ +[Unit] +Description=CSIT Initialize Docker SUT +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=True +ExecStart=docker compose -f /opt/csit-docker-images/docker-compose-sut.yaml up --detach +ExecStop=docker compose -f /opt/csit-docker-images/docker-compose-sut.yaml down + +[Install] +WantedBy=default.target diff --git a/fdio.infra.ansible/roles/docker_images/files/csit-initialize-docker-tg.service b/fdio.infra.ansible/roles/docker_images/files/csit-initialize-docker-tg.service new file mode 100644 index 0000000000..2c93724a4c --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/files/csit-initialize-docker-tg.service @@ -0,0 +1,12 @@ +[Unit] +Description=CSIT Initialize Docker TG +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=True +ExecStart=docker compose -f /opt/csit-docker-images/docker-compose-tg.yaml up --detach +ExecStop=docker compose -f /opt/csit-docker-images/docker-compose-tg.yaml down + +[Install] +WantedBy=default.target diff --git a/fdio.infra.ansible/roles/docker_images/files/csit-sut/Dockerfile b/fdio.infra.ansible/roles/docker_images/files/csit-sut/Dockerfile new file mode 100644 index 0000000000..85537bc32f --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/files/csit-sut/Dockerfile @@ -0,0 +1,7 @@ +FROM base-ubuntu2204:local + +EXPOSE 2222 + +COPY supervisord.conf /etc/supervisor/supervisord.conf + +CMD ["sh", "-c", "rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api; /usr/bin/supervisord -c /etc/supervisor/supervisord.conf; /usr/sbin/sshd -D -p 2222"]
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/files/csit-sut/supervisord.conf b/fdio.infra.ansible/roles/docker_images/files/csit-sut/supervisord.conf new file mode 100644 index 0000000000..22a36be5c6 --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/files/csit-sut/supervisord.conf @@ -0,0 +1,24 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[supervisord] +pidfile = /tmp/supervisord.pid +identifier = supervisor +directory = /tmp +logfile = /tmp/supervisord.log +loglevel = debug +nodaemon = false + +[program:vpp] +command = /usr/bin/vpp -c /etc/vpp/startup.conf +autostart = false +autorestart = true +redirect_stderr = true +priority = 1
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/handlers/main.yaml b/fdio.infra.ansible/roles/docker_images/handlers/main.yaml new file mode 100644 index 0000000000..766eec432a --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/handlers/main.yaml @@ -0,0 +1,18 @@ +--- +# file: handlers/main.yaml + +- name: "Start csit-initialize-docker-sut.service" + ansible.builtin.systemd: + enabled: true + state: "started" + name: "csit-initialize-docker-sut.service" + tags: + - docker-sut + +- name: "Start csit-initialize-docker-tg.service" + ansible.builtin.systemd: + enabled: true + state: "started" + name: "csit-initialize-docker-tg.service" + tags: + - docker-tg
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/tasks/base.yaml b/fdio.infra.ansible/roles/docker_images/tasks/base.yaml new file mode 100644 index 0000000000..69b3f6217d --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/tasks/base.yaml @@ -0,0 +1,63 @@ +--- +# file: tasks/base.yaml + +- name: "Create a Directory For Docker Images" + ansible.builtin.file: + path: "/opt/csit-docker-images/" + state: "directory" + mode: 0755 + tags: + - docker-base + +- name: "Copy Build Items" + ansible.builtin.copy: + src: "{{ item }}" + dest: "/opt/csit-docker-images/{{ item }}" + owner: "root" + group: "root" + mode: 0755 + with_items: + - "base/" + - "csit-sut/" + tags: + - docker-base + +- name: "Build CSIT Base Docker Image" + ansible.builtin.shell: "docker build -t base-ubuntu2204:local ." + args: + chdir: "/opt/csit-docker-images/base" + async: 3000 + poll: 0 + register: "docker_built" + tags: + - docker-base + +- name: "Check if CSIT Base Docker Image is Built" + async_status: + jid: "{{ docker_built.ansible_job_id }}" + register: "docker_built" + until: "docker_built.finished" + delay: 10 + retries: 300 + tags: + - docker-base + +- name: "Build CSIT OLD Docker Image" + ansible.builtin.shell: "docker build -t csit_sut-ubuntu2204:local ." + args: + chdir: "/opt/csit-docker-images/csit-sut" + async: 3000 + poll: 0 + register: "docker_built" + tags: + - docker-base + +- name: "Check if CSIT OLD Docker Image is Built" + async_status: + jid: "{{ docker_built.ansible_job_id }}" + register: "docker_built" + until: "docker_built.finished" + delay: 10 + retries: 300 + tags: + - docker-base
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/tasks/main.yaml b/fdio.infra.ansible/roles/docker_images/tasks/main.yaml new file mode 100644 index 0000000000..1005e024f2 --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/tasks/main.yaml @@ -0,0 +1,21 @@ +--- +# file: tasks/main.yaml + +- name: "Build Base Docker Images" + import_tasks: "base.yaml" + tags: + - docker-base + +- name: "Docker Orchestration for TG" + import_tasks: "tg.yaml" + when: > + docker_tg is defined + tags: + - docker-tg + +- name: "Docker Orchestration for SUT" + import_tasks: "sut.yaml" + when: > + docker_sut is defined + tags: + - docker-sut
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/tasks/sut.yaml b/fdio.infra.ansible/roles/docker_images/tasks/sut.yaml new file mode 100644 index 0000000000..8ac179573d --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/tasks/sut.yaml @@ -0,0 +1,28 @@ +--- +# file: tasks/sut.yaml + +- name: "Template Compose File" + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/opt/csit-docker-images/{{ item }}" + owner: "root" + group: "root" + mode: 0755 + with_items: + - "docker-compose-sut.yaml" + tags: + - docker-sut + +- name: "Copy csit-initialize-docker-sut.service" + ansible.builtin.copy: + src: "files/csit-initialize-docker-sut.service" + dest: "/etc/systemd/system/" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Start csit-initialize-docker-sut.service" + tags: + - docker-sut + +- meta: flush_handlers
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/tasks/tg.yaml b/fdio.infra.ansible/roles/docker_images/tasks/tg.yaml new file mode 100644 index 0000000000..0623616073 --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/tasks/tg.yaml @@ -0,0 +1,28 @@ +--- +# file: tasks/tg.yaml + +- name: "Template Compose File" + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/opt/csit-docker-images/{{ item }}" + owner: "root" + group: "root" + mode: 0755 + with_items: + - "docker-compose-tg.yaml" + tags: + - docker-tg + +- name: "Start csit-initialize-docker-tg.service" + ansible.builtin.copy: + src: "files/csit-initialize-docker-tg.service" + dest: "/etc/systemd/system/" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Start csit-initialize-docker-tg.service" + tags: + - docker-tg + +- meta: flush_handlers
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/templates/docker-compose-sut.yaml.j2 b/fdio.infra.ansible/roles/docker_images/templates/docker-compose-sut.yaml.j2 new file mode 100644 index 0000000000..b4713d8552 --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/templates/docker-compose-sut.yaml.j2 @@ -0,0 +1,42 @@ +version: "3" +services: + numa-0: + build: + context: "base/" + dockerfile: "Dockerfile" + cap_add: + - NET_RAW + command: ["/usr/sbin/sshd","-D", "-p", "6001"] + expose: + - "6001" + hostname: "{{ ansible_hostname[:-1] }}1" + network_mode: "host" + privileged: true + restart: "always" + shm_size: "4G" + volumes: +{% for volume in docker_volumes %} + - type: "bind" + source: "{{ volume.source }}" + target: "{{ volume.target }}" +{% endfor %} + numa-1: + build: + context: "base/" + dockerfile: "Dockerfile" + cap_add: + - NET_RAW + command: ["/usr/sbin/sshd","-D", "-p", "6002"] + expose: + - "6002" + hostname: "{{ ansible_hostname[:-1] }}2" + network_mode: "host" + privileged: true + restart: "always" + shm_size: "4G" + volumes: +{% for volume in docker_volumes %} + - type: "bind" + source: "{{ volume.source }}" + target: "{{ volume.target }}" +{% endfor %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/docker_images/templates/docker-compose-tg.yaml.j2 b/fdio.infra.ansible/roles/docker_images/templates/docker-compose-tg.yaml.j2 new file mode 100644 index 0000000000..2cee85e169 --- /dev/null +++ b/fdio.infra.ansible/roles/docker_images/templates/docker-compose-tg.yaml.j2 @@ -0,0 +1,38 @@ +version: "3" +services: + tg-0: + build: + context: "base/" + dockerfile: "Dockerfile" + command: ["/usr/sbin/sshd","-D", "-p", "6001"] + expose: + - "6001" + hostname: "{{ ansible_hostname }}" + network_mode: "host" + privileged: true + restart: "always" + shm_size: "4G" + volumes: +{% for volume in docker_volumes %} + - type: "bind" + source: "{{ volume.source }}" + target: "{{ volume.target }}" +{% endfor %} + tg-1: + build: + context: "base/" + dockerfile: "Dockerfile" + command: ["/usr/sbin/sshd","-D", "-p", "6002"] + expose: + - "6002" + hostname: "{{ ansible_hostname }}" + network_mode: "host" + privileged: true + restart: "always" + shm_size: "4G" + volumes: +{% for volume in docker_volumes %} + - type: "bind" + source: "{{ volume.source }}" + target: "{{ volume.target }}" +{% endfor %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/dpdk/defaults/main.yaml b/fdio.infra.ansible/roles/dpdk/defaults/main.yaml new file mode 100644 index 0000000000..d94e9ac91f --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/defaults/main.yaml @@ -0,0 +1,24 @@ +--- +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + jammy: + - "build-essential" + - "libnuma-dev" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +dpdk_target_dir: "/opt" +dpdk_version: + - "23.11" +dpdk_url: "https://fast.dpdk.org/rel" diff --git a/fdio.infra.ansible/roles/dpdk/files/dpdk-mlx5.patch b/fdio.infra.ansible/roles/dpdk/files/dpdk-mlx5.patch new file mode 100644 index 0000000000..a3928d70f7 --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/files/dpdk-mlx5.patch @@ -0,0 +1,19 @@ +diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c +index d7d3bc73c..c21c38485 100644 +--- a/drivers/net/mlx5/mlx5_ethdev.c ++++ b/drivers/net/mlx5/mlx5_ethdev.c +@@ -1032,11 +1032,14 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, + ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); + dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); ++#if 0 ++ /* FIXME: this does not work on Azure w/ CX4-LX */ + if (((dev_link.link_speed && !dev_link.link_status) || + (!dev_link.link_speed && dev_link.link_status))) { + rte_errno = EAGAIN; + return -rte_errno; + } ++#endif + *link = dev_link; + return 0; + } diff --git a/fdio.infra.ansible/roles/dpdk/meta/main.yaml b/fdio.infra.ansible/roles/dpdk/meta/main.yaml new file mode 100644 index 0000000000..3ca2918d36 --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/meta/main.yaml @@ -0,0 +1,18 @@ +--- +# file: meta/main.yaml + +dependencies: [] + +galaxy_info: + role_name: "dpdk" + author: "fd.io" + description: "DPDK for Linux." + company: "none" + license: "license (Apache)" + min_ansible_version: 2.9 + platforms: + - name: "Ubuntu" + versions: + - "jammy" + galaxy_tags: + - "dpdk" diff --git a/fdio.infra.ansible/roles/dpdk/molecule/default/converge.yml b/fdio.infra.ansible/roles/dpdk/molecule/default/converge.yml new file mode 100644 index 0000000000..ab8392c3dc --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/molecule/default/converge.yml @@ -0,0 +1,9 @@ +--- +# file: molecule/default/converge.yaml + +- name: Converge + hosts: all + become: true + + roles: + - role: csit.dpdk diff --git a/fdio.infra.ansible/roles/dpdk/molecule/default/molecule.yml b/fdio.infra.ansible/roles/dpdk/molecule/default/molecule.yml new file mode 100644 index 0000000000..060f842db7 --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/molecule/default/molecule.yml @@ -0,0 +1,21 @@ +--- +# file: molecule/default/molecule.yaml + +dependency: + name: galaxy +driver: + name: docker +lint: | + yamllint . + ansible-lint +platforms: + - name: ${DISTRO:-ubuntu-22.04} + image: "pmikus/docker-${MOLECULE_DISTRO:-ubuntu-22.04}-ansible:latest" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + pre_build_image: true +provisioner: + name: ansible + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} diff --git a/fdio.infra.ansible/roles/dpdk/tasks/deploy_block.yaml b/fdio.infra.ansible/roles/dpdk/tasks/deploy_block.yaml new file mode 100644 index 0000000000..1f972f5320 --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/tasks/deploy_block.yaml @@ -0,0 +1,33 @@ +--- +# file: tasks/deploy_block.yaml + +- name: Download Release {{ item }} + ansible.builtin.get_url: + url: "{{ dpdk_url }}/dpdk-{{ item }}.tar.xz" + dest: "{{ dpdk_target_dir }}/dpdk-{{ item }}.tar.xz" + mode: 0644 + register: dpdk_downloaded + +- name: Extract Release {{ item }} + ansible.builtin.unarchive: + remote_src: true + src: "{{ dpdk_target_dir }}/dpdk-{{ item }}.tar.xz" + dest: "{{ dpdk_target_dir }}/" + creates: "{{ dpdk_target_dir }}/dpdk-{{ item }}" + when: dpdk_downloaded + register: dpdk_extracted + +- name: Compile Release I + ansible.builtin.command: "meson -Dexamples=l3fwd build" + args: + chdir: "{{ dpdk_target_dir }}/dpdk-{{ item }}" + environment: + CFLAGS: "-DRTE_LIBRTE_I40E_16BYTE_RX_DESC=y" + register: dpdk_compiled + +- name: Compile Release II + ansible.builtin.command: "ninja -C build" + args: + chdir: "{{ dpdk_target_dir }}/dpdk-{{ item }}" + environment: + CFLAGS: "-DRTE_LIBRTE_I40E_16BYTE_RX_DESC=y"
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/dpdk/tasks/main.yaml b/fdio.infra.ansible/roles/dpdk/tasks/main.yaml new file mode 100644 index 0000000000..4f6c9ec9f2 --- /dev/null +++ b/fdio.infra.ansible/roles/dpdk/tasks/main.yaml @@ -0,0 +1,24 @@ +--- +# file: tasks/main.yaml + +- name: Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution == 'Ubuntu' + tags: + - dpdk-inst-prerequisites + +- name: Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: "latest" + tags: + - dpdk-inst-prerequisites + +- name: Multiple DPDK Versions + include_tasks: deploy_block.yaml + loop: "{{ dpdk_version }}" + tags: + - dpdk-inst diff --git a/fdio.infra.ansible/roles/intel/defaults/main.yaml b/fdio.infra.ansible/roles/intel/defaults/main.yaml new file mode 100644 index 0000000000..9a3c5c0f0c --- /dev/null +++ b/fdio.infra.ansible/roles/intel/defaults/main.yaml @@ -0,0 +1,111 @@ +--- +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "build-essential" + - "autoconf" + - "automake" + - "autotools-dev" + - "libtool" + - "pkgconf" + - "asciidoc" + - "xmlto" + - "uuid-dev" + - "libjson-c-dev" + - "libkeyutils-dev" + - "libz-dev" + - "libssl-dev" + - "debhelper" + - "devscripts" + - "debmake" + - "quilt" + - "fakeroot" + - "lintian" + - "asciidoctor" + - "file" + - "gnupg" + - "patch" + - "patchutils" + - "libboost-dev" + - "libboost-regex-dev" +# - "libudev-dev" http://security.ubuntu.com/ubuntu/pool/main/s/systemd/ + - "yasm" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +intel_sourceforge_download_url: "https://sourceforge.net/projects/e1000/files" +intel_download_url: "https://downloadmirror.intel.com" +intel_extract_dir: "/opt" + +intel_700_compatibility_matrix: + dpdk22.07: + # https://doc.dpdk.org/guides/rel_notes/release_22_07.html + i40e: "2.19.3" + iavf: "4.5.3" + nvm: "8.70" + dpdk23.11: + # https://doc.dpdk.org/guides/rel_notes/release_23_11.html + i40e: "2.23.17" + iavf: "4.9.5" + nvm: "9.30" + +intel_800_compatibility_matrix: + dpdk22.03: + # custom for vpp_device + ice: "1.13.7" + ddp: "1.3.45.0" + iavf: "4.9.5" + nvm: "4.40" + dpdk22.07: + # https://doc.dpdk.org/guides/rel_notes/release_22_07.html + ice: "1.9.7" + ddp: "1.3.37.0" + iavf: "4.5.3" + nvm: "4.00" + dpdk23.11: + # https://doc.dpdk.org/guides/rel_notes/release_23_11.html + ice: "1.13.7" + ddp: "1.3.45.0" + iavf: "4.9.5" + nvm: "4.40" + +intel_dsa_compatibility_matrix: + dsa: "4.0" + +intel_qat_compatibility_matrix: + qat2: "1.0.20-00008" + qat1: "4.22.0-00001" + +intel_i40e_url: + "2.19.3": "i40e%20stable/2.19.3/i40e-2.19.3.tar.gz/download" + "2.23.17": "i40e%20stable/2.23.17/i40e-2.23.17.tar.gz/download" + +intel_ice_url: + "1.9.7": "ice%20stable/1.9.7/ice-1.9.7.tar.gz/download" + "1.13.7": "ice%20stable/1.13.7/ice-1.13.7.tar.gz/download" + +intel_iavf_url: + "4.3.19": "iavf%20stable/4.3.19/iavf-4.3.19.tar.gz/download" + "4.5.3": "iavf%20stable/4.5.3/iavf-4.5.3.tar.gz/download" + "4.9.5": "iavf%20stable/4.9.5/iavf-4.9.5.tar.gz/download" + +intel_ddp_url: + "1.3.37.0": "738733/800%20Series%20DDP%20Comms%20Package%201.3.37.0.zip" + "1.3.45.0": "785846/738693_ice_comms-1.3.45.0.zip" + +intel_dsa_url: + "4.0": "https://github.com/intel/idxd-config/archive/refs/tags/accel-config-v4.0.tar.gz" + +intel_qat_url: + "1.0.20-00008": "777529/QAT20.L.1.0.20-00008.tar.gz" + "4.22.0-00001": "780675/QAT.L.4.22.0-00001.tar.gz" diff --git a/fdio.infra.ansible/roles/intel/tasks/dsa.yaml b/fdio.infra.ansible/roles/intel/tasks/dsa.yaml new file mode 100644 index 0000000000..2f038b0e9f --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/dsa.yaml @@ -0,0 +1,39 @@ +--- +# file: tasks/dsa.yaml + +- name: Get DSA Driver + ansible.builtin.uri: + url: "{{ intel_dsa_url[dsa] }}" + follow_redirects: "all" + force: true + dest: "{{ intel_extract_dir }}/accel-config-v{{ dsa }}.tar.gz" + mode: "0644" + failed_when: false + tags: + - intel-inst + +- name: Extract DSA Driver + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/accel-config-v{{ dsa }}.tar.gz" + dest: "{{ intel_extract_dir }}/" + creates: "{{ intel_extract_dir }}/idxd-config-accel-config-v{{ dsa }}" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Install DSA Driver + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ intel_extract_dir }}/idxd-config-accel-config-v{{ dsa }}" + become: true + with_items: + - "./autogen.sh" + - "./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib64" + - "make" + - "make check" + - "make install" + when: + - intel_driver_extracted + tags: + - intel-inst diff --git a/fdio.infra.ansible/roles/intel/tasks/i40e.yaml b/fdio.infra.ansible/roles/intel/tasks/i40e.yaml new file mode 100644 index 0000000000..8b069bf9b2 --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/i40e.yaml @@ -0,0 +1,37 @@ +--- +# file: tasks/i40e.yaml + +- name: Get i40e Network Adapter Driver + ansible.builtin.uri: + url: "{{ intel_sourceforge_download_url }}/{{ intel_i40e_url[i40e] }}" + follow_redirects: "all" + force: true + dest: "{{ intel_extract_dir }}/i40e-{{ i40e }}.tar.gz" + mode: "0644" + failed_when: false + tags: + - intel-inst + +- name: Extract i40e Network Adapter Driver + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/i40e-{{ i40e }}.tar.gz" + dest: "{{ intel_extract_dir }}/" + creates: "{{ intel_extract_dir }}/i40e-{{ i40e }}" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Install i40e Network Adapter Driver + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ intel_extract_dir }}/i40e-{{ i40e }}/src" + become: true + with_items: + - "make install" + #- "modprobe -r i40e" + - "modprobe i40e" + when: + - intel_driver_extracted + tags: + - intel-inst diff --git a/fdio.infra.ansible/roles/intel/tasks/iavf.yaml b/fdio.infra.ansible/roles/intel/tasks/iavf.yaml new file mode 100644 index 0000000000..127e31bee2 --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/iavf.yaml @@ -0,0 +1,37 @@ +--- +# file: tasks/iavf.yaml + +- name: Get iavf Network Adapter Driver + ansible.builtin.uri: + url: "{{ intel_sourceforge_download_url }}/{{ intel_iavf_url[iavf] }}" + follow_redirects: "all" + force: true + dest: "{{ intel_extract_dir }}/iavf-{{ iavf }}.tar.gz" + mode: "0644" + failed_when: false + tags: + - intel-inst + +- name: Extract iavf Network Adapter Driver + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/iavf-{{ iavf }}.tar.gz" + dest: "{{ intel_extract_dir }}/" + creates: "{{ intel_extract_dir }}/iavf-{{ iavf }}" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Install iavf Network Adapter Driver + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ intel_extract_dir }}/iavf-{{ iavf }}/src" + become: true + with_items: + - "make install" + - "modprobe -r iavf" + - "modprobe iavf" + when: + - intel_driver_extracted + tags: + - intel-inst diff --git a/fdio.infra.ansible/roles/intel/tasks/ice.yaml b/fdio.infra.ansible/roles/intel/tasks/ice.yaml new file mode 100644 index 0000000000..c773a65a34 --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/ice.yaml @@ -0,0 +1,91 @@ +--- +# file: tasks/ice.yaml + +- name: Get ice Network Adapter Driver + ansible.builtin.uri: + url: "{{ intel_sourceforge_download_url }}/{{ intel_ice_url[ice] }}" + follow_redirects: "all" + force: true + dest: "{{ intel_extract_dir }}/ice-{{ ice }}.tar.gz" + mode: "0644" + failed_when: false + tags: + - intel-inst + +- name: Extract ice Network Adapter Driver + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/ice-{{ ice }}.tar.gz" + dest: "{{ intel_extract_dir }}/" + creates: "{{ intel_extract_dir }}/ice-{{ ice }}" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Install ice Network Adapter Driver + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ intel_extract_dir }}/ice-{{ ice }}/src" + become: true + with_items: + - "make install" + #- "modprobe -r ice" + - "modprobe ice" + when: + - intel_driver_extracted + tags: + - intel-inst + +- name: Get Dynamic Device Personalization (DDP) Package + ansible.builtin.get_url: + url: "{{ intel_download_url }}/{{ intel_ddp_url[ddp] }}" + dest: "{{ intel_extract_dir }}/800-Series-Comms-Binary-Package-{{ ddp }}.zip" + mode: 0644 + tags: + - intel-inst + +- name: Extract Dynamic Device Personalization (DDP) Package + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/800-Series-Comms-Binary-Package-{{ ddp }}.zip" + dest: "{{ intel_extract_dir }}/" + creates: "{{ intel_extract_dir }}/ice_comms-{{ ddp }}.zip" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Extract Dynamic Device Personalization (DDP) Package + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/ice_comms-{{ ddp }}.zip" + dest: "{{ intel_extract_dir }}/" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Copy Dynamic Device Personalization (DDP) Package + ansible.builtin.copy: + src: "{{ intel_extract_dir }}/ice_comms-{{ ddp }}.pkg" + dest: "/lib/firmware/updates/intel/ice/ddp/ice-{{ ddp }}.pkg" + remote_src: true + follow: true + tags: + - intel-inst + +- name: Link Dynamic Device Personalization (DDP) Package + ansible.builtin.file: + src: "ice-{{ ddp }}.pkg" + dest: "/lib/firmware/updates/intel/ice/ddp/ice.pkg" + state: link + tags: + - intel-inst + +- name: Extract Dynamic Device Personalization (DDP) Package (cleanup) + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_items: + - "{{ intel_extract_dir }}/E810 DDP for Comms TechGuide_Rev2.3.pdf" + - "{{ intel_extract_dir }}/Intel_800_series_market_segment_DDP_license.txt" + tags: + - intel-inst diff --git a/fdio.infra.ansible/roles/intel/tasks/main.yaml b/fdio.infra.ansible/roles/intel/tasks/main.yaml new file mode 100644 index 0000000000..d7598deca7 --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/main.yaml @@ -0,0 +1,146 @@ +--- +# file: tasks/main.yaml + +- name: Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - intel-inst-drivers + +- name: Install Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - intel-inst-drivers + +- name: Check Presence of Intel Ethernet 700 Series + ansible.builtin.shell: "lspci -d 8086:1583; lspci -d 8086:1585; lspci -d 8086:1572; lspci -d 8086:158a; lspci -d 8086:158b" + register: intel_700_pcis + failed_when: false + changed_when: false + tags: + - intel-inst-drivers + +- name: Check Presence of Intel Ethernet 800 Series + ansible.builtin.shell: "lspci -d 8086:1592; lspci -d 8086:1891; lspci -d 8086:188c" + register: intel_800_pcis + failed_when: false + changed_when: false + tags: + - intel-inst-drivers + +- name: Check Presence of Intel DSA + ansible.builtin.shell: "lspci -d 8086:0b25" + register: intel_dsa_pcis + failed_when: false + changed_when: false + tags: + - intel-inst-drivers + +- name: Check Presence of Intel C4XXX + ansible.builtin.shell: "lspci -d 8086:18a0" + register: intel_qat1_pcis + failed_when: false + changed_when: false + tags: + - intel-inst-drivers + +- name: Check Presence of Intel 4XXX + ansible.builtin.shell: "lspci -d 8086:4942" + register: intel_qat2_pcis + failed_when: false + changed_when: false + tags: + - intel-inst-drivers + +- name: Get Intel Ethernet 700 Series driver versions + ansible.builtin.set_fact: + i40e: "{{ intel_700_compatibility_matrix[intel_700_matrix]['i40e'] }}" + iavf: "{{ intel_700_compatibility_matrix[intel_700_matrix]['iavf'] }}" + nvm: "{{ intel_700_compatibility_matrix[intel_700_matrix]['nvm'] }}" + when: > + intel_700_matrix is defined + tags: + - intel-inst-drivers + +- name: Get Intel Ethernet 800 Series driver versions + ansible.builtin.set_fact: + ice: "{{ intel_800_compatibility_matrix[intel_800_matrix]['ice'] }}" + ddp: "{{ intel_800_compatibility_matrix[intel_800_matrix]['ddp'] }}" + iavf: "{{ intel_800_compatibility_matrix[intel_800_matrix]['iavf'] }}" + nvm: "{{ intel_800_compatibility_matrix[intel_800_matrix]['nvm'] }}" + when: > + intel_800_matrix is defined + tags: + - intel-inst-drivers + +- name: Get Intel DSA driver versions + ansible.builtin.set_fact: + dsa: "{{ intel_dsa_compatibility_matrix['dsa'] }}" + when: > + intel_dsa_matrix is defined + tags: + - intel-inst-drivers + +- name: Get Intel QAT driver versions + ansible.builtin.set_fact: + qat1: "{{ intel_qat_compatibility_matrix['qat1'] }}" + qat2: "{{ intel_qat_compatibility_matrix['qat2'] }}" + when: > + intel_qat_matrix is defined + tags: + - intel-inst-drivers + +- name: Driver Intel Ethernet 700 Series + import_tasks: i40e.yaml + when: > + intel_700_pcis.stdout_lines | length > 0 and + intel_700_matrix is defined + tags: + - intel-inst-drivers + +- name: Driver Intel Ethernet 800 Series + import_tasks: ice.yaml + when: > + intel_800_pcis.stdout_lines | length > 0 and + intel_800_matrix is defined + tags: + - intel-inst-drivers + +- name: Driver Intel iAVF + import_tasks: iavf.yaml + when: > + (intel_700_pcis.stdout_lines | length > 0 and + intel_700_matrix is defined) or + (intel_800_pcis.stdout_lines | length > 0 and + intel_800_matrix is defined) + tags: + - intel-inst-drivers + +- name: Driver Intel DSA + import_tasks: dsa.yaml + when: > + intel_dsa_pcis.stdout_lines | length > 0 and + intel_dsa_matrix is defined + tags: + - intel-inst-drivers + +- name: Driver Intel QAT 1.x + import_tasks: qat1.yaml + when: > + intel_qat1_pcis.stdout_lines | length > 0 and + intel_qat_matrix is defined + tags: + - intel-inst-drivers + +- name: Driver Intel QAT 2.x + import_tasks: qat2.yaml + when: > + intel_qat2_pcis.stdout_lines | length > 0 and + intel_qat_matrix is defined + tags: + - intel-inst-drivers
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/intel/tasks/qat1.yaml b/fdio.infra.ansible/roles/intel/tasks/qat1.yaml new file mode 100644 index 0000000000..701c0c1bf1 --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/qat1.yaml @@ -0,0 +1,54 @@ +--- +# file: tasks/qat1.yaml + +- name: Get QAT 1.x Driver + ansible.builtin.uri: + url: "{{ intel_download_url }}/{{ intel_qat_url[qat1] }}" + follow_redirects: "all" + force: true + dest: "{{ intel_extract_dir }}/QAT.L.{{ qat1 }}.tar.gz" + mode: "0644" + failed_when: false + tags: + - intel-inst + +- name: Create a Directory For QAT 1.x Driver + ansible.builtin.file: + path: "{{ intel_extract_dir }}/QAT.L.{{ qat1 }}/" + state: "directory" + mode: "0755" + tags: + - intel-inst + +- name: Extract QAT 1.x Driver + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/QAT.L.{{ qat1 }}.tar.gz" + dest: "{{ intel_extract_dir }}/QAT.L.{{ qat1 }}/" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Install QAT1.x Driver + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ intel_extract_dir }}/QAT.L.{{ qat1 }}" + become: true + with_items: + - "./configure --enable-icp-sriov=host --enable-icp-sym-only" + - "make" + - "make install" + when: + - intel_driver_extracted + tags: + - intel-inst + +- name: Load Kernel Modules By Default + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "qat_c4xxx" + tags: + - intel-inst
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/intel/tasks/qat2.yaml b/fdio.infra.ansible/roles/intel/tasks/qat2.yaml new file mode 100644 index 0000000000..a560f16b2c --- /dev/null +++ b/fdio.infra.ansible/roles/intel/tasks/qat2.yaml @@ -0,0 +1,57 @@ +--- +# file: tasks/qat2.yaml + +- name: Get QAT 2.x Driver + ansible.builtin.uri: + url: "{{ intel_download_url }}/{{ intel_qat_url[qat2] }}" + follow_redirects: "all" + force: true + dest: "{{ intel_extract_dir }}/QAT20.L.{{ qat2 }}.tar.gz" + mode: "0644" + failed_when: false + tags: + - intel-inst + +- name: Create a Directory For QAT 2.x Driver + ansible.builtin.file: + path: "{{ intel_extract_dir }}/QAT20.L.{{ qat2 }}/" + state: "directory" + mode: "0755" + tags: + - intel-inst + +- name: Extract QAT 2.x Driver + ansible.builtin.unarchive: + remote_src: true + src: "{{ intel_extract_dir }}/QAT20.L.{{ qat2 }}.tar.gz" + dest: "{{ intel_extract_dir }}/QAT20.L.{{ qat2 }}/" + register: intel_driver_extracted + tags: + - intel-inst + +- name: Install QAT 2.x Driver + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ intel_extract_dir }}/QAT20.L.{{ qat2 }}" + become: true + with_items: + - "wget http://security.ubuntu.com/ubuntu/pool/main/s/systemd/libudev-dev_249.11-0ubuntu3.7_amd64.deb" + - "dpkg -i ./libudev-dev_249.11-0ubuntu3.7_amd64.deb" + - "./configure --enable-icp-sriov=host --enable-icp-sym-only" + - "make" + - "make install" + - "apt remove -y libudev-dev" + when: + - intel_driver_extracted + tags: + - intel-inst + +- name: Load Kernel Modules By Default + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "qat_4xxx" + tags: + - intel-inst
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/iperf/defaults/main.yaml b/fdio.infra.ansible/roles/iperf/defaults/main.yaml new file mode 100644 index 0000000000..f757b287b7 --- /dev/null +++ b/fdio.infra.ansible/roles/iperf/defaults/main.yaml @@ -0,0 +1,23 @@ +--- +# file: roles/iperf/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + jammy: + - "build-essential" + #- "lib32z1" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +iperf_target_dir: "/opt" +iperf_version: + - "3.7" diff --git a/fdio.infra.ansible/roles/iperf/tasks/main.yaml b/fdio.infra.ansible/roles/iperf/tasks/main.yaml new file mode 100644 index 0000000000..6184ba25f1 --- /dev/null +++ b/fdio.infra.ansible/roles/iperf/tasks/main.yaml @@ -0,0 +1,62 @@ +--- +# file: roles/iperf/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - iperf-inst-prerequisites + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - iperf-inst-prerequisites + +- name: Get Release Archive + ansible.builtin.get_url: + url: "https://downloads.es.net/pub/iperf/iperf-{{ item }}.tar.gz" + dest: "{{ iperf_target_dir }}/iperf-{{ item }}.tar.gz" + validate_certs: false + mode: 0644 + loop: "{{ iperf_version }}" + tags: + - iperf-inst + +- name: Extract Release Archive + ansible.builtin.unarchive: + remote_src: true + src: "{{ iperf_target_dir }}/iperf-{{ item }}.tar.gz" + dest: "{{ iperf_target_dir }}/" + creates: "{{ iperf_target_dir }}/iperf-{{ item }}/src" + loop: "{{ iperf_version }}" + tags: + - iperf-inst + +- name: Compile Release I + ansible.builtin.command: "./configure" + args: + chdir: "{{ iperf_target_dir }}/iperf-{{ item }}/" + loop: "{{ iperf_version }}" + tags: + - iperf-inst + +- name: Compile Release II + ansible.builtin.command: "make" + args: + chdir: "{{ iperf_target_dir }}/iperf-{{ item }}/" + loop: "{{ iperf_version }}" + tags: + - iperf-inst + +- name: Compile Release III + ansible.builtin.command: "make install" + args: + chdir: "{{ iperf_target_dir }}/iperf-{{ item }}/" + loop: "{{ iperf_version }}" + tags: + - iperf-inst diff --git a/fdio.infra.ansible/roles/jenkins_job_health_exporter/defaults/main.yaml b/fdio.infra.ansible/roles/jenkins_job_health_exporter/defaults/main.yaml new file mode 100644 index 0000000000..ab54aac516 --- /dev/null +++ b/fdio.infra.ansible/roles/jenkins_job_health_exporter/defaults/main.yaml @@ -0,0 +1,28 @@ +--- +# file: roles/jenkins_job_health_exporter/defaults/main.yaml + +# Conf - Jenkins Job Health Exporter. +jenkins_host: "jenkins.fd.io" +poll_interval_sec: 1800 +req_timeout_sec: 30 +bind_to: "0.0.0.0:9186" +last_builds: 10 +jobs: + - "vpp-csit-verify-api-crc-master-ubuntu2204-x86_64" + - "vpp-gcc-verify-master-ubuntu2204-x86_64" + - "vpp-verify-master-ubuntu2204-aarch64" + - "vpp-verify-master-ubuntu2204-x86_64" + - "vpp-debug-verify-master-ubuntu2204-x86_64" + - "vpp-checkstyle-verify-master-ubuntu2204-x86_64" + - "vpp-docs-verify-master-ubuntu2204-x86_64" + - "vpp-csit-verify-device-master-ubuntu2204-x86_64-1n-skx" + - "vpp-csit-verify-device-master-ubuntu2204-aarch64-1n-tx2" + +# Conf - Service. +jenkins_job_health_exporter_restart_handler_state: "restarted" + +# Inst - System paths. +jenkins_job_health_exporter_target_dir: "/usr/bin" +jenkins_job_health_exporter_conf_dir: "/etc" +jenkins_job_health_exporter_url: "https://github.com/ayourtch/jenkins-job-health-exporter/releases/download" +jenkins_job_health_exporter_version: "v0.0.3" diff --git a/fdio.infra.ansible/roles/jenkins_job_health_exporter/handlers/main.yaml b/fdio.infra.ansible/roles/jenkins_job_health_exporter/handlers/main.yaml new file mode 100644 index 0000000000..01849fd92e --- /dev/null +++ b/fdio.infra.ansible/roles/jenkins_job_health_exporter/handlers/main.yaml @@ -0,0 +1,9 @@ +--- +# file roles/jenkins_job_health_exporter/handlers/main.yaml + +- name: Restart Jenkins Job Health Exporter + ansible.builtin.systemd: + daemon_reload: true + enabled: true + name: "jenkins-job-health-exporter" + state: "{{ jenkins_job_health_exporter_restart_handler_state }}" diff --git a/fdio.infra.ansible/roles/jenkins_job_health_exporter/tasks/main.yaml b/fdio.infra.ansible/roles/jenkins_job_health_exporter/tasks/main.yaml new file mode 100644 index 0000000000..efdc26a6d2 --- /dev/null +++ b/fdio.infra.ansible/roles/jenkins_job_health_exporter/tasks/main.yaml @@ -0,0 +1,38 @@ +--- +# file: roles/jenkins_job_health_exporter/tasks/main.yaml + +- name: Conf - Jenkins Job Health Exporter Config + ansible.builtin.template: + src: "templates/jenkins-job-health-exporter.j2" + dest: "/etc/jenkins-job-health-exporter.json" + owner: "root" + group: "root" + mode: "0644" + when: + - ansible_hostname == "s22-nomad" + tags: + - conf-jenkins-job-json + +- name: Inst - Jenkins Job Health Exporter Binary + ansible.builtin.get_url: + url: "{{ jenkins_job_health_exporter_url }}/{{ jenkins_job_health_exporter_version }}/jenkins-job-health-exporter" + dest: "{{ jenkins_job_health_exporter_target_dir }}/jenkins-job-health-exporter" + mode: "0755" + when: + - ansible_hostname == "s22-nomad" + tags: + - inst-jenkins-job-binary + +- name: Inst - Jenkins Job Health Exporter Service + ansible.builtin.template: + src: "templates/jenkins-job-health-exporter.service.j2" + dest: "/lib/systemd/system/jenkins-job-health-exporter.service" + owner: "root" + group: "root" + mode: "0644" + when: + - ansible_hostname == "s22-nomad" + notify: + - "Restart Jenkins Job Health Exporter" + tags: + - inst-jenkins-job-service diff --git a/fdio.infra.ansible/roles/jenkins_job_health_exporter/templates/jenkins-job-health-exporter.j2 b/fdio.infra.ansible/roles/jenkins_job_health_exporter/templates/jenkins-job-health-exporter.j2 new file mode 100644 index 0000000000..5942b782e0 --- /dev/null +++ b/fdio.infra.ansible/roles/jenkins_job_health_exporter/templates/jenkins-job-health-exporter.j2 @@ -0,0 +1,16 @@ +{ + "jenkins_host": "{{ jenkins_host }}", + "poll_interval_sec": {{ poll_interval_sec }}, + "req_timeout_sec": {{ req_timeout_sec }}, + "bind_to": "{{ bind_to }}", + "last_builds": {{ last_builds }}, + "jobs": [ +{% for item in jobs %} + "{{ item }}" +{%- if not loop.last %}, +{% endif %} +{% endfor %} + + ], + "verbose": 3 +}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/jenkins_job_health_exporter/templates/jenkins-job-health-exporter.service.j2 b/fdio.infra.ansible/roles/jenkins_job_health_exporter/templates/jenkins-job-health-exporter.service.j2 new file mode 100644 index 0000000000..38073d0a8c --- /dev/null +++ b/fdio.infra.ansible/roles/jenkins_job_health_exporter/templates/jenkins-job-health-exporter.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Jenkins Job Health Exporter +Documentation=https://github.com/ayourtch/jenkins-job-health-exporter + +[Service] +Restart=always +ExecStart={{ jenkins_job_health_exporter_target_dir }}/jenkins-job-health-exporter {{ jenkins_job_health_exporter_conf_dir }}/jenkins-job-health-exporter.json +ExecReload=/bin/kill -HUP $MAINPID +TimeoutStopSec=20s +SendSIGKILL=no + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/kernel/defaults/main.yaml b/fdio.infra.ansible/roles/kernel/defaults/main.yaml new file mode 100644 index 0000000000..ef628c93f5 --- /dev/null +++ b/fdio.infra.ansible/roles/kernel/defaults/main.yaml @@ -0,0 +1,31 @@ +--- +# file: roles/kernel/defaults/main.yaml + +# Kernel version to install (Default to any version). +kernel_version: "{{ kernel_version_by_distro[ansible_distribution|lower][ansible_distribution_release] | join(' ') }}" + +kernel_version_by_distro: + ubuntu: + jammy: + - "5.15.0-46" + +kernel_packages: "{{ kernel_packages_by_distro[ansible_distribution|lower][ansible_distribution_release] | flatten(levels=1) }}" + +kernel_packages_by_distro: + ubuntu: + jammy: + - "linux-image" + - "linux-headers" + - "linux-modules" + - "linux-modules-extra" + - "linux-tools" + +# Packages to remove in relation to kernel upgrade. +absent_packages: "{{ absent_packages_by_distro[ansible_distribution|lower][ansible_distribution_release] | flatten(levels=1) }}" + +absent_packages_by_distro: + ubuntu: + jammy: + - "amd64-microcode" + - "intel-microcode" + - "iucode-tool" diff --git a/fdio.infra.ansible/roles/kernel/filter_plugins/main.py b/fdio.infra.ansible/roles/kernel/filter_plugins/main.py new file mode 100644 index 0000000000..7d909b90e8 --- /dev/null +++ b/fdio.infra.ansible/roles/kernel/filter_plugins/main.py @@ -0,0 +1,143 @@ + +"""Extra Ansible filters""" + +def deb_kernel(packages, kernel_version, current_version): + """ + Return best matching kernel version. + Args: + packages (dict): apt-cache showpkg output. + kernel_version (str): Kernel version to install. + current_version (str): Current kernel version. + Returns: + str: kernel version. + """ + kernels = set() + + # List all available kernel version and associated repository + for line in packages['stdout'].splitlines(): + line = line.strip() + if line.startswith('Package: ') and ( + line.endswith('-common') or # Debian + line.endswith('-generic')): # Ubuntu + kernel = line.split()[1] + + for string in ('linux-headers-', 'common', 'generic'): + kernel = kernel.replace(string, '') + kernel = kernel.strip('-') + + if kernel: + kernels.add(kernel) + + # Sort Kernel versions + versions = {} + for kernel in kernels: + try: + version, build = kernel.split('-', 1) + except ValueError: + version = kernel + build = '' + versions[kernel] = list( + int(ver) for ver in version.split('.')) + [build] + kernels = sorted(versions.keys(), key=versions.get, reverse=True) + + # Return more recent kernel package that match version requirement + for kernel in kernels: + if kernel.startswith(kernel_version): + return kernel + + raise RuntimeError( + 'No kernel matching to "%s". Available kernel versions: %s' % ( + kernel_version, ', '.join(reversed(kernels)))) + + +def _deb_kernel_package(kernel, dist, arch, name): + """ + Return kernel package name. + Args: + kernel (str): Kernel version. + dist (str): Distribution. + arch (str): Architecture. + name (str): Package name. + Returns: + str: kernel package. + """ + # Define package suffix + if dist == 'Ubuntu': + suffix = 'generic' + elif name == 'linux-image': + suffix = arch.replace('x86_64', 'amd64') + else: + suffix = 'common' + + return '-'.join((name, kernel, suffix)) + + +def deb_kernel_pkg(packages, kernel_version, current_version, dist, arch, name): + """ + Return kernel package to install. + Args: + packages (dict): apt-cache showpkg output. + kernel_version (str): Kernel version to install. + current_version (str): Current kernel version. + dist (str): Distribution. + arch (str): Architecture. + name (str): Package name. + Returns: + str: kernel package to install. + """ + return _deb_kernel_package( + deb_kernel(packages, kernel_version, current_version), dist, arch, name) + + +def deb_installed_kernel(installed, packages, kernel_version, current_version): + """ + Return old kernel packages to remove. + Args: + installed (dict): dpkg -l output. + packages (dict): apt-cache showpkg output. + kernel_version (str): Kernel version to install. + current_version (str): Current kernel version. + Returns: + list of str: Kernel packages to remove. + """ + # Filter installed package to keep + to_keep = deb_kernel(packages, kernel_version, current_version) + + # Return installed package to remove + to_remove = [] + for line in installed['stdout'].splitlines(): + if ' linux-' not in line: + continue + + package = line.split()[1] + if ((package.startswith('linux-image-') or + package.startswith('linux-headers-')) and not ( + package.startswith('linux-image-' + to_keep) or + package.startswith('linux-headers-' + to_keep))): + to_remove.append(package) + + return to_remove + + +def kernel_match(kernel, kernel_spec): + """ + Check if kernel version match. + Args: + kernel (str): Kernel + kernel_spec (str): Kernel to match. + Returns: + bool: True if Kernel match. + """ + return kernel.startswith(kernel_spec) + + +class FilterModule(object): + """Return filter plugin""" + + @staticmethod + def filters(): + """Return filter""" + return {'deb_kernel': deb_kernel, + 'deb_kernel_pkg': deb_kernel_pkg, + 'deb_installed_kernel': deb_installed_kernel, + 'kernel_match': kernel_match} diff --git a/fdio.infra.ansible/roles/kernel/handlers/main.yaml b/fdio.infra.ansible/roles/kernel/handlers/main.yaml new file mode 100644 index 0000000000..d0be276a5b --- /dev/null +++ b/fdio.infra.ansible/roles/kernel/handlers/main.yaml @@ -0,0 +1,8 @@ +--- +# file roles/kernel/handlers/main.yaml + +- name: Reboot Server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/kernel/tasks/main.yaml b/fdio.infra.ansible/roles/kernel/tasks/main.yaml new file mode 100644 index 0000000000..431e344fb8 --- /dev/null +++ b/fdio.infra.ansible/roles/kernel/tasks/main.yaml @@ -0,0 +1,9 @@ +--- +# file: roles/kernel/tasks/main.yaml + +- name: Inst - Prerequisites + include_tasks: "{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml" + tags: + - kernel-inst-prerequisites + +- meta: flush_handlers diff --git a/fdio.infra.ansible/roles/kernel/tasks/ubuntu_jammy.yaml b/fdio.infra.ansible/roles/kernel/tasks/ubuntu_jammy.yaml new file mode 100644 index 0000000000..af987d4e5a --- /dev/null +++ b/fdio.infra.ansible/roles/kernel/tasks/ubuntu_jammy.yaml @@ -0,0 +1,62 @@ +--- +# file: roles/kernel/tasks/ubuntu_jammy.yaml + +- name: Get Available Kernel Versions + ansible.builtin.command: "apt-cache showpkg linux-headers-*" + changed_when: false + register: apt_kernel_list + tags: + - kernel-inst + +- name: Get installed packages with APT + ansible.builtin.command: "dpkg -l" + changed_when: false + register: apt_packages_list + tags: + - kernel-inst + +- name: Set target APT kernel version + ansible.builtin.set_fact: + _kernel: "{{ apt_kernel_list | deb_kernel( + kernel_version, ansible_kernel) }}" + tags: + - kernel-inst + +- name: Disable APT auto upgrade + ansible.builtin.lineinfile: + path: "/etc/apt/apt.conf.d/20auto-upgrades" + state: "present" + regexp: "APT::Periodic::Unattended-Upgrade \"[0-9]\";" + line: "APT::Periodic::Unattended-Upgrade \"0\";" + create: true + mode: 0644 + tags: + - kernel-inst + +- name: Ensure Packages Versions + ansible.builtin.apt: + name: "{{ apt_kernel_list | deb_kernel_pkg( + kernel_version, ansible_kernel, ansible_distribution, + ansible_architecture, item) }}" + loop: "{{ kernel_packages }}" + tags: + - kernel-inst + +- name: Ensure Any Other Kernel Packages Are Removed + ansible.builtin.apt: + name: "{{ apt_packages_list | deb_installed_kernel( + apt_kernel_list, kernel_version, ansible_kernel) }}" + state: "absent" + purge: true + notify: + - "Reboot Server" + tags: + - kernel-inst + +- name: Ensure Any Microcode Is Absent + ansible.builtin.apt: + name: "{{ absent_packages }}" + state: "absent" + purge: true + tags: + - kernel-inst diff --git a/fdio.infra.ansible/roles/kernel_vm/files/initramfs_modules b/fdio.infra.ansible/roles/kernel_vm/files/initramfs_modules new file mode 100644 index 0000000000..00ae8e03e7 --- /dev/null +++ b/fdio.infra.ansible/roles/kernel_vm/files/initramfs_modules @@ -0,0 +1,4 @@ +9p +9pnet +9pnet_virtio +vfio-pci
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/kernel_vm/files/initramfs_resume b/fdio.infra.ansible/roles/kernel_vm/files/initramfs_resume new file mode 100644 index 0000000000..820819823b --- /dev/null +++ b/fdio.infra.ansible/roles/kernel_vm/files/initramfs_resume @@ -0,0 +1 @@ +RESUME=none
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/kernel_vm/tasks/main.yaml b/fdio.infra.ansible/roles/kernel_vm/tasks/main.yaml new file mode 100644 index 0000000000..cd8eb15c57 --- /dev/null +++ b/fdio.infra.ansible/roles/kernel_vm/tasks/main.yaml @@ -0,0 +1,92 @@ +--- +# file: roles/kernel_vm/tasks/main.yaml + +- name: Inst - Backup remote initramfs modules + ansible.builtin.copy: + src: "/etc/initramfs-tools/modules" + dest: "/tmp/initramfs_modules.bkp" + remote_src: true + ignore_errors: true + register: __initramfs_modules_backuped + tags: + - kernel-inst-image + +- name: Inst - Backup remote initramfs resume config + ansible.builtin.copy: + src: "/etc/initramfs-tools/conf.d/resume" + dest: "/tmp/initramfs-resume.bkp" + remote_src: true + ignore_errors: true + register: __initramfs_resume_backuped + tags: + - kernel-inst-image + +- name: Inst - Update remote initramfs modules + ansible.builtin.copy: + src: "../files/initramfs_modules" + dest: "/etc/initramfs-tools/modules" + tags: + - kernel-inst-image + +- name: Inst - Update remote initramfs resume config + ansible.builtin.copy: + src: "../files/initramfs_resume" + dest: "/etc/initramfs-tools/conf.d/resume" + tags: + - kernel-inst-image + +- name: Inst - Create target kernel dir + ansible.builtin.file: + path: "/opt/boot" + state: "directory" + tags: + - kernel-inst-image + +- name: Inst - Build initrd image + ansible.builtin.shell: "update-initramfs -k {{ ansible_kernel }} -c -b /opt/boot" + tags: + - kernel-inst-image + +- name: Inst - Copy corresponding kernel img + ansible.builtin.copy: + src: "/boot/vmlinuz-{{ ansible_kernel }}" + dest: "/opt/boot/vmlinuz-{{ ansible_kernel }}" + remote_src: true + tags: + - kernel-inst-image + +- name: Inst - Restore remote initramfs modules + ansible.builtin.copy: + src: "/tmp/initramfs_modules.bkp" + dest: "/etc/initramfs-tools/modules" + remote_src: true + ignore_errors: true + when: __initramfs_modules_backuped + tags: + - kernel-inst-image + +- name: Inst - Remove remote backup initramfs modules + ansible.builtin.file: + path: "/tmp/initramfs_modules.bkp" + state: "absent" + when: __initramfs_modules_backuped + tags: + - kernel-inst-image + +- name: Inst - Restore remote initramfs resume config + ansible.builtin.copy: + src: "/tmp/initramfs-resume.bkp" + dest: "/etc/initramfs-tools/conf.d/resume" + remote_src: true + ignore_errors: true + when: __initramfs_resume_backuped + tags: + - kernel-inst-image + +- name: Inst - Remove remote backup initramfs resume config + ansible.builtin.file: + path: "/tmp/initramfs-resume.bkp" + state: "absent" + when: __initramfs_resume_backuped + tags: + - kernel-inst-image diff --git a/fdio.infra.ansible/roles/mellanox/defaults/main.yaml b/fdio.infra.ansible/roles/mellanox/defaults/main.yaml new file mode 100644 index 0000000000..de66be2d6b --- /dev/null +++ b/fdio.infra.ansible/roles/mellanox/defaults/main.yaml @@ -0,0 +1,30 @@ +--- +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "build-essential" + - "libnl-3-dev" + - "libnl-route-3-dev" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +mellanox_download_url: "http://content.mellanox.com/ofed" +mellanox_extract_dir: "/opt" + +mellanox_compatibility_matrix: + dpdk22.07: + # https://doc.dpdk.org/guides/rel_notes/release_22_07.html + ofed: "5.9-0.5.6.0" + dpdk23.11: + # https://doc.dpdk.org/guides/rel_notes/release_23_11.html + ofed: "23.07-0.5.0.0" diff --git a/fdio.infra.ansible/roles/mellanox/tasks/main.yaml b/fdio.infra.ansible/roles/mellanox/tasks/main.yaml new file mode 100644 index 0000000000..53376eb997 --- /dev/null +++ b/fdio.infra.ansible/roles/mellanox/tasks/main.yaml @@ -0,0 +1,43 @@ +--- +# file: tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - mellanox-inst-drivers + +- name: Inst - Prerequisites + package: + name: "{{ packages | flatten(levels=1) }}" + state: "latest" + tags: + - mellanox-inst-drivers + +- name: Inst - Check Presence of Mellanox + shell: "lspci | grep Mellanox | awk '{print $1}'" + register: mellanox_pcis + failed_when: false + changed_when: false + tags: + - mellanox-inst-drivers + +- name: Inst - Get Mellanox OFED driver versions + set_fact: + ofed: "{{ mellanox_compatibility_matrix[mellanox_matrix]['ofed'] }}" + when: > + mellanox_pcis.stdout_lines | length > 0 and + mellanox_matrix is defined + tags: + - mellanox-inst-drivers + +- name: Inst - Driver Mellanox + import_tasks: ofed.yaml + when: > + mellanox_pcis.stdout_lines | length > 0 and + mellanox_matrix is defined + tags: + - mellanox-inst-drivers diff --git a/fdio.infra.ansible/roles/mellanox/tasks/ofed.yaml b/fdio.infra.ansible/roles/mellanox/tasks/ofed.yaml new file mode 100644 index 0000000000..c39975bab6 --- /dev/null +++ b/fdio.infra.ansible/roles/mellanox/tasks/ofed.yaml @@ -0,0 +1,37 @@ +--- +# file: tasks/ofed.yaml + +- name: Inst - Get OFED + ansible.builtin.get_url: + url: "{{ mellanox_download_url }}/MLNX_OFED-{{ ofed }}/MLNX_OFED_LINUX-{{ ofed }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz" + dest: "{{ mellanox_extract_dir }}/MLNX_OFED_LINUX-{{ ofed }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz" + mode: "0644" + when: mellanox_pcis.stdout_lines | length > 0 + tags: + - mellanox-inst-drivers + +- name: Inst - Extract OFED + ansible.builtin.unarchive: + remote_src: true + src: "{{ mellanox_extract_dir }}/MLNX_OFED_LINUX-{{ ofed }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz" + dest: "{{ mellanox_extract_dir }}/" + creates: "{{ mellanox_extract_dir }}/MLNX_OFED_LINUX-{{ ofed }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}" + register: mellanox_firmware_extracted + tags: + - mellanox-inst-drivers + +- name: Inst - OFED + ansible.builtin.command: "./mlnxofedinstall --with-mft --dpdk --force --upstream-libs" # --without-fw-update" + args: + chdir: "{{ mellanox_extract_dir }}/MLNX_OFED_LINUX-{{ ofed }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}" + when: mellanox_firmware_extracted + tags: + - mellanox-inst-drivers + +- name: Inst - Switch Infiniband to Ethernet + ansible.builtin.command: "mlxconfig --yes --dev {{ item }} set LINK_TYPE_P1=2 LINK_TYPE_P2=2" + with_items: "{{ mellanox_pcis.stdout_lines }}" + failed_when: false + changed_when: false + tags: + - mellanox-inst-drivers
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/defaults/main.yaml b/fdio.infra.ansible/roles/nomad/defaults/main.yaml new file mode 100644 index 0000000000..535db2bb2c --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/defaults/main.yaml @@ -0,0 +1,193 @@ +--- +# file: roles/nomad/defaults/main.yaml + +# Prerequisites +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" +packages_base: + - "curl" + - "unzip" +packages_by_distro: + ubuntu: + - [] +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +# Package +nomad_version: "1.4.3" +nomad_architecture_map: + amd64: "amd64" + x86_64: "amd64" + armv7l: "arm" + aarch64: "arm64" + 32-bit: "386" + 64-bit: "amd64" +nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}" +nomad_pkg: "nomad_{{ nomad_version }}_linux_{{nomad_architecture}}.zip" +nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version }}_linux_{{nomad_architecture}}.zip" +nomad_checksum_file_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version}}_SHA256SUMS" +nomad_force_update: false + +# Paths +nomad_inst_dir: "/opt" +nomad_bin_dir: "/usr/local/bin" +nomad_config_dir: "/etc/nomad.d" +nomad_data_dir: "/var/nomad" +nomad_plugin_dir: "{{ nomad_data_dir }}/plugins" +nomad_lockfile: "/var/lock/subsys/nomad" +nomad_run_dir: "/var/run/nomad" +nomad_ssl_dir: "/etc/nomad.d/ssl" + +# Initialization and startup script templates +nomad_service_mgr: "" + +# System user and group +nomad_group: "nomad" +nomad_user: "nomad" + +# Nomad settings +nomad_datacenter: "dc1" +nomad_region: "global" +nomad_log_level: "INFO" +nomad_syslog_enable: true +nomad_iface: "{{ ansible_default_ipv4.interface }}" +nomad_node_name: "{{ inventory_hostname }}" +nomad_node_role: "server" +nomad_leave_on_terminate: true +nomad_leave_on_interrupt: false +nomad_disable_update_check: true +nomad_enable_debug: false + +# Server settings +nomad_bootstrap_expect: 2 +nomad_encrypt: "" +nomad_retry_join: true +# Specifies how long a node must be in a terminal state before it is garbage +# collected and purged from the system. +nomad_node_gc_threshold: "24h" +# Specifies the interval between the job garbage collections. Only jobs who have +# been terminal for at least job_gc_threshold will be collected. +nomad_job_gc_interval: "10m" +# Specifies the minimum time a job must be in the terminal state before it is +# eligible for garbage collection. +nomad_job_gc_threshold: "4h" +# Specifies the minimum time an evaluation must be in the terminal state before +# it is eligible for garbage collection. +nomad_eval_gc_threshold: "1h" +# Specifies the minimum time a deployment must be in the terminal state before +# it is eligible for garbage collection. +nomad_deployment_gc_threshold: "1h" +nomad_encrypt_enable: false +nomad_raft_protocol: 2 + +# Client settings +nomad_node_class: "compute" +nomad_no_host_uuid: true +nomad_max_kill_timeout: "30s" +nomad_gc_interval: "1m" +nomad_gc_disk_usage_threshold: 80 +nomad_gc_inode_usage_threshold: 70 +nomad_gc_parallel_destroys: 2 +nomad_reserved: + cpu: "{{ nomad_reserved_cpu | default('0', true) }}" + memory: "{{ nomad_reserved_memory | default('0', true) }}" + disk: "{{ nomad_reserved_disk | default('0', true) }}" + ports: "{{ nomad_reserved_ports | default('22', true) }}" +nomad_volumes: [] +nomad_options: {} +nomad_meta: {} +nomad_chroot_env: false +nomad_plugins: {} + +# Addresses +nomad_bind_address: "{{ hostvars[inventory_hostname]['ansible_'+ nomad_iface ]['ipv4']['address'] }}" +nomad_advertise_address: "{{ hostvars[inventory_hostname]['ansible_' + nomad_iface]['ipv4']['address'] }}" + +# Ports +nomad_ports: + http: "{{ nomad_ports_http | default('4646', true) }}" + rpc: "{{ nomad_ports_rpc | default('4647', true) }}" + serf: "{{ nomad_ports_serf | default('4648', true) }}" + +# Servers +nomad_group_name: "nomad" +nomad_servers: "\ + {% if nomad_use_consul==false %}\ + {% set _nomad_servers = [] %}\ + {% for host in groups[nomad_group_name] %}\ + {% set _nomad_node_role = hostvars[host]['nomad_node_role'] | default('client', true) %}\ + {% if (_nomad_node_role == 'server' or _nomad_node_role == 'both') %}\ + {% if _nomad_servers.append(host) %}{% endif %}\ + {% endif %}\ + {% endfor %}\ + {{ _nomad_servers }}\ + {% else %}\ + []\ + {% endif %}" +nomad_gather_server_facts: false + +# Consul +nomad_use_consul: true +nomad_consul_address: "localhost:8500" +nomad_consul_token: "" +nomad_consul_servers_service_name: "nomad" +nomad_consul_clients_service_name: "nomad-client" +nomad_consul_tags: {} +nomad_consul_use_ssl: false + +# ACLs +nomad_acl_enabled: false +nomad_acl_token_ttl: "30s" +nomad_acl_policy_ttl: "30s" +nomad_acl_replication_token: "" + +# Docker +nomad_docker_enable: false +nomad_docker_dmsetup: true + +# Autopilot +nomad_autopilot_cleanup_dead_servers: true +nomad_autopilot_last_contact_threshold: "200ms" +nomad_autopilot_max_trailing_logs: 250 +nomad_autopilot_server_stabilization_time: "10s" + +# Telemetry. +nomad_use_telemetry: true +nomad_telemetry_disable_hostname: false +nomad_telemetry_collection_interval: "1s" +nomad_telemetry_use_node_name: false +nomad_telemetry_publish_allocation_metrics: true +nomad_telemetry_publish_node_metrics: true +nomad_telemetry_prometheus_metrics: true + +# TLS. +nomad_use_tls: true +nomad_tls_ca_file: "{{ nomad_ssl_dir }}/nomad-ca.pem" +nomad_tls_cert_file: "{{ nomad_ssl_dir }}/nomad.pem" +nomad_tls_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem" +nomad_tls_cli_cert_file: "{{ nomad_ssl_dir }}/nomad-cli.pem" +nomad_tls_cli_key_file: "{{ nomad_ssl_dir }}/nomad-cli-key.pem" +nomad_tls_http: false +nomad_tls_rpc: false +nomad_tls_rpc_upgrade_mode: false +nomad_tls_verify_https_client: false +nomad_tls_verify_server_hostname: false + +# Vault +nomad_use_vault: false +nomad_vault_address: "http://vault.service.consul:8200" +nomad_vault_allow_unauthenticated: true +nomad_vault_enabled: false +nomad_vault_create_from_role: "" +nomad_vault_task_token_ttl: "72h" +nomad_vault_use_ssl: false +nomad_vault_ca_file: "" +nomad_vault_ca_path: "" +nomad_vault_cert_file: "" +nomad_vault_key_file: "" +nomad_vault_namespace: "" +nomad_vault_tls_server_name: "" +nomad_vault_tls_skip_verify: false +nomad_vault_token: "" diff --git a/fdio.infra.ansible/roles/nomad/handlers/main.yaml b/fdio.infra.ansible/roles/nomad/handlers/main.yaml new file mode 100644 index 0000000000..32e5798e3e --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/handlers/main.yaml @@ -0,0 +1,9 @@ +--- +# file handlers/main.yaml + +- name: Restart Nomad + ansible.builtin.systemd: + daemon_reload: true + enabled: true + name: "nomad" + state: "restarted" diff --git a/fdio.infra.ansible/roles/nomad/meta/main.yaml b/fdio.infra.ansible/roles/nomad/meta/main.yaml new file mode 100644 index 0000000000..098aafe2fb --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/meta/main.yaml @@ -0,0 +1,21 @@ +--- +# file: meta/main.yaml + +dependencies: ["docker"] + +galaxy_info: + role_name: "nomad" + author: "pmikus" + description: "Hashicorp Nomad." + company: "none" + license: "license (Apache)" + min_ansible_version: "2.9" + platforms: + - name: "Ubuntu" + release: + - "focal" + - "jammy" + - "kinetic" + galaxy_tags: + - "nomad" + - "hashicorp" diff --git a/fdio.infra.ansible/roles/nomad/tasks/main.yaml b/fdio.infra.ansible/roles/nomad/tasks/main.yaml new file mode 100644 index 0000000000..72b78458f8 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/tasks/main.yaml @@ -0,0 +1,151 @@ +--- +# file: tasks/main.yaml + +- name: Update Repositories Cache + ansible.builtin.apt: + update_cache: true + when: + - ansible_os_family == 'Debian' + tags: + - nomad-inst-package + +- name: Dependencies + ansible.builtin.apt: + name: "{{ packages | flatten(levels=1) }}" + state: "present" + cache_valid_time: 3600 + install_recommends: false + when: + - ansible_os_family == 'Debian' + tags: + - nomad-inst-dependencies + +- name: Add Nomad Group + ansible.builtin.group: + name: "{{ nomad_group }}" + state: "present" + tags: + - nomad-conf-user + +- name: Add Nomad user + ansible.builtin.user: + name: "{{ nomad_user }}" + group: "{{ nomad_group }}" + state: "present" + system: true + tags: + - nomad-conf-user + +- name: Download Nomad + ansible.builtin.get_url: + url: "{{ nomad_zip_url }}" + dest: "{{ nomad_inst_dir }}/{{ nomad_pkg }}" + mode: 0644 + tags: + - nomad-inst-package + +- name: Clean Nomad + ansible.builtin.file: + path: "{{ nomad_inst_dir }}/nomad" + state: "absent" + when: + - nomad_force_update | bool + tags: + - nomad-inst-package + +- name: Unarchive Nomad + ansible.builtin.unarchive: + src: "{{ nomad_inst_dir }}/{{ nomad_pkg }}" + dest: "{{ nomad_inst_dir }}/" + remote_src: true + tags: + - nomad-inst-package + +- name: Nomad + ansible.builtin.copy: + src: "{{ nomad_inst_dir }}/nomad" + dest: "{{ nomad_bin_dir }}" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + force: true + mode: 0755 + remote_src: true + tags: + - nomad-inst-package + +- name: Create Directories + ansible.builtin.file: + dest: "{{ item }}" + state: "directory" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0755 + with_items: + - "{{ nomad_data_dir }}" + - "{{ nomad_config_dir }}" + - "{{ nomad_ssl_dir }}" + tags: + - nomad-conf + +- name: Base Configuration + ansible.builtin.template: + src: "{{ item }}.hcl.j2" + dest: "{{ nomad_config_dir }}/{{ item }}.hcl" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0644 + with_items: + - "base" + - "consul" + - "client" + - "server" + - "telemetry" + - "tls" + - "vault" + tags: + - nomad-conf + +- name: Conf - Copy Certificates And Keys + ansible.builtin.copy: + content: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0600 + no_log: true + loop: "{{ nomad_certificates | flatten(levels=1) }}" + when: + - nomad_certificates is defined + tags: + - nomad-conf + +- name: Nomad CLI Environment Variables + ansible.builtin.lineinfile: + path: "/etc/profile.d/nomad.sh" + line: "{{ item }}" + mode: 0644 + create: true + loop: + - "export NOMAD_ADDR=https://nomad-server.service.consul:4646" + - "export NOMAD_CACERT={{ nomad_tls_ca_file }}" + - "export NOMAD_CLIENT_CERT={{ nomad_tls_cli_cert_file }}" + - "export NOMAD_CLIENT_KEY={{ nomad_tls_cli_key_file }}" + tags: + - nomad-conf + +- name: System.d Script + ansible.builtin.template: + src: "nomad_systemd.service.j2" + dest: "/lib/systemd/system/nomad.service" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Restart Nomad" + when: + - nomad_service_mgr == "systemd" + tags: + - nomad-conf + +- name: Meta - Flush handlers + ansible.builtin.meta: flush_handlers diff --git a/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 new file mode 100644 index 0000000000..cd7fb54f9c --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 @@ -0,0 +1,26 @@ +name = "{{ nomad_node_name }}" +region = "{{ nomad_region }}" +datacenter = "{{ nomad_datacenter }}" + +enable_debug = {{ nomad_enable_debug | bool | lower }} +disable_update_check = {{ nomad_disable_update_check | bool | lower }} + +bind_addr = "{{ nomad_bind_address }}" +advertise { + http = "{{ nomad_advertise_address }}:{{ nomad_ports.http }}" + rpc = "{{ nomad_advertise_address }}:{{ nomad_ports.rpc }}" + serf = "{{ nomad_advertise_address }}:{{ nomad_ports.serf }}" +} +ports { + http = {{ nomad_ports['http'] }} + rpc = {{ nomad_ports['rpc'] }} + serf = {{ nomad_ports['serf'] }} +} + +data_dir = "{{ nomad_data_dir }}" + +log_level = "{{ nomad_log_level }}" +enable_syslog = {{ nomad_syslog_enable | bool | lower }} + +leave_on_terminate = {{ nomad_leave_on_terminate | bool | lower }} +leave_on_interrupt = {{ nomad_leave_on_interrupt | bool | lower }} diff --git a/fdio.infra.ansible/roles/nomad/templates/cfssl.json b/fdio.infra.ansible/roles/nomad/templates/cfssl.json new file mode 100644 index 0000000000..2b603e9b84 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/cfssl.json @@ -0,0 +1,8 @@ +{ + "signing": { + "default": { + "expiry": "87600h", + "usages": ["signing", "key encipherment", "server auth", "client auth"] + } + } +}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 new file mode 100644 index 0000000000..f82f38a4e4 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 @@ -0,0 +1,60 @@ +client { + enabled = {{ nomad_node_client | bool | lower }} + + node_class = "{{ nomad_node_class }}" + no_host_uuid = {{ nomad_no_host_uuid | bool | lower }} + +{% if nomad_use_consul == False %} + {% if nomad_servers -%} + servers = [ {% for ip_port in nomad_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ] + {% endif -%} +{% endif %} + + {% if nomad_network_interface is defined -%} + network_interface = "{{ nomad_network_interface }}" + {% endif -%} + {% if nomad_network_speed is defined -%} + network_speed = "{{ nomad_network_speed }}" + {% endif -%} + {% if nomad_cpu_total_compute is defined -%} + cpu_total_compute = {{ nomad_cpu_total_compute }} + {% endif -%} + + reserved { + cpu = {{ nomad_reserved['cpu'] }} + memory = {{ nomad_reserved['memory'] }} + disk = {{ nomad_reserved['disk'] }} + } + + {% for nomad_host_volume in nomad_volumes -%} + host_volume "{{ nomad_host_volume.name }}" { + path = "{{ nomad_host_volume.path }}" + read_only = {{ nomad_host_volume.read_only | bool | lower }} + } + {% endfor %} + + {% if nomad_chroot_env != False -%} + chroot_env = { + {% for key, value in nomad_chroot_env.items() %} + "{{ key }}" = "{{ value }}" + {% endfor -%} + } + {% endif %} + + {% if nomad_options -%} + options = { + {% for key, value in nomad_options.items() %} + "{{ key }}" = "{{ value }}" + {% endfor -%} + } + {% endif %} + + {% if nomad_meta -%} + meta = { + {% for key, value in nomad_meta.items() %} + "{{ key }}" = "{{ value }}" + {% endfor -%} + } + {% endif %} + +} diff --git a/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 new file mode 100644 index 0000000000..a9c1aff7b2 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 @@ -0,0 +1,63 @@ +{% if nomad_use_consul | bool == True %} +consul { + # Specifies the address to the local Consul agent, given in the format + # host:port. + address = "{{ nomad_consul_address }}" + + # Specifies if Nomad should advertise its services in Consul. The services + # are named according to server_service_name and client_service_name. Nomad + # servers and clients advertise their respective services, each tagged + # appropriately with either http or rpc tag. Nomad servers also advertise a + # serf tagged service. + auto_advertise = true + + # Specifies if the Nomad clients should automatically discover servers in + # the same region by searching for the Consul service name defined in the + # server_service_name option. The search occurs if the client is not + # registered with any servers or it is unable to heartbeat to the leader of + # the region, in which case it may be partitioned and searches for other + # servers. + client_auto_join = true + + # Specifies the name of the service in Consul for the Nomad clients. + client_service_name = "{{ nomad_consul_clients_service_name }}" + + # Specifies the name of the service in Consul for the Nomad servers. + server_service_name = "{{ nomad_consul_servers_service_name }}" + + # Specifies if the Nomad servers should automatically discover and join + # other Nomad servers by searching for the Consul service name defined in + # the server_service_name option. This search only happens if the server + # does not have a leader. + server_auto_join = true + + # Specifies optional Consul tags to be registered with the Nomad server and + # agent services. + tags = {{ nomad_consul_tags | to_json }} + + # Specifies the token used to provide a per-request ACL token. This option + # overrides the Consul Agent's default token. If the token is not set here + # or on the Consul agent, it will default to Consul's anonymous policy, + # which may or may not allow writes. + token = "{{ nomad_consul_token }}" + + {% if nomad_consul_use_ssl | bool == True -%} + # Specifies if the transport scheme should use HTTPS to communicate with the + # Consul agent. + ssl = true + + # Specifies an optional path to the CA certificate used for Consul + # communication. This defaults to the system bundle if unspecified. + ca_file = "{{ nomad_ca_file }}" + + # Specifies the path to the certificate used for Consul communication. If + # this is set then you need to also set key_file. + cert_file = "{{ nomad_cert_file }}" + + # Specifies the path to the private key used for Consul communication. If + # this is set then you need to also set cert_file. + key_file = "{{ nomad_key_file }}" + {% endif %} + +} +{% endif %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2 b/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2 new file mode 100644 index 0000000000..564505781b --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=Nomad +Documentation=https://nomadproject.io/docs/ +Wants=network-online.target +After=network-online.target + +# When using Nomad with Consul it is not necessary to start Consul first. These +# lines start Consul before Nomad as an optimization to avoid Nomad logging +# that Consul is unavailable at startup. +#Wants=consul.service +#After=consul.service + +[Service] +# Nomad server should be run as the nomad user. Nomad clients +# should be run as root +#User=nomad +#Group=nomad + +ExecReload=/bin/kill -HUP $MAINPID +ExecStart={{ nomad_bin_dir }}/nomad agent -config {{ nomad_config_dir }} +KillMode=process +KillSignal=SIGINT +LimitNOFILE=infinity +LimitNPROC=infinity +Restart=on-failure +RestartSec=2 +StartLimitBurst=3 +StartLimitInterval=10 +TasksMax=infinity +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 new file mode 100644 index 0000000000..e19dea9e6f --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 @@ -0,0 +1,58 @@ +{% if nomad_node_server | bool == True %} +server { + enabled = {{ nomad_node_server | bool | lower }} + + {% if nomad_node_server | bool -%} + bootstrap_expect = {{ nomad_bootstrap_expect }} + {%- endif %} + + {% if nomad_node_server | bool -%} + raft_protocol = 3 + {%- endif %} + + {% if nomad_authoritative_region is defined %} + authoritative_region = "{{ nomad_authoritative_region }}" + {% endif %} + +{% if nomad_use_consul == False %} + {% if nomad_retry_join | bool -%} + retry_join = [ + {%- set comma = joiner(",") -%} + {% for server in nomad_servers -%} + {{ comma() }}"{{ hostvars[server]['nomad_advertise_address'] | ipwrap }}" + {%- endfor -%} ] + retry_max = {{ nomad_retry_max }} + retry_interval = "{{ nomad_retry_interval }}" + {% else -%} + start_join = [ + {%- set comma = joiner(",") -%} + {% for server in nomad_servers -%} + {{ comma() }}"{{ hostvars[server]['nomad_advertise_address'] | ipwrap }}" + {%- endfor -%} ] + {%- endif %} +{% endif %} + + encrypt = "{{ nomad_encrypt | default('') }}" + + {% if nomad_node_gc_threshold -%} + node_gc_threshold = "{{ nomad_node_gc_threshold }}" + {%- endif %} + + {% if nomad_job_gc_interval -%} + job_gc_interval = "{{ nomad_job_gc_interval }}" + {%- endif %} + + {% if nomad_job_gc_threshold -%} + job_gc_threshold = "{{ nomad_job_gc_threshold }}" + {%- endif %} + + {% if nomad_eval_gc_threshold -%} + eval_gc_threshold = "{{ nomad_eval_gc_threshold }}" + {%- endif %} + + {% if nomad_deployment_gc_threshold -%} + deployment_gc_threshold = "{{ nomad_deployment_gc_threshold }}" + {%- endif %} + +} +{% endif %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 new file mode 100644 index 0000000000..14be0d9548 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 @@ -0,0 +1,26 @@ +{% if nomad_use_telemetry | bool == True %} +telemetry { + # Specifies if gauge values should be prefixed with the local hostname. + disable_hostname = {{ nomad_telemetry_disable_hostname | bool | lower }} + + # Specifies the time interval at which the Nomad agent collects telemetry + # data. + collection_interval = "{{ nomad_telemetry_collection_interval }}" + + # Specifies if gauge values should be prefixed with the name of the node, + # instead of the hostname. If set it will override disable_hostname value. + use_node_name = {{ nomad_telemetry_use_node_name | bool | lower }} + + # Specifies if Nomad should publish runtime metrics of allocations. + publish_allocation_metrics = {{ nomad_telemetry_publish_allocation_metrics | bool | lower }} + + # Specifies if Nomad should publish runtime metrics of nodes. + publish_node_metrics = {{ nomad_telemetry_publish_node_metrics | bool | lower }} + + # Specifies whether the agent should make Prometheus formatted metrics + # available at /v1/metrics?format=prometheus.Specifies whether the agent + # should make Prometheus formatted metrics available at + # /v1/metrics?format=prometheus. + prometheus_metrics = {{ nomad_telemetry_prometheus_metrics | bool | lower }} +} +{% endif %} diff --git a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 new file mode 100644 index 0000000000..0a1a5b20a4 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 @@ -0,0 +1,36 @@ +{% if nomad_use_tls | bool %} +tls { + # Specifies the path to the CA certificate to use for Nomad's TLS + # communication. + ca_file = "{{ nomad_tls_ca_file }}" + + # Specifies the path to the certificate file used for Nomad's TLS + # communication. + cert_file = "{{ nomad_tls_cert_file }}" + + # Specifies the path to the key file to use for Nomad's TLS communication. + key_file = "{{ nomad_tls_key_file }}" + + # Specifies if TLS should be enabled on the HTTP endpoints on the Nomad + # agent, including the API. + http = {{ nomad_tls_http | bool | lower }} + + # Specifies if TLS should be enabled on the RPC endpoints and Raft traffic + # between the Nomad servers. Enabling this on a Nomad client makes the + # client use TLS for making RPC requests to the Nomad servers. + rpc = {{ nomad_tls_rpc | bool | lower }} + + # This option should be used only when the cluster is being upgraded to + # TLS, and removed after the migration is complete. This allows the agent + # to accept both TLS and plaintext traffic. + rpc_upgrade_mode = {{ nomad_tls_rpc_upgrade_mode | bool | lower }} + + # Specifies agents should require client certificates for all incoming + # HTTPS requests. The client certificates must be signed by the same CA + # as Nomad. + verify_https_client = {{ nomad_tls_verify_https_client | bool | lower }} + + # Specifies if outgoing TLS connections should verify the server's hostname. + verify_server_hostname = {{ nomad_tls_verify_server_hostname | bool | lower }} +} +{% endif %} diff --git a/fdio.infra.ansible/roles/nomad/templates/vault.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/vault.hcl.j2 new file mode 100644 index 0000000000..7911cbc5c4 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/templates/vault.hcl.j2 @@ -0,0 +1,69 @@ +{% if nomad_use_tls | bool == True %} +vault { + # Specifies the address to the Vault server. This must include the protocol, + # host/ip, and port given in the format protocol://host:port. If your Vault + # installation is behind a load balancer, this should be the address of the + # load balancer. + address = "{{ nomad_vault_address }}" + + # Specifies if users submitting jobs to the Nomad server should be required + # to provide their own Vault token, proving they have access to the policies + # listed in the job. This option should be disabled in an untrusted + # environment. + allow_unauthenticated = {{ nomad_vault_allow_unauthenticated | bool | lower }} + + # Specifies if the Vault integration should be activated. + enabled = {{ nomad_vault_enabled | bool | lower }} + + # Specifies the role to create tokens from. The token given to Nomad does + # not have to be created from this role but must have "update" capability + # on "auth/token/create/<create_from_role>" path in Vault. If this value is + # unset and the token is created from a role, the value is defaulted to the + # role the token is from. This is largely for backwards compatibility. It + # is recommended to set the create_from_role field if Nomad is deriving + # child tokens from a role. + create_from_role = "{{ nomad_vault_create_from_role }}" + + # Specifies the TTL of created tokens when using a root token. This is + # specified using a label suffix like "30s" or "1h". + task_token_ttl = "{{ nomad_vault_task_token_ttl }}" + + {% if nomad_vault_use_ssl | bool == True -%} + # Specifies an optional path to the CA certificate used for Vault + # communication. If unspecified, this will fallback to the default system + # CA bundle, which varies by OS and version. + ca_file = "{{ nomad_vault_ca_file }}" + + # Specifies an optional path to a folder containing CA certificates to be + # used for Vault communication. If unspecified, this will fallback to the + # default system CA bundle, which varies by OS and version. + ca_path = "{{ nomad_vault_ca_path }}" + + # Specifies the path to the certificate used for Vault communication. This + # must be set if tls_require_and_verify_client_cert is enabled in Vault. + cert_file = "{{ nomad_vault_cert_file }}" + + # Specifies the path to the private key used for Vault communication. If + # this is set then you need to also set cert_file. This must be set if + # tls_require_and_verify_client_cert is enabled in Vault. + key_file = "{{ nomad_vault_key_file }}" + + # Specifies the Vault namespace used by the Vault integration. If non-empty, + # this namespace will be used on all Vault API calls. + namespace = "{{ nomad_vault_namespace }}" + + # Specifies an optional string used to set the SNI host when connecting to + # Vault via TLS. + tls_server_name = "{{ nomad_vault_tls_server_name }}" + + # Specifies if SSL peer validation should be enforced. + tls_skip_verify = {{ nomad_vault_tls_skip_verify | bool | lower }} + {% endif %} + + # Specifies the parent Vault token to use to derive child tokens for jobs + # requesting tokens. Only required on Nomad servers. Nomad client agents + # use the allocation's token when contacting Vault. Visit the Vault + # Integration Guide to see how to generate an appropriate token in Vault. + token = "{{ nomad_vault_token }}" +} +{% endif %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/nomad/vars/main.yaml b/fdio.infra.ansible/roles/nomad/vars/main.yaml new file mode 100644 index 0000000000..791eeadb06 --- /dev/null +++ b/fdio.infra.ansible/roles/nomad/vars/main.yaml @@ -0,0 +1,5 @@ +--- +# file: vars/main.yaml + +nomad_node_client: "{{ (nomad_node_role == 'client') or (nomad_node_role == 'both') }}" +nomad_node_server: "{{ (nomad_node_role == 'server') or (nomad_node_role == 'both') }}" diff --git a/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml b/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml new file mode 100644 index 0000000000..5a732e5539 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/defaults/main.yaml @@ -0,0 +1,18 @@ +--- +# file: roles/performance_tuning/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - "cpufrequtils" + +packages_by_distro: + ubuntu: + jammy: + - [] + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] diff --git a/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils b/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils new file mode 100644 index 0000000000..03070fefe1 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/files/cpufrequtils @@ -0,0 +1 @@ +GOVERNOR="performance" diff --git a/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service b/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service new file mode 100644 index 0000000000..e04729de50 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/files/disable-turbo-boost.service @@ -0,0 +1,10 @@ +[Unit] +Description=Disable Turbo Boost on Intel CPU + +[Service] +ExecStart=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x4000850089; done' +ExecStop=/bin/sh -c 'for core in `cat /proc/cpuinfo | grep processor | awk \'{print $3}\'`; do sudo wrmsr -p$core 0x1a0 0x850089; done' +RemainAfterExit=yes + +[Install] +WantedBy=sysinit.target diff --git a/fdio.infra.ansible/roles/performance_tuning/files/irqbalance b/fdio.infra.ansible/roles/performance_tuning/files/irqbalance new file mode 100644 index 0000000000..861be02fb3 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/files/irqbalance @@ -0,0 +1,25 @@ +# irqbalance is a daemon process that distributes interrupts across +# CPUS on SMP systems. The default is to rebalance once every 10 +# seconds. This is the environment file that is specified to systemd via the +# EnvironmentFile key in the service unit file (or via whatever method the init +# system you're using has. +# +# ONESHOT=yes +# after starting, wait for a minute, then look at the interrupt +# load and balance it once; after balancing exit and do not change +# it again. +#IRQBALANCE_ONESHOT= + +# +# IRQBALANCE_BANNED_CPUS +# 64 bit bitmask which allows you to indicate which cpu's should +# be skipped when reblancing irqs. Cpu numbers which have their +# corresponding bits set to one in this mask will not have any +# irq's assigned to them on rebalance +# +IRQBALANCE_BANNED_CPUS="{{ ansible_processor_cores | irqbalance_banned_cpu_mask(ansible_processor_count, ansible_processor_threads_per_core) }}" +# +# IRQBALANCE_ARGS +# append any args here to the irqbalance daemon as documented in the man page +# +#IRQBALANCE_ARGS= diff --git a/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py b/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py new file mode 100644 index 0000000000..d76f6fe166 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/filter_plugins/main.py @@ -0,0 +1,29 @@ +"""Extra Ansible filters""" + +def irqbalance_banned_cpu_mask( + processor_cores, processor_count, processor_threads_per_core): + """ + Return irqbalance CPU mask. + Args: + processor_cores (int): Physical processor unit. + processor_counts (int): Processors per physical unit. + processor_threads_per_core (int): Threads per physical unit. + Returns: + str: irqbalance_banned_cpus. + """ + mask = int("1" * 128, 2) + + for i in range(processor_count * processor_threads_per_core): + mask &= ~(1 << i * processor_cores) + + import re + return ",".join(re.findall('.{1,8}', str(hex(mask))[2:])) + + +class FilterModule(object): + """Return filter plugin""" + + @staticmethod + def filters(): + """Return filter""" + return {'irqbalance_banned_cpu_mask': irqbalance_banned_cpu_mask} diff --git a/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml b/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml new file mode 100644 index 0000000000..5a48fc37b4 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/handlers/main.yaml @@ -0,0 +1,13 @@ +--- +# file handlers/main.yaml + +- name: Update GRUB + ansible.builtin.command: update-grub + tags: + - update-grub + +- name: Reboot Server + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml b/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml new file mode 100644 index 0000000000..cc904e23e9 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/tasks/main.yaml @@ -0,0 +1,210 @@ +--- +# file: roles/performance_tuning/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - perf-inst-prerequisites + +- name: Inst - Machine Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - perf-inst-prerequisites + +- name: Conf - Turbo Boost + import_tasks: turbo_boost.yaml + when: > + cpu_microarchitecture == "skylake" or + cpu_microarchitecture == "cascadelake" or + cpu_microarchitecture == "icelake" or + cpu_microarchitecture == "sapphirerapids" + tags: + - perf-conf-turbo-boost + +- name: Conf - Adjust max_map_count + # this file contains the maximum number of memory map areas a process + # may have. memory map areas are used as a side-effect of calling + # malloc, directly by mmap and mprotect, and also when loading shared + # libraries. + # + # while most applications need less than a thousand maps, certain + # programs, particularly malloc debuggers, may consume lots of them, + # e.g., up to one or two maps per allocation. + # must be greater than or equal to (2 * vm.nr_hugepages). + ansible.builtin.sysctl: + name: "vm.max_map_count" + value: "{{ sysctl.vm.nr_hugepages * 4 }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust hugetlb_shm_group + # hugetlb_shm_group contains group id that is allowed to create sysv + # shared memory segment using hugetlb page. + ansible.builtin.sysctl: + name: "vm.hugetlb_shm_group" + value: "1000" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust swappiness + # this control is used to define how aggressive the kernel will swap + # memory pages. higher values will increase agressiveness, lower values + # decrease the amount of swap. a value of 0 instructs the kernel not to + # initiate swap until the amount of free and file-backed pages is less + # than the high water mark in a zone. + ansible.builtin.sysctl: + name: "vm.swappiness" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust shmmax + # shared memory max must be greator or equal to the total size of hugepages. + # for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024 + # if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) + # is greater than the calculated totalhugepagesize then set this parameter + # to current shmmax value. + ansible.builtin.sysctl: + name: "kernel.shmmax" + value: "{{ sysctl.vm.nr_hugepages * 2 * 1024 * 1024 }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust watchdog_cpumask + # this value can be used to control on which cpus the watchdog may run. + # the default cpumask is all possible cores, but if no_hz_full is + # enabled in the kernel config, and cores are specified with the + # nohz_full= boot argument, those cores are excluded by default. + # offline cores can be included in this mask, and if the core is later + # brought online, the watchdog will be started based on the mask value. + # + # typically this value would only be touched in the nohz_full case + # to re-enable cores that by default were not running the watchdog, + # if a kernel lockup was suspected on those cores. + ansible.builtin.sysctl: + name: "kernel.watchdog_cpumask" + value: "{{ sysctl.kernel.watchdog_cpumask }}" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Adjust randomize_va_space + # this option can be used to select the type of process address + # space randomization that is used in the system, for architectures + # that support this feature. + # 0 - turn the process address space randomization off. this is the + # default for architectures that do not support this feature anyways, + # and kernels that are booted with the "norandmaps" parameter. + ansible.builtin.sysctl: + name: "kernel.randomize_va_space" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - perf-conf-sysctl + +- name: Conf - Cpufrequtils + ansible.builtin.copy: + src: "files/cpufrequtils" + dest: "/etc/default/cpufrequtils" + owner: "root" + group: "root" + mode: 0644 + tags: + - perf-conf-cpufrequtils + +- name: Conf - Irqbalance + ansible.builtin.template: + src: "files/irqbalance" + dest: "/etc/default/irqbalance" + owner: "root" + group: "root" + mode: 0644 + tags: + - perf-conf-irqbalance + +- name: Conf - Kernel Parameters + ansible.builtin.lineinfile: + path: "/etc/default/grub" + state: "present" + regexp: "^GRUB_CMDLINE_LINUX=" + line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value is sameas true %}{{key}} {% else %}{{key}}={{value}} {% endif %}{% endfor %}\"" + notify: + - "Update GRUB" + tags: + - perf-conf-grub + +- meta: flush_handlers + +- name: Conf - Load Kernel Modules By Default + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "vfio-pci" + notify: + - "Reboot Server" + tags: + - perf-conf-load-kernel-modules + +- name: Conf - Create a directory for 1G HugeTLBs hugepages + ansible.builtin.file: + path: "/dev/hugepages1G" + state: "directory" + mode: 0755 + tags: + - perf-conf-hugepages-1g + +- name: Conf - Mount 1G HugeTLBs hugepages + ansible.builtin.mount: + path: "/dev/hugepages1G" + src: "hugetlbfs" + opts: "pagesize=1G" + boot: false + state: "mounted" + fstype: "hugetlbfs" + tags: + - perf-conf-hugepages-1g + +- name: Create a directory if it does not exist + ansible.builtin.file: + path: "/dev/hugepages2M" + state: "directory" + mode: 0755 + tags: + - perf-conf-hugepages-2m + +- name: Conf - Create a directory for 2M HugeTLBs hugepages + ansible.builtin.mount: + path: "/dev/hugepages2M" + src: "hugetlbfs" + opts: "pagesize=2M" + boot: false + state: "mounted" + fstype: "hugetlbfs" + tags: + - perf-conf-hugepages-2m + +- meta: flush_handlers diff --git a/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml new file mode 100644 index 0000000000..cff71e9ce3 --- /dev/null +++ b/fdio.infra.ansible/roles/performance_tuning/tasks/turbo_boost.yaml @@ -0,0 +1,44 @@ +--- +# file: roles/performance_tuning/tasks/turbo_boost.yaml + +- name: Inst - Update Package Cache (APT) + apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - turbo-inst-prerequisites + +- name: Inst - msr-tools + package: + name: + - "msr-tools" + state: latest + tags: + - turbo-inst-prerequisites + +- name: Conf - Load msr By Default + lineinfile: + path: "/etc/modules" + state: "present" + line: "msr" + tags: + - turbo-conf-msr + +- name: Conf - Custom Startup Service Hook + copy: + src: "files/disable-turbo-boost.service" + dest: "/etc/systemd/system/disable-turbo-boost.service" + owner: "root" + group: "root" + mode: 0644 + tags: + - turbo-conf-msr + +- name: Conf - Custom Startup Service Hook Enable + service: + name: "disable-turbo-boost" + enabled: true + tags: + - turbo-conf-msr diff --git a/fdio.infra.ansible/roles/prometheus_exporter/defaults/main.yaml b/fdio.infra.ansible/roles/prometheus_exporter/defaults/main.yaml new file mode 100644 index 0000000000..7291ce0276 --- /dev/null +++ b/fdio.infra.ansible/roles/prometheus_exporter/defaults/main.yaml @@ -0,0 +1,31 @@ +--- +# file: roles/prometheus_exporter/defaults/main.yaml + +# Inst - Exporters. +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "python3-docker" + - "python3-dockerpty" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +ne_image: "{{ ne_image_by_arch[ansible_machine] }}" + +ne_image_by_arch: + aarch64: "prom/node-exporter:v1.3.1" + x86_64: "prom/node-exporter:v1.3.1" + +be_image: "{{ be_image_by_arch[ansible_machine] }}" + +be_image_by_arch: + aarch64: "prom/blackbox-exporter:v0.21.1" + x86_64: "prom/blackbox-exporter:v0.21.1" diff --git a/fdio.infra.ansible/roles/prometheus_exporter/files/blackbox.yml b/fdio.infra.ansible/roles/prometheus_exporter/files/blackbox.yml new file mode 100644 index 0000000000..526dcf5dce --- /dev/null +++ b/fdio.infra.ansible/roles/prometheus_exporter/files/blackbox.yml @@ -0,0 +1,25 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + no_follow_redirects: false + fail_if_ssl: false + fail_if_not_ssl: true + tls_config: + insecure_skip_verify: false + preferred_ip_protocol: "ip4" + icmp_v4: + prober: icmp + timeout: 5s + icmp: + preferred_ip_protocol: "ip4" + dns_udp: + prober: dns + timeout: 5s + dns: + query_name: "jenkins.fd.io" + query_type: "A" + valid_rcodes: + - NOERROR diff --git a/fdio.infra.ansible/roles/prometheus_exporter/meta/main.yaml b/fdio.infra.ansible/roles/prometheus_exporter/meta/main.yaml new file mode 100644 index 0000000000..7d8b861882 --- /dev/null +++ b/fdio.infra.ansible/roles/prometheus_exporter/meta/main.yaml @@ -0,0 +1,18 @@ +--- +# file: roles/prometheus_exporter/meta/main.yaml + +dependencies: [docker] + +galaxy_info: + role_name: prometheus_exporter + author: fd.io + description: Prometheus Exporters. + company: none + license: "license (Apache)" + min_ansible_version: 2.9 + platforms: + - name: Ubuntu + versions: + - jammy + galaxy_tags: + - prometheus diff --git a/fdio.infra.ansible/roles/prometheus_exporter/tasks/main.yaml b/fdio.infra.ansible/roles/prometheus_exporter/tasks/main.yaml new file mode 100644 index 0000000000..ef9da40175 --- /dev/null +++ b/fdio.infra.ansible/roles/prometheus_exporter/tasks/main.yaml @@ -0,0 +1,72 @@ +--- +# file: roles/prometheus_exporter/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - prometheus-inst + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - prometheus-inst + +- name: Inst - Start a NodeExporter container + docker_container: + name: "NodeExporter" + image: "{{ ne_image }}" + state: "started" + restart_policy: "unless-stopped" + detach: true + ports: + - "9100:9100" + privileged: true + command: + - "--path.procfs=/host/proc" + - "--path.rootfs=/rootfs" + - "--path.sysfs=/host/sys" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + volumes: + - "/:/rootfs:ro" + - "/proc:/host/proc:ro" + - "/sys:/host/sys:ro" + tags: + - prometheus-inst + +- name: Inst - Create a Config Directory + ansible.builtin.file: + path: "/etc/prometheus/" + state: "directory" + mode: "0755" + tags: + - prometheus-conf-blackbox-exporter + +- name: Conf - Prometheus Blackbox Exporter + ansible.builtin.copy: + src: "files/blackbox.yml" + dest: "/etc/prometheus/blackbox.yml" + tags: + - prometheus-conf-blackbox-exporter + +- name: Inst - Start a BlackBoxExporter container + docker_container: + name: "BlackBoxExporter" + image: "{{ be_image }}" + state: "started" + restart_policy: "unless-stopped" + detach: true + ports: + - "9115:9115" + privileged: true + command: + - "--config.file=/config/blackbox.yml" + volumes: + - "/etc/prometheus/blackbox.yml:/config/blackbox.yml:ro" + tags: + - prometheus-inst diff --git a/fdio.infra.ansible/roles/python_env/defaults/main.yaml b/fdio.infra.ansible/roles/python_env/defaults/main.yaml new file mode 100644 index 0000000000..4b572c0dd0 --- /dev/null +++ b/fdio.infra.ansible/roles/python_env/defaults/main.yaml @@ -0,0 +1,25 @@ +--- +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - "virtualenv" + +packages_by_distro: + ubuntu: + jammy: + - "python3-all" + - "python3-apt" + - "python3-cffi" + - "python3-cffi-backend" + - "python3-dev" + - "python3-pip" + - "python3-pyelftools" + - "python3-setuptools" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] diff --git a/fdio.infra.ansible/roles/python_env/tasks/main.yaml b/fdio.infra.ansible/roles/python_env/tasks/main.yaml new file mode 100644 index 0000000000..02850110a9 --- /dev/null +++ b/fdio.infra.ansible/roles/python_env/tasks/main.yaml @@ -0,0 +1,62 @@ +--- +# file: tasks/main.yaml + +- name: Inst - Update package cache (apt) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - common-inst-prerequisites + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - common-inst-prerequisites + +- name: Inst - CSIT PIP requirements + ansible.builtin.pip: + name: + - "ecdsa==0.18.0" + - "paramiko==3.3.1" + - "pycrypto==2.6.1" + - "python-dateutil==2.8.2" + - "PyYAML==6.0.1" + - "requests==2.31.0" + - "robotframework==6.1.1" + - "scapy==2.4.5" + - "scp==0.14.5" + - "ansible==8.2.0" + - "ansible-core==2.15.2" + - "dill==0.3.7" + - "numpy==1.25.2" + - "scipy==1.11.1" + - "ply==3.11" + - "jsonschema==4.18.4" + - "rfc3339-validator==0.1.4" + - "rfc3987==1.3.8" + - "attrs==23.1.0" + - "bcrypt==4.0.1" + - "certifi==2023.7.22" + - "cffi==1.15.1" + - "charset-normalizer==3.2.0" + - "cryptography==41.0.3" + - "idna==3.4" + - "Jinja2==3.1.2" + - "jsonschema-specifications==2023.7.1" + - "MarkupSafe==2.1.3" + - "packaging==23.1" + - "pycparser==2.21" + - "PyNaCl==1.5.0" + - "referencing==0.30.0" + - "resolvelib==1.0.1" + - "rpds-py==0.9.2" + - "six==1.16.0" + - "urllib3==2.0.4" + environment: + ANSIBLE_SKIP_CONFLICT_CHECK: 1 + tags: + - common-inst-pip diff --git a/fdio.infra.ansible/roles/topology/tasks/main.yaml b/fdio.infra.ansible/roles/topology/tasks/main.yaml new file mode 100644 index 0000000000..1dc704331d --- /dev/null +++ b/fdio.infra.ansible/roles/topology/tasks/main.yaml @@ -0,0 +1,23 @@ +--- +# file: tasks/main.yaml + +- name: Create Topology File + ansible.builtin.template: + src: "templates/topology-{{ cloud_topology }}.j2" + dest: "../topologies/available/{{ cloud_topology }}-{{ testbed_name }}.yaml" + tags: + - create-topology-file + +- name: Create Inventory Folder + ansible.builtin.file: + path: "./inventories/cloud_inventory/" + state: directory + tags: + - create-inventory-folder-cloud + +- name: Create Hosts File + ansible.builtin.template: + src: "templates/hosts.j2" + dest: "./inventories/cloud_inventory/hosts" + tags: + - create-hosts-file-cloud diff --git a/fdio.infra.ansible/roles/topology/templates/hosts.j2 b/fdio.infra.ansible/roles/topology/templates/hosts.j2 new file mode 100644 index 0000000000..f02586cc99 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/hosts.j2 @@ -0,0 +1,9 @@ +all: + children: + tg: + hosts: + {{ tg_public_ip }} + sut: + hosts: + {{ dut1_public_ip | default() }} + {{ dut2_public_ip | default() }} diff --git a/fdio.infra.ansible/roles/topology/templates/topology-1n-aws-c5n.j2 b/fdio.infra.ansible/roles/topology/templates/topology-1n-aws-c5n.j2 new file mode 100644 index 0000000000..649d7e746c --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-1n-aws-c5n.j2 @@ -0,0 +1,30 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/1_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 1-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 50GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-50G + port2: + # tg_instance/p2 - 50GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link1 + model: Amazon-Nitro-50G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-1n-c6gn.j2 b/fdio.infra.ansible/roles/topology/templates/topology-1n-c6gn.j2 new file mode 100644 index 0000000000..647a40b1e7 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-1n-c6gn.j2 @@ -0,0 +1,30 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/1_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 1-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 100GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-100G + port2: + # tg_instance/p2 - 100GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link1 + model: Amazon-Nitro-100G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-1n-c6in.j2 b/fdio.infra.ansible/roles/topology/templates/topology-1n-c6in.j2 new file mode 100644 index 0000000000..7d3f4e5318 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-1n-c6in.j2 @@ -0,0 +1,30 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/1_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 1-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 200GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-200G + port2: + # tg_instance/p2 - 200GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link1 + model: Amazon-Nitro-200G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-2n-aws-c5n.j2 b/fdio.infra.ansible/roles/topology/templates/topology-2n-aws-c5n.j2 new file mode 100644 index 0000000000..de43291cc6 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-2n-aws-c5n.j2 @@ -0,0 +1,51 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/2_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 2-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 50GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-50G + port2: + # tg_instance/p2 - 50GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-50G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut1_instance/p1 - 50GE port1 on ENA NIC. + mac_address: {{ dut1_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-50G + port2: + # dut1_instance/p2 - 50GE port2 on ENA NIC. + mac_address: {{ dut1_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-50G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-2n-c6gn.j2 b/fdio.infra.ansible/roles/topology/templates/topology-2n-c6gn.j2 new file mode 100644 index 0000000000..e693f6c42c --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-2n-c6gn.j2 @@ -0,0 +1,51 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/2_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 2-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 100GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-100G + port2: + # tg_instance/p2 - 100GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-100G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut1_instance/p1 - 100GE port1 on ENA NIC. + mac_address: {{ dut1_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-100G + port2: + # dut1_instance/p2 - 100GE port2 on ENA NIC. + mac_address: {{ dut1_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-100G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-2n-c6in.j2 b/fdio.infra.ansible/roles/topology/templates/topology-2n-c6in.j2 new file mode 100644 index 0000000000..ef7b464967 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-2n-c6in.j2 @@ -0,0 +1,51 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/2_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 2-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 200GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-200G + port2: + # tg_instance/p2 - 200GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-200G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut1_instance/p1 - 200GE port1 on ENA NIC. + mac_address: {{ dut1_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-200G + port2: + # dut1_instance/p2 - 200GE port2 on ENA NIC. + mac_address: {{ dut1_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-200G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-3n-aws-c5n.j2 b/fdio.infra.ansible/roles/topology/templates/topology-3n-aws-c5n.j2 new file mode 100644 index 0000000000..b353aa5ad1 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-3n-aws-c5n.j2 @@ -0,0 +1,73 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/3_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 3-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 50GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-50G + port2: + # tg_instance/p2 - 50GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-50G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut1_instance/p1 - 50GE port1 on ENA NIC. + mac_address: {{ dut1_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-50G + port2: + # dut1_instance/p2 - 50GE port2 on ENA NIC. + mac_address: {{ dut1_if2_mac }} + pci_address: "0000:00:07.0" + link: link21 + model: Amazon-Nitro-50G + DUT2: + type: DUT + host: "{{ dut2_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut2_instance/p1 - 50GE port1 on ENA NIC. + mac_address: {{ dut2_if1_mac }} + pci_address: "0000:00:06.0" + link: link21 + model: Amazon-Nitro-50G + port2: + # dut2_instance/p2 - 50GE port1 on ENA NIC. + mac_address: {{ dut2_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-50G + diff --git a/fdio.infra.ansible/roles/topology/templates/topology-3n-azure-Fsv2.j2 b/fdio.infra.ansible/roles/topology/templates/topology-3n-azure-Fsv2.j2 new file mode 100644 index 0000000000..e4dd6cdbf2 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-3n-azure-Fsv2.j2 @@ -0,0 +1,82 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/3_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 3-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 40GE port1 on Mellanox NIC. + mac_address: "{{ tg_if1_mac | lower | replace('-',':') }}" + pci_address: "0002:00:02.0" + link: link1 + model: Azure-MLX-40G + port2: + # tg_instance/p2 - 40GE port2 on Mellanox NIC. + mac_address: "{{ tg_if2_mac | lower | replace('-',':') }}" + pci_address: "0003:00:02.0" + link: link2 + model: Azure-MLX-40G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + honeycomb: + user: admin + passwd: admin + port: 8183 + netconf_port: 2831 + interfaces: + port1: + # dut1_instance/p1 - 40GE port1 on Mellanox NIC. + mac_address: "{{ dut1_if1_mac | lower | replace('-',':') }}" + pci_address: "0002:00:02.0" + link: link1 + model: Azure-MLX-40G + port2: + # dut2_instance/p1 - 40GE port2 on Mellanox NIC. + mac_address: "{{ dut1_if2_mac | lower | replace('-',':') }}" + pci_address: "0003:00:02.0" + link: link21 + model: Azure-MLX-40G + DUT2: + type: DUT + host: "{{ dut2_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + honeycomb: + user: admin + passwd: admin + port: 8183 + netconf_port: 2831 + interfaces: + port1: + # dut1_instance/p1 - 40GE port1 on Mellanox NIC. + mac_address: "{{ dut2_if1_mac | lower | replace('-',':') }}" + pci_address: "0002:00:02.0" + link: link21 + model: Azure-MLX-40G + port2: + # dut2_instance/p1 - 40GE port2 on Mellanox NIC. + mac_address: "{{ dut2_if2_mac | lower | replace('-',':') }}" + pci_address: "0003:00:02.0" + link: link2 + model: Azure-MLX-40G diff --git a/fdio.infra.ansible/roles/topology/templates/topology-3n-c6gn.j2 b/fdio.infra.ansible/roles/topology/templates/topology-3n-c6gn.j2 new file mode 100644 index 0000000000..295d457f49 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-3n-c6gn.j2 @@ -0,0 +1,73 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/3_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 3-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 100GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-100G + port2: + # tg_instance/p2 - 100GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-100G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut1_instance/p1 - 100GE port1 on ENA NIC. + mac_address: {{ dut1_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-100G + port2: + # dut1_instance/p2 - 100GE port2 on ENA NIC. + mac_address: {{ dut1_if2_mac }} + pci_address: "0000:00:07.0" + link: link21 + model: Amazon-Nitro-100G + DUT2: + type: DUT + host: "{{ dut2_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut2_instance/p1 - 100GE port1 on ENA NIC. + mac_address: {{ dut2_if1_mac }} + pci_address: "0000:00:06.0" + link: link21 + model: Amazon-Nitro-100G + port2: + # dut2_instance/p2 - 100GE port1 on ENA NIC. + mac_address: {{ dut2_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-100G + diff --git a/fdio.infra.ansible/roles/topology/templates/topology-3n-c6in.j2 b/fdio.infra.ansible/roles/topology/templates/topology-3n-c6in.j2 new file mode 100644 index 0000000000..c280f4e7e1 --- /dev/null +++ b/fdio.infra.ansible/roles/topology/templates/topology-3n-c6in.j2 @@ -0,0 +1,73 @@ +--- +metadata: + version: 0.1 + schema: + - resources/topology_schemas/3_node_topology.sch.yaml + - resources/topology_schemas/topology.sch.yaml + tags: [hw, 3-node] + +nodes: + TG: + type: TG + subtype: TREX + host: "{{ tg_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + interfaces: + port1: + # tg_instance/p1 - 200GE port1 on ENA NIC. + mac_address: {{ tg_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-200G + port2: + # tg_instance/p2 - 200GE port2 on ENA NIC. + mac_address: {{ tg_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-200G + DUT1: + type: DUT + host: "{{ dut1_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut1_instance/p1 - 200GE port1 on ENA NIC. + mac_address: {{ dut1_if1_mac }} + pci_address: "0000:00:06.0" + link: link1 + model: Amazon-Nitro-200G + port2: + # dut1_instance/p2 - 200GE port2 on ENA NIC. + mac_address: {{ dut1_if2_mac }} + pci_address: "0000:00:07.0" + link: link21 + model: Amazon-Nitro-200G + DUT2: + type: DUT + host: "{{ dut2_public_ip }}" + arch: x86_64 + port: 22 + username: testuser + password: Csit1234 + uio_driver: vfio-pci + interfaces: + port1: + # dut2_instance/p1 - 200GE port1 on ENA NIC. + mac_address: {{ dut2_if1_mac }} + pci_address: "0000:00:06.0" + link: link21 + model: Amazon-Nitro-200G + port2: + # dut2_instance/p2 - 200GE port1 on ENA NIC. + mac_address: {{ dut2_if2_mac }} + pci_address: "0000:00:07.0" + link: link2 + model: Amazon-Nitro-200G + diff --git a/fdio.infra.ansible/roles/trex/defaults/main.yaml b/fdio.infra.ansible/roles/trex/defaults/main.yaml new file mode 100644 index 0000000000..18a2b56bda --- /dev/null +++ b/fdio.infra.ansible/roles/trex/defaults/main.yaml @@ -0,0 +1,33 @@ +--- +# file: defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - [] + +packages_by_distro: + ubuntu: + - "build-essential" + - "gcc-9" + - "g++-9" + - "libmnl-dev" + - "libnuma-dev" + - "libpcap-dev" + - "librdmacm-dev" + - "librdmacm1" + - "libssl-dev" + - "pciutils" + - "python3-pip" + - "zlib1g-dev" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +trex_target_dir: "/opt" +trex_url: "https://github.com/cisco-system-traffic-generator/trex-core/archive/" +trex_version: + - "3.03" diff --git a/fdio.infra.ansible/roles/trex/files/t-rex.patch b/fdio.infra.ansible/roles/trex/files/t-rex.patch new file mode 100644 index 0000000000..e7db647779 --- /dev/null +++ b/fdio.infra.ansible/roles/trex/files/t-rex.patch @@ -0,0 +1,548 @@ +diff --git a/linux_dpdk/ws_main.py b/linux_dpdk/ws_main.py +index e8d0cd51..a0c01adb 100755 +--- a/linux_dpdk/ws_main.py ++++ b/linux_dpdk/ws_main.py +@@ -209,7 +209,7 @@ def check_ofed(ctx): + + ofed_ver= 42 + ofed_ver_show= '4.2' +- ++ return True + if not os.path.isfile(ofed_info): + ctx.end_msg('not found', 'YELLOW') + return False +@@ -1552,8 +1552,6 @@ class build_option: + flags += ['-DNDEBUG']; + else: + flags += ['-UNDEBUG']; +- if bld.env.OFED_OK: +- flags += ['-DHAVE_IBV_MLX4_WQE_LSO_SEG=1'] + return (flags) + + def get_bnxt_flags(self): +diff --git a/src/dpdk/drivers/net/mlx4/mlx4_autoconf.h b/src/dpdk/drivers/net/mlx4/mlx4_autoconf.h +index b3d68683..35474409 100644 +--- a/src/dpdk/drivers/net/mlx4/mlx4_autoconf.h ++++ b/src/dpdk/drivers/net/mlx4/mlx4_autoconf.h +@@ -1,3 +1,6 @@ +-#ifndef HAVE_IBV_MLX4_WQE_LSO_SEG +-#define HAVE_IBV_MLX4_WQE_LSO_SEG +-#endif ++/* HAVE_IBV_MLX4_BUF_ALLOCATORS is not defined. */ ++ ++/* HAVE_IBV_MLX4_UAR_MMAP_OFFSET is not defined. */ ++ ++/* HAVE_IBV_MLX4_WQE_LSO_SEG is not defined. */ ++ +diff --git a/src/dpdk/drivers/net/mlx5/mlx5_autoconf.h b/src/dpdk/drivers/net/mlx5/mlx5_autoconf.h +index 8770fdde..75db5ae8 100644 +--- a/src/dpdk/drivers/net/mlx5/mlx5_autoconf.h ++++ b/src/dpdk/drivers/net/mlx5/mlx5_autoconf.h +@@ -1,54 +1,362 @@ +-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT +-#define HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT +-#endif ++/* HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT is not defined. */ + +-#ifndef HAVE_IBV_FLOW_DV_SUPPORT +-#define HAVE_IBV_FLOW_DV_SUPPORT +-#endif ++#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT ++#define HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1 ++#endif /* HAVE_IBV_DEVICE_TUNNEL_SUPPORT */ + +-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 +-#define HAVE_IBV_DEVICE_COUNTERS_SET_V45 +-#endif ++/* HAVE_IBV_DEVICE_MPLS_SUPPORT is not defined. */ + +-#ifndef HAVE_IBV_FLOW_DEVX_COUNTERS +-#define HAVE_IBV_FLOW_DEVX_COUNTERS +-#endif ++#ifndef HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING ++#define HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING 1 ++#endif /* HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING */ + +-#ifndef HAVE_IBV_MLX4_WQE_LSO_SEG +-#define HAVE_IBV_MLX4_WQE_LSO_SEG +-#endif ++/* HAVE_IBV_WQ_FLAG_RX_END_PADDING is not defined. */ + ++#ifndef HAVE_IBV_MLX5_MOD_SWP ++#define HAVE_IBV_MLX5_MOD_SWP 1 ++#endif /* HAVE_IBV_MLX5_MOD_SWP */ + +-#ifdef SUPPORTED_40000baseKR4_Full ++#ifndef HAVE_IBV_MLX5_MOD_MPW ++#define HAVE_IBV_MLX5_MOD_MPW 1 ++#endif /* HAVE_IBV_MLX5_MOD_MPW */ ++ ++#ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP ++#define HAVE_IBV_MLX5_MOD_CQE_128B_COMP 1 ++#endif /* HAVE_IBV_MLX5_MOD_CQE_128B_COMP */ ++ ++#ifndef HAVE_IBV_MLX5_MOD_CQE_128B_PAD ++#define HAVE_IBV_MLX5_MOD_CQE_128B_PAD 1 ++#endif /* HAVE_IBV_MLX5_MOD_CQE_128B_PAD */ ++ ++/* HAVE_IBV_FLOW_DV_SUPPORT is not defined. */ ++ ++/* HAVE_MLX5DV_DR is not defined. */ ++ ++/* HAVE_MLX5DV_DR_ESWITCH is not defined. */ ++ ++/* HAVE_IBV_DEVX_OBJ is not defined. */ ++ ++/* HAVE_IBV_FLOW_DEVX_COUNTERS is not defined. */ ++ ++#ifndef HAVE_ETHTOOL_LINK_MODE_25G ++#define HAVE_ETHTOOL_LINK_MODE_25G 1 ++#endif /* HAVE_ETHTOOL_LINK_MODE_25G */ ++ ++#ifndef HAVE_ETHTOOL_LINK_MODE_50G ++#define HAVE_ETHTOOL_LINK_MODE_50G 1 ++#endif /* HAVE_ETHTOOL_LINK_MODE_50G */ ++ ++#ifndef HAVE_ETHTOOL_LINK_MODE_100G ++#define HAVE_ETHTOOL_LINK_MODE_100G 1 ++#endif /* HAVE_ETHTOOL_LINK_MODE_100G */ ++ ++/* HAVE_IBV_DEVICE_COUNTERS_SET_V42 is not defined. */ ++ ++/* HAVE_IBV_DEVICE_COUNTERS_SET_V45 is not defined. */ ++ ++#ifndef HAVE_RDMA_NL_NLDEV ++#define HAVE_RDMA_NL_NLDEV 1 ++#endif /* HAVE_RDMA_NL_NLDEV */ ++ ++#ifndef HAVE_RDMA_NLDEV_CMD_GET ++#define HAVE_RDMA_NLDEV_CMD_GET 1 ++#endif /* HAVE_RDMA_NLDEV_CMD_GET */ ++ ++#ifndef HAVE_RDMA_NLDEV_CMD_PORT_GET ++#define HAVE_RDMA_NLDEV_CMD_PORT_GET 1 ++#endif /* HAVE_RDMA_NLDEV_CMD_PORT_GET */ ++ ++#ifndef HAVE_RDMA_NLDEV_ATTR_DEV_INDEX ++#define HAVE_RDMA_NLDEV_ATTR_DEV_INDEX 1 ++#endif /* HAVE_RDMA_NLDEV_ATTR_DEV_INDEX */ ++ ++#ifndef HAVE_RDMA_NLDEV_ATTR_DEV_NAME ++#define HAVE_RDMA_NLDEV_ATTR_DEV_NAME 1 ++#endif /* HAVE_RDMA_NLDEV_ATTR_DEV_NAME */ ++ ++#ifndef HAVE_RDMA_NLDEV_ATTR_PORT_INDEX ++#define HAVE_RDMA_NLDEV_ATTR_PORT_INDEX 1 ++#endif /* HAVE_RDMA_NLDEV_ATTR_PORT_INDEX */ ++ ++/* HAVE_RDMA_NLDEV_ATTR_NDEV_INDEX is not defined. */ ++ ++#ifndef HAVE_IFLA_NUM_VF ++#define HAVE_IFLA_NUM_VF 1 ++#endif /* HAVE_IFLA_NUM_VF */ ++ ++#ifndef HAVE_IFLA_EXT_MASK ++#define HAVE_IFLA_EXT_MASK 1 ++#endif /* HAVE_IFLA_EXT_MASK */ ++ ++#ifndef HAVE_IFLA_PHYS_SWITCH_ID ++#define HAVE_IFLA_PHYS_SWITCH_ID 1 ++#endif /* HAVE_IFLA_PHYS_SWITCH_ID */ ++ ++#ifndef HAVE_IFLA_PHYS_PORT_NAME ++#define HAVE_IFLA_PHYS_PORT_NAME 1 ++#endif /* HAVE_IFLA_PHYS_PORT_NAME */ ++ ++#ifndef HAVE_IFLA_VXLAN_COLLECT_METADATA ++#define HAVE_IFLA_VXLAN_COLLECT_METADATA 1 ++#endif /* HAVE_IFLA_VXLAN_COLLECT_METADATA */ ++ ++#ifndef HAVE_TCA_CHAIN ++#define HAVE_TCA_CHAIN 1 ++#endif /* HAVE_TCA_CHAIN */ ++ ++#ifndef HAVE_TCA_FLOWER_ACT ++#define HAVE_TCA_FLOWER_ACT 1 ++#endif /* HAVE_TCA_FLOWER_ACT */ ++ ++#ifndef HAVE_TCA_FLOWER_FLAGS ++#define HAVE_TCA_FLOWER_FLAGS 1 ++#endif /* HAVE_TCA_FLOWER_FLAGS */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE ++#define HAVE_TCA_FLOWER_KEY_ETH_TYPE 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ETH_TYPE */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST ++#define HAVE_TCA_FLOWER_KEY_ETH_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ETH_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_ETH_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ETH_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC ++#define HAVE_TCA_FLOWER_KEY_ETH_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ETH_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO ++#define HAVE_TCA_FLOWER_KEY_IP_PROTO 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IP_PROTO */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC ++#define HAVE_TCA_FLOWER_KEY_IPV4_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV4_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST ++#define HAVE_TCA_FLOWER_KEY_IPV4_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV4_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC ++#define HAVE_TCA_FLOWER_KEY_IPV6_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV6_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST ++#define HAVE_TCA_FLOWER_KEY_IPV6_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV6_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC ++#define HAVE_TCA_FLOWER_KEY_TCP_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_TCP_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST ++#define HAVE_TCA_FLOWER_KEY_TCP_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_TCP_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_TCP_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_TCP_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC ++#define HAVE_TCA_FLOWER_KEY_UDP_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_UDP_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST ++#define HAVE_TCA_FLOWER_KEY_UDP_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_UDP_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_UDP_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_UDP_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID ++#define HAVE_TCA_FLOWER_KEY_VLAN_ID 1 ++#endif /* HAVE_TCA_FLOWER_KEY_VLAN_ID */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO ++#define HAVE_TCA_FLOWER_KEY_VLAN_PRIO 1 ++#endif /* HAVE_TCA_FLOWER_KEY_VLAN_PRIO */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE ++#define HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE 1 ++#endif /* HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS ++#define HAVE_TCA_FLOWER_KEY_TCP_FLAGS 1 ++#endif /* HAVE_TCA_FLOWER_KEY_TCP_FLAGS */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK ++#define HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IP_TOS ++#define HAVE_TCA_FLOWER_KEY_IP_TOS 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IP_TOS */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IP_TOS_MASK ++#define HAVE_TCA_FLOWER_KEY_IP_TOS_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IP_TOS_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IP_TTL ++#define HAVE_TCA_FLOWER_KEY_IP_TTL 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IP_TTL */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_IP_TTL_MASK ++#define HAVE_TCA_FLOWER_KEY_IP_TTL_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_IP_TTL_MASK */ ++ ++#ifndef HAVE_TC_ACT_GOTO_CHAIN ++#define HAVE_TC_ACT_GOTO_CHAIN 1 ++#endif /* HAVE_TC_ACT_GOTO_CHAIN */ ++ ++#ifndef HAVE_TC_ACT_VLAN ++#define HAVE_TC_ACT_VLAN 1 ++#endif /* HAVE_TC_ACT_VLAN */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_KEY_ID ++#define HAVE_TCA_FLOWER_KEY_ENC_KEY_ID 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_KEY_ID */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV4_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK ++#define HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_IPV6_DST_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT ++#define HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK ++#define HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT ++#define HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT */ ++ ++#ifndef HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK ++#define HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK 1 ++#endif /* HAVE_TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK */ ++ ++/* HAVE_TCA_FLOWER_KEY_ENC_IP_TOS is not defined. */ ++ ++/* HAVE_TCA_FLOWER_KEY_ENC_IP_TOS_MASK is not defined. */ ++ ++/* HAVE_TCA_FLOWER_KEY_ENC_IP_TTL is not defined. */ ++ ++/* HAVE_TCA_FLOWER_KEY_ENC_IP_TTL_MASK is not defined. */ ++ ++#ifndef HAVE_TC_ACT_TUNNEL_KEY ++#define HAVE_TC_ACT_TUNNEL_KEY 1 ++#endif /* HAVE_TC_ACT_TUNNEL_KEY */ ++ ++#ifndef HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT ++#define HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT 1 ++#endif /* HAVE_TCA_TUNNEL_KEY_ENC_DST_PORT */ ++ ++/* HAVE_TCA_TUNNEL_KEY_ENC_TOS is not defined. */ ++ ++/* HAVE_TCA_TUNNEL_KEY_ENC_TTL is not defined. */ ++ ++#ifndef HAVE_TCA_TUNNEL_KEY_NO_CSUM ++#define HAVE_TCA_TUNNEL_KEY_NO_CSUM 1 ++#endif /* HAVE_TCA_TUNNEL_KEY_NO_CSUM */ ++ ++#ifndef HAVE_TC_ACT_PEDIT ++#define HAVE_TC_ACT_PEDIT 1 ++#endif /* HAVE_TC_ACT_PEDIT */ ++ ++#ifndef HAVE_SUPPORTED_40000baseKR4_Full + #define HAVE_SUPPORTED_40000baseKR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_40000baseKR4_Full */ + +-#ifdef SUPPORTED_40000baseCR4_Full ++#ifndef HAVE_SUPPORTED_40000baseCR4_Full + #define HAVE_SUPPORTED_40000baseCR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_40000baseCR4_Full */ + +-#ifdef SUPPORTED_40000baseSR4_Full ++#ifndef HAVE_SUPPORTED_40000baseSR4_Full + #define HAVE_SUPPORTED_40000baseSR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_40000baseSR4_Full */ + +-#ifdef SUPPORTED_40000baseLR4_Full ++#ifndef HAVE_SUPPORTED_40000baseLR4_Full + #define HAVE_SUPPORTED_40000baseLR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_40000baseLR4_Full */ + +-#ifdef SUPPORTED_56000baseKR4_Full ++#ifndef HAVE_SUPPORTED_56000baseKR4_Full + #define HAVE_SUPPORTED_56000baseKR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_56000baseKR4_Full */ + +-#ifdef SUPPORTED_56000baseCR4_Full ++#ifndef HAVE_SUPPORTED_56000baseCR4_Full + #define HAVE_SUPPORTED_56000baseCR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_56000baseCR4_Full */ + +-#ifdef SUPPORTED_56000baseSR4_Full ++#ifndef HAVE_SUPPORTED_56000baseSR4_Full + #define HAVE_SUPPORTED_56000baseSR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_56000baseSR4_Full */ + +-#ifdef SUPPORTED_56000baseLR4_Full ++#ifndef HAVE_SUPPORTED_56000baseLR4_Full + #define HAVE_SUPPORTED_56000baseLR4_Full 1 +-#endif ++#endif /* HAVE_SUPPORTED_56000baseLR4_Full */ + ++#ifndef HAVE_STATIC_ASSERT ++#define HAVE_STATIC_ASSERT 1 ++#endif /* HAVE_STATIC_ASSERT */ + +diff --git a/src/dpdk/drivers/net/tap/rte_eth_tap.c b/src/dpdk/drivers/net/tap/rte_eth_tap.c +index bc889c19..47a2b68f 100644 +--- a/src/dpdk/drivers/net/tap/rte_eth_tap.c ++++ b/src/dpdk/drivers/net/tap/rte_eth_tap.c +@@ -34,8 +34,8 @@ + #include <unistd.h> + #include <arpa/inet.h> + #include <net/if.h> +-#include <linux_tap/if_tun.h> +-#include <linux_tap/if_ether.h> ++#include <linux/if_tun.h> ++#include <linux/if_ether.h> + #include <fcntl.h> + #include <ctype.h> + +diff --git a/src/dpdk/drivers/net/tap/rte_eth_tap.h b/src/dpdk/drivers/net/tap/rte_eth_tap.h +index 66cd3441..dc3579ac 100644 +--- a/src/dpdk/drivers/net/tap/rte_eth_tap.h ++++ b/src/dpdk/drivers/net/tap/rte_eth_tap.h +@@ -11,7 +11,7 @@ + #include <inttypes.h> + #include <net/if.h> + +-#include <linux_tap/if_tun.h> ++#include <linux/if_tun.h> + + #include <rte_ethdev_driver.h> + #include <rte_ether.h> +diff --git a/src/dpdk/drivers/net/tap/tap_autoconf.h b/src/dpdk/drivers/net/tap/tap_autoconf.h +index dddd4ae6..d5880608 100644 +--- a/src/dpdk/drivers/net/tap/tap_autoconf.h ++++ b/src/dpdk/drivers/net/tap/tap_autoconf.h +@@ -1,14 +1,24 @@ + #ifndef HAVE_TC_FLOWER + #define HAVE_TC_FLOWER 1 +-#endif ++#endif /* HAVE_TC_FLOWER */ + ++#ifndef HAVE_TC_VLAN_ID ++#define HAVE_TC_VLAN_ID 1 ++#endif /* HAVE_TC_VLAN_ID */ + + #ifndef HAVE_TC_BPF + #define HAVE_TC_BPF 1 +-#endif ++#endif /* HAVE_TC_BPF */ + +-#ifndef HAVE_TC_VLAN_ID +-#define HAVE_TC_VLAN_ID 1 +-#endif ++#ifndef HAVE_TC_BPF_FD ++#define HAVE_TC_BPF_FD 1 ++#endif /* HAVE_TC_BPF_FD */ ++ ++#ifndef HAVE_TC_ACT_BPF ++#define HAVE_TC_ACT_BPF 1 ++#endif /* HAVE_TC_ACT_BPF */ + ++#ifndef HAVE_TC_ACT_BPF_FD ++#define HAVE_TC_ACT_BPF_FD 1 ++#endif /* HAVE_TC_ACT_BPF_FD */ + +diff --git a/src/dpdk/drivers/net/tap/tap_netlink.h b/src/dpdk/drivers/net/tap/tap_netlink.h +index 900ce375..faa73ba1 100644 +--- a/src/dpdk/drivers/net/tap/tap_netlink.h ++++ b/src/dpdk/drivers/net/tap/tap_netlink.h +@@ -8,8 +8,8 @@ + + #include <ctype.h> + #include <inttypes.h> +-#include <linux_tap/rtnetlink.h> +-#include <linux_tap/netlink.h> ++#include <linux/rtnetlink.h> ++#include <linux/netlink.h> + #include <stdio.h> + + #include <rte_log.h> +diff --git a/src/dpdk/drivers/net/tap/tap_tcmsgs.h b/src/dpdk/drivers/net/tap/tap_tcmsgs.h +index 782de540..8cedea84 100644 +--- a/src/dpdk/drivers/net/tap/tap_tcmsgs.h ++++ b/src/dpdk/drivers/net/tap/tap_tcmsgs.h +@@ -7,13 +7,13 @@ + #define _TAP_TCMSGS_H_ + + #include <tap_autoconf.h> +-#include <linux_tap/if_ether.h> +-#include <linux_tap/rtnetlink.h> +-#include <linux_tap/pkt_sched.h> +-#include <linux_tap/pkt_cls.h> +-#include <linux_tap/tc_act/tc_mirred.h> +-#include <linux_tap/tc_act/tc_gact.h> +-#include <linux_tap/tc_act/tc_skbedit.h> ++#include <linux/if_ether.h> ++#include <linux/rtnetlink.h> ++#include <linux/pkt_sched.h> ++#include <linux/pkt_cls.h> ++#include <linux/tc_act/tc_mirred.h> ++#include <linux/tc_act/tc_gact.h> ++#include <linux/tc_act/tc_skbedit.h> + #ifdef HAVE_TC_ACT_BPF + #include <linux/tc_act/tc_bpf.h> + #endif +diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp +index 0f66b07a..8c37ea15 100644 +--- a/src/main_dpdk.cpp ++++ b/src/main_dpdk.cpp +@@ -6969,6 +6969,7 @@ COLD_FUNC bool DpdkTRexPortAttr::update_link_status_nowait(){ + bool changed = false; + rte_eth_link_get_nowait(m_repid, &new_link); + ++ new_link.link_speed = ETH_SPEED_NUM_50G; + if (new_link.link_speed != m_link.link_speed || + new_link.link_duplex != m_link.link_duplex || + new_link.link_autoneg != m_link.link_autoneg || diff --git a/fdio.infra.ansible/roles/trex/tasks/deploy_block.yaml b/fdio.infra.ansible/roles/trex/tasks/deploy_block.yaml new file mode 100644 index 0000000000..1a747f68d5 --- /dev/null +++ b/fdio.infra.ansible/roles/trex/tasks/deploy_block.yaml @@ -0,0 +1,63 @@ +--- +# file: tasks/deploy_block.yaml + +- name: Get Release {{ item }} + ansible.builtin.get_url: + url: "{{ trex_url }}/v{{ item }}.tar.gz" + dest: "{{ trex_target_dir }}/trex-core-{{ item }}.tar.gz" + validate_certs: false + mode: 0644 + register: trex_downloaded + +- name: Create Directory {{ item }} + ansible.builtin.file: + path: "{{ trex_target_dir }}/trex-core-{{ item }}" + state: "directory" + +- name: Extract Release {{ item }} + ansible.builtin.unarchive: + remote_src: true + src: "{{ trex_target_dir }}/trex-core-{{ item }}.tar.gz" + dest: "{{ trex_target_dir }}/" + creates: "{{ trex_target_dir }}/trex-core-{{ item }}/linux_dpdk/" + register: trex_extracted + +- name: Compile Release {{ item }} Part I + ansible.builtin.command: "./b configure" + args: + chdir: "{{ trex_target_dir }}/trex-core-{{ item }}/linux_dpdk/" + when: trex_extracted.changed + +- name: Compile Release {{ item }} Part II + ansible.builtin.command: "./b build" + args: + chdir: "{{ trex_target_dir }}/trex-core-{{ item }}/linux_dpdk/" + async: 3000 + poll: 0 + register: trex_built + when: trex_extracted.changed + +- name: Check if T-Rex is Compiled + async_status: + jid: "{{ trex_built.ansible_job_id }}" + register: trex_built + until: trex_built.finished + delay: 10 + retries: 300 + when: trex_extracted.changed + +- name: Compile Release {{ item }} Part III + ansible.builtin.command: "make -j 16" + args: + chdir: "{{ trex_target_dir }}/trex-core-{{ item }}/scripts/ko/src" + when: trex_extracted.changed + +- name: Compile Release {{ item }} Part IV + ansible.builtin.command: "make install" + args: + chdir: "{{ trex_target_dir }}/trex-core-{{ item }}/scripts/ko/src" + when: trex_extracted.changed + +- name: Link libc.a to liblibc.a + ansible.builtin.command: "ln -s -f /usr/lib/x86_64-linux-gnu/libc.a /usr/lib/x86_64-linux-gnu/liblibc.a" + when: trex_extracted.changed diff --git a/fdio.infra.ansible/roles/trex/tasks/main.yaml b/fdio.infra.ansible/roles/trex/tasks/main.yaml new file mode 100644 index 0000000000..d0509f7544 --- /dev/null +++ b/fdio.infra.ansible/roles/trex/tasks/main.yaml @@ -0,0 +1,24 @@ +--- +# file: tasks/main.yaml + +- name: Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - trex-inst-prerequisites + +- name: Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - trex-inst-prerequisites + +- name: Multiple T-Rex Versions + include_tasks: deploy_block.yaml + loop: "{{ trex_version }}" + tags: + - trex-inst diff --git a/fdio.infra.ansible/roles/user_add/defaults/main.yaml b/fdio.infra.ansible/roles/user_add/defaults/main.yaml new file mode 100644 index 0000000000..643ad7dfd7 --- /dev/null +++ b/fdio.infra.ansible/roles/user_add/defaults/main.yaml @@ -0,0 +1,14 @@ +--- +# file: roles/user_add/defaults/main.yaml + +# Default shell for a user if none is specified. +users_shell: /bin/bash + +# Default create home dirs for new users. +users_create_homedirs: true + +# Default list of users to create. +users: [] + +# Default enable password login. +sshd_disable_password_login: false diff --git a/fdio.infra.ansible/roles/user_add/handlers/main.yaml b/fdio.infra.ansible/roles/user_add/handlers/main.yaml new file mode 100644 index 0000000000..5f1f71a332 --- /dev/null +++ b/fdio.infra.ansible/roles/user_add/handlers/main.yaml @@ -0,0 +1,7 @@ +--- +# file: roles/user_add/handlers/main.yaml + +- name: Restart SSHd + ansible.builtin.service: + name: sshd + state: restarted diff --git a/fdio.infra.ansible/roles/user_add/tasks/main.yaml b/fdio.infra.ansible/roles/user_add/tasks/main.yaml new file mode 100644 index 0000000000..329c6abd07 --- /dev/null +++ b/fdio.infra.ansible/roles/user_add/tasks/main.yaml @@ -0,0 +1,39 @@ +--- +# file: roles/user_add/tasks/main.yaml + +- name: Conf - Add User + ansible.builtin.user: + append: "{{ item.append | default(omit) }}" + createhome: "{{ 'yes' if users_create_homedirs else 'no' }}" + generate_ssh_key: "{{ item.generate_ssh_key | default(omit) }}" + groups: "{{ item.groups | join(',') if 'groups' in item else '' }}" + name: "{{ item.username }}" + password: "{{ item.password if item.password is defined else '!' }}" + shell: "{{ item.shell if item.shell is defined else users_shell }}" + state: present + with_items: "{{ users }}" + tags: + - user-add-conf + +- name: Conf - SSH keys + ansible.builtin.authorized_key: + user: "{{ item.0.username }}" + key: "{{ item.1 }}" + with_subelements: + - "{{ users }}" + - ssh_key + - skip_missing: true + tags: + - user-add-conf + +- name: Conf - Disable Password Login + ansible.builtin.lineinfile: + dest: "/etc/ssh/sshd_config" + regexp: "^PasswordAuthentication yes" + line: "PasswordAuthentication no" + notify: + - "Restart SSHd" + when: + - sshd_disable_password_login + tags: + - user-add-conf diff --git a/fdio.infra.ansible/roles/vagrant/defaults/main.yml b/fdio.infra.ansible/roles/vagrant/defaults/main.yml new file mode 100644 index 0000000000..caa3339bb0 --- /dev/null +++ b/fdio.infra.ansible/roles/vagrant/defaults/main.yml @@ -0,0 +1,14 @@ +--- +# file: vagrant/defaults/main.yml + +# Settings for VPP Device host group +csit: + home: "/home/vagrant/csit" + test_user: + name: "testuser" + password: "$6$/mAr/JDJc0u6/i$sLBptji85Xo/vdAv43bP4NpTaAfSBY8p3G7Uj9p4fKysrvs7XF8.FmlC56j4AzOun6nnf7PA.elytvfWoEHCL1" + home: "/home/testuser" + shell: "/bin/bash" + repository: + url: "https://gerrit.fd.io/r/csit" + version: "HEAD" diff --git a/fdio.infra.ansible/roles/vagrant/files/99-vppdevice.yaml b/fdio.infra.ansible/roles/vagrant/files/99-vppdevice.yaml new file mode 100644 index 0000000000..bcaa67099d --- /dev/null +++ b/fdio.infra.ansible/roles/vagrant/files/99-vppdevice.yaml @@ -0,0 +1,28 @@ +network: + version: 2 + renderer: networkd + ethernets: + enp0s8: + match: + macaddress: 08:00:27:0f:e0:4d + set-name: enpTGa + enp0s9: + match: + macaddress: 08:00:27:61:f7:ad + set-name: enpTGb + enp0s17: + match: + macaddress: 08:00:27:dc:5d:a4 + set-name: enpTGc + enp0s10: + match: + macaddress: 08:00:27:38:5e:58 + set-name: enpSUTa + enp0s16: + match: + macaddress: 08:00:27:e3:f5:42 + set-name: enpSUTb + enp0s18: + match: + macaddress: 08:00:27:4f:7c:63 + set-name: enpSUTc diff --git a/fdio.infra.ansible/roles/vagrant/tasks/main.yml b/fdio.infra.ansible/roles/vagrant/tasks/main.yml new file mode 100644 index 0000000000..1716ebe0d5 --- /dev/null +++ b/fdio.infra.ansible/roles/vagrant/tasks/main.yml @@ -0,0 +1,43 @@ +--- +# file: vagrant/tasks/main.yml + +# General +- name: Adjust number of hugepages + sysctl: + name: "vm.nr_hugepages" + value: "512" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: true + +- name: "Add user for running tests: {{ csit.test_user.name }}" + user: + name: "{{ csit.test_user.name }}" + password: "{{ csit.test_user.password }}" + home: "{{ csit.test_user.home }}" + shell: "{{ csit.test_user.shell }}" + +- name: Add vagrant user to docker group + user: + name: "vagrant" + groups: + - "docker" + +- name: Reload groups for current session + command: "/usr/bin/newgrp docker" + +# Disabling CSIT repo cloning in the VM as the repo is synced from the host +# - name: Clone CSIT repository +# become_user: vagrant +# git: +# repo: "{{ csit.repository.url }}" +# dest: "{{ csit.home }}" +# accept_hostkey: true +# version: "{{ csit.repository.version }}" + +- name: Load csit docker image from local path if exists (/vagrant/csit-sut.tar) + shell: | + if [ -z "$(docker images -q `cat {{ csit.home }}/VPP_DEVICE_IMAGE`)" ] && [ -e /vagrant/csit-sut.tar ]; then + docker load -i /vagrant/csit-sut.tar; + fi; + ignore_errors: true diff --git a/fdio.infra.ansible/roles/vault/defaults/main.yaml b/fdio.infra.ansible/roles/vault/defaults/main.yaml new file mode 100644 index 0000000000..5dd3db63c1 --- /dev/null +++ b/fdio.infra.ansible/roles/vault/defaults/main.yaml @@ -0,0 +1,159 @@ +--- +# file: roles/vault/defaults/main.yaml + +# Inst - Prerequisites. +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" +packages_base: + - "curl" + - "unzip" +packages_by_distro: + ubuntu: + - [] +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +# Inst - Vault Map. +vault_version: "1.11.0" +vault_architecture_map: + amd64: "amd64" + x86_64: "amd64" + armv7l: "arm" + aarch64: "arm64" + 32-bit: "386" + 64-bit: "amd64" +vault_architecture: "{{ vault_architecture_map[ansible_architecture] }}" +vault_os: "{{ ansible_system|lower }}" +vault_pkg: "vault_{{ vault_version }}_{{ vault_os }}_{{ vault_architecture }}.zip" +vault_zip_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/{{ vault_pkg }}" + +# Conf - Service. +vault_node_role: "server" +vault_restart_handler_state: "restarted" +vault_systemd_service_name: "vault" + +# Inst - System paths. +vault_bin_dir: "/usr/local/bin" +vault_config_dir: "/etc/vault.d" +vault_data_dir: "/var/vault" +vault_inst_dir: "/opt" +vault_run_dir: "/var/run/vault" +vault_ssl_dir: "/etc/vault.d/ssl" + +# Conf - User and group. +vault_group: "vault" +vault_group_state: "present" +vault_user: "vault" +vault_user_state: "present" + +# Conf - Main +vault_group_name: "vault_instances" +vault_cluster_name: "yul1" +vault_datacenter: "yul1" +vault_log_level: "{{ lookup('env','VAULT_LOG_LEVEL') | default('info', true) }}" +vault_iface: "{{ lookup('env','VAULT_IFACE') | default(ansible_default_ipv4.interface, true) }}" +vault_address: "{{ hostvars[inventory_hostname]['ansible_'+vault_iface]['ipv4']['address'] }}" +vault_ui: "{{ lookup('env', 'VAULT_UI') | default(true, true) }}" +vault_port: 8200 +vault_use_config_path: false +vault_main_config: "{{ vault_config_dir }}/vault_main.hcl" +vault_main_configuration_template: "vault_main_configuration.hcl.j2" +vault_listener_localhost_enable: false +vault_http_proxy: "" +vault_https_proxy: "" +vault_no_proxy: "" + +# Conf - Listeners +vault_tcp_listeners: + - vault_address: "{{ vault_address }}" + vault_port: "{{ vault_port }}" + vault_cluster_address: "{{ vault_cluster_address }}" + vault_tls_disable: "{{ vault_tls_disable }}" + vault_tls_config_path: "{{ vault_tls_config_path }}" + vault_tls_cert_file: "{{ vault_tls_cert_file }}" + vault_tls_key_file: "{{ vault_tls_key_file }}" + vault_tls_ca_file: "{{ vault_tls_ca_file }}" + vault_tls_min_version: "{{ vault_tls_min_version }}" + vault_tls_cipher_suites: "{{ vault_tls_cipher_suites }}" + vault_tls_prefer_server_cipher_suites: "{{ vault_tls_prefer_server_cipher_suites }}" + vault_tls_require_and_verify_client_cert: "{{ vault_tls_require_and_verify_client_cert }}" + vault_tls_disable_client_certs: "{{ vault_tls_disable_client_certs }}" + vault_disable_mlock: true + +# Conf - Backend +vault_backend_consul: "vault_backend_consul.j2" +vault_backend_file: "vault_backend_file.j2" +vault_backend_raft: "vault_backend_raft.j2" +vault_backend_etcd: "vault_backend_etcd.j2" +vault_backend_s3: "vault_backend_s3.j2" +vault_backend_dynamodb: "vault_backend_dynamodb.j2" +vault_backend_mysql: "vault_backend_mysql.j2" +vault_backend_gcs: "vault_backend_gcs.j2" + +vault_cluster_disable: false +vault_cluster_address: "{{ hostvars[inventory_hostname]['ansible_'+vault_iface]['ipv4']['address'] }}:{{ (vault_port | int) + 1}}" +vault_cluster_addr: "{{ vault_protocol }}://{{ vault_cluster_address }}" +vault_api_addr: "{{ vault_protocol }}://{{ vault_redirect_address | default(hostvars[inventory_hostname]['ansible_'+vault_iface]['ipv4']['address']) }}:{{ vault_port }}" + +vault_max_lease_ttl: "768h" +vault_default_lease_ttl: "768h" + +vault_backend_tls_src_files: "{{ vault_tls_src_files }}" +vault_backend_tls_config_path: "{{ vault_tls_config_path }}" +vault_backend_tls_cert_file: "{{ vault_tls_cert_file }}" +vault_backend_tls_key_file: "{{ vault_tls_key_file }}" +vault_backend_tls_ca_file: "{{ vault_tls_ca_file }}" + +vault_consul: "127.0.0.1:8500" +vault_consul_path: "vault" +vault_consul_service: "vault" +vault_consul_scheme: "http" + +vault_backend: "consul" + +# Conf - Service registration +vault_service_registration_consul_enable: true +vault_service_registration_consul_template: "vault_service_registration_consul.hcl.j2" +vault_service_registration_consul_check_timeout: "5s" +vault_service_registration_consul_address: "127.0.0.1:8500" +vault_service_registration_consul_service: "vault" +vault_service_registration_consul_service_tags: "" +vault_service_registration_consul_service_address: +vault_service_registration_consul_disable_registration: false +vault_service_registration_consul_scheme: "http" + +vault_service_registration_consul_tls_config_path: "{{ vault_tls_config_path }}" +vault_service_registration_consul_tls_cert_file: "{{ vault_tls_cert_file }}" +vault_service_registration_consul_tls_key_file: "{{ vault_tls_key_file }}" +vault_service_registration_consul_tls_ca_file: "{{ vault_tls_ca_file }}" +vault_service_registration_consul_tls_min_version: "{{ vault_tls_min_version }}" +vault_service_registration_consul_tls_skip_verify: false + +# Conf - Telemetry +vault_telemetry_enabled: true +vault_telemetry_disable_hostname: false +vault_prometheus_retention_time: 30s + +# Conf - TLS +validate_certs_during_api_reachable_check: true + +vault_tls_config_path: "{{ lookup('env','VAULT_TLS_DIR') | default('/etc/vault/tls', true) }}" +vault_tls_src_files: "{{ lookup('env','VAULT_TLS_SRC_FILES') | default(role_path+'/files', true) }}" + +vault_tls_disable: "{{ lookup('env','VAULT_TLS_DISABLE') | default(1, true) }}" +vault_tls_gossip: "{{ lookup('env','VAULT_TLS_GOSSIP') | default(0, true) }}" + +vault_tls_copy_keys: true +vault_protocol: "{% if vault_tls_disable %}http{% else %}https{% endif %}" +vault_tls_cert_file: "{{ lookup('env','VAULT_TLS_CERT_FILE') | default('server.crt', true) }}" +vault_tls_key_file: "{{ lookup('env','VAULT_TLS_KEY_FILE') | default('server.key', true) }}" +vault_tls_ca_file: "{{ lookup('env','VAULT_TLS_CA_CRT') | default('ca.crt', true) }}" + +vault_tls_min_version: "{{ lookup('env','VAULT_TLS_MIN_VERSION') | default('tls12', true) }}" +vault_tls_cipher_suites: "" +vault_tls_prefer_server_cipher_suites: "{{ lookup('env','VAULT_TLS_PREFER_SERVER_CIPHER_SUITES') | default('false', true) }}" +vault_tls_files_remote_src: false +vault_tls_require_and_verify_client_cert: false +vault_tls_disable_client_certs: false diff --git a/fdio.infra.ansible/roles/vault/handlers/main.yaml b/fdio.infra.ansible/roles/vault/handlers/main.yaml new file mode 100644 index 0000000000..ff2944f115 --- /dev/null +++ b/fdio.infra.ansible/roles/vault/handlers/main.yaml @@ -0,0 +1,9 @@ +--- +# file roles/vault/handlers/main.yaml + +- name: Restart Vault + ansible.builtin.systemd: + daemon_reload: true + enabled: true + name: "{{ vault_systemd_service_name }}" + state: "{{ vault_restart_handler_state }}" diff --git a/fdio.infra.ansible/roles/vault/meta/main.yaml b/fdio.infra.ansible/roles/vault/meta/main.yaml new file mode 100644 index 0000000000..22a62dd438 --- /dev/null +++ b/fdio.infra.ansible/roles/vault/meta/main.yaml @@ -0,0 +1,18 @@ +--- +# file: roles/vault/meta/main.yaml + +dependencies: [] + +galaxy_info: + role_name: vault + author: fd.io + description: Hashicorp Vault. + company: none + license: "license (Apache)" + min_ansible_version: 2.9 + platforms: + - name: Ubuntu + versions: + - jammy + galaxy_tags: + - vault diff --git a/fdio.infra.ansible/roles/vault/tasks/main.yaml b/fdio.infra.ansible/roles/vault/tasks/main.yaml new file mode 100644 index 0000000000..3fceadfb4a --- /dev/null +++ b/fdio.infra.ansible/roles/vault/tasks/main.yaml @@ -0,0 +1,133 @@ +--- +# file: roles/vault/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - vault-inst-prerequisites + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - vault-inst-prerequisites + +- name: Conf - Add Vault Group + ansible.builtin.group: + name: "{{ vault_group }}" + state: "{{ vault_user_state }}" + tags: + - vault-conf-user + +- name: Conf - Add Vault user + ansible.builtin.user: + name: "{{ vault_user }}" + group: "{{ vault_group }}" + state: "{{ vault_group_state }}" + system: true + tags: + - vault-conf-user + +- name: Inst - Clean Vault + ansible.builtin.file: + path: "{{ vault_inst_dir }}/vault" + state: "absent" + tags: + - vault-inst-package + +- name: Inst - Download Vault + ansible.builtin.get_url: + url: "{{ vault_zip_url }}" + dest: "{{ vault_inst_dir }}/{{ vault_pkg }}" + tags: + - vault-inst-package + +- name: Inst - Unarchive Vault + ansible.builtin.unarchive: + src: "{{ vault_inst_dir }}/{{ vault_pkg }}" + dest: "{{ vault_inst_dir }}/" + creates: "{{ vault_inst_dir }}/vault" + remote_src: true + tags: + - vault-inst-package + +- name: Inst - Vault + ansible.builtin.copy: + src: "{{ vault_inst_dir }}/vault" + dest: "{{ vault_bin_dir }}" + owner: "{{ vault_user }}" + group: "{{ vault_group }}" + force: true + mode: 0755 + remote_src: true + tags: + - vault-inst-package + +- name: Inst - Check Vault mlock capability + ansible.builtin.command: "setcap cap_ipc_lock=+ep {{ vault_bin_dir }}/vault" + changed_when: false # read-only task + ignore_errors: true + register: vault_mlock_capability + tags: + - vault-inst-package + +- name: Inst - Enable non root mlock capability + ansible.builtin.command: "setcap cap_ipc_lock=+ep {{ vault_bin_dir }}/vault" + when: vault_mlock_capability is failed + tags: + - vault-inst-package + +- name: Conf - Create directories + ansible.builtin.file: + dest: "{{ item }}" + state: directory + owner: "{{ vault_user }}" + group: "{{ vault_group }}" + mode: 0750 + with_items: + - "{{ vault_data_dir }}" + - "{{ vault_config_dir }}" + - "{{ vault_ssl_dir }}" + tags: + - vault-conf + +- name: Conf - Vault main configuration + ansible.builtin.template: + src: "{{ vault_main_configuration_template }}" + dest: "{{ vault_main_config }}" + owner: "{{ vault_user }}" + group: "{{ vault_group }}" + mode: 0400 + tags: + - vault-conf + +# - name: Conf - Copy Certificates And Keys +# copy: +# content: "{{ item.src }}" +# dest: "{{ item.dest }}" +# owner: "{{ vault_user }}" +# group: "{{ vault_group }}" +# mode: 0600 +# no_log: true +# loop: "{{ vault_certificates | flatten(levels=1) }}" +# tags: +# - vault-conf + +- name: Conf - System.d Script + ansible.builtin.template: + src: "vault_systemd.service.j2" + dest: "/lib/systemd/system/vault.service" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Restart Vault" + tags: + - vault-conf + +- meta: flush_handlers diff --git a/fdio.infra.ansible/roles/vault/templates/vault_backend_consul.j2 b/fdio.infra.ansible/roles/vault/templates/vault_backend_consul.j2 new file mode 100644 index 0000000000..c45498af90 --- /dev/null +++ b/fdio.infra.ansible/roles/vault/templates/vault_backend_consul.j2 @@ -0,0 +1,15 @@ +backend "consul" { + address = "{{ vault_consul }}" + path = "{{ vault_consul_path }}" + service = "{{ vault_consul_service }}" + {% if vault_consul_token is defined and vault_consul_token -%} + token = "{{ vault_consul_token }}" + {% endif -%} + scheme = "{{ vault_consul_scheme }}" + {% if vault_tls_gossip | bool -%} + tls_cert_file = "{{ vault_backend_tls_config_path }}/{{ vault_backend_tls_cert_file }}" + tls_key_file = "{{ vault_backend_tls_config_path }}/{{ vault_backend_tls_key_file }}" + tls_ca_file="{{ vault_backend_tls_config_path }}/{{ vault_backend_tls_ca_file }}" + {% endif %} + +}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/vault/templates/vault_main_configuration.hcl.j2 b/fdio.infra.ansible/roles/vault/templates/vault_main_configuration.hcl.j2 new file mode 100644 index 0000000000..dec4fff8d9 --- /dev/null +++ b/fdio.infra.ansible/roles/vault/templates/vault_main_configuration.hcl.j2 @@ -0,0 +1,93 @@ +cluster_name = "{{ vault_cluster_name }}" +max_lease_ttl = "{{ vault_max_lease_ttl }}" +default_lease_ttl = "{{ vault_default_lease_ttl }}" + +disable_clustering = "{{ vault_cluster_disable | bool | lower }}" +cluster_addr = "{{ vault_cluster_addr }}" +api_addr = "{{ vault_api_addr }}" + +{% for l in vault_tcp_listeners %} +listener "tcp" { + address = "{{ l.vault_address }}:{{ l.vault_port }}" + cluster_address = "{{ l.vault_cluster_address }}" + {% if (l.vault_proxy_protocol_behavior is defined and l.vault_proxy_protocol_behavior) -%} + proxy_protocol_behavior = "{{ l.vault_proxy_protocol_behavior }}" + {% if (l.vault_proxy_protocol_authorized_addrs is defined) -%} + proxy_protocol_authorized_addrs = "{{ l.vault_proxy_protocol_authorized_addrs }}" + {% endif -%} + {% endif -%} + {% if not (l.vault_tls_disable | bool) -%} + tls_cert_file = "{{ l.vault_tls_config_path }}/{{ l.vault_tls_cert_file }}" + tls_key_file = "{{ l.vault_tls_config_path }}/{{ l.vault_tls_key_file }}" + tls_client_ca_file="{{ l.vault_tls_config_path }}/{{ l.vault_tls_ca_file }}" + tls_min_version = "{{ l.vault_tls_min_version }}" + {% if vault_tls_cipher_suites is defined and vault_tls_cipher_suites -%} + tls_cipher_suites = "{{ l.vault_tls_cipher_suites}}" + {% endif -%} + tls_prefer_server_cipher_suites = "{{ l.vault_tls_prefer_server_cipher_suites }}" + {% if (l.vault_tls_require_and_verify_client_cert | bool) -%} + tls_require_and_verify_client_cert = "{{ l.vault_tls_require_and_verify_client_cert | bool | lower}}" + {% endif -%} + {% if (l.vault_tls_disable_client_certs | bool) -%} + tls_disable_client_certs = "{{ l.vault_tls_disable_client_certs | bool | lower}}" + {% endif -%} + {% endif -%} + tls_disable = "{{ l.vault_tls_disable | bool | lower }}" +} +{% endfor %} + +{% if (vault_listener_localhost_enable | bool) -%} +listener "tcp" { + address = "127.0.0.1:{{ vault_port }}" + cluster_address = "127.0.0.1:8201" + tls_disable = "true" +} +{% endif -%} + +{# + Select which storage backend you want generated and placed + in the vault configuration file. +#} +{%- if vault_backend == 'consul' -%} + {% include vault_backend_consul with context %} +{% elif vault_backend == 'etcd' -%} + {% include vault_backend_etcd with context %} +{% elif vault_backend == 'file' -%} + {% include vault_backend_file with context %} +{% elif vault_backend == 's3' -%} + {% include vault_backend_s3 with context %} +{% elif vault_backend == 'dynamodb' -%} + {% include vault_backend_dynamodb with context %} +{% elif vault_backend == 'mysql' -%} + {% include vault_backend_mysql with context %} +{% elif vault_backend == 'gcs' -%} + {% include vault_backend_gcs with context %} +{% elif vault_backend == 'raft' -%} + {% include vault_backend_raft with context %} +{% endif %} + +{% if vault_service_registration_consul_enable -%} + {% include vault_service_registration_consul_template with context %} +{% endif %} + +{% if vault_ui %} +ui = {{ vault_ui | bool | lower }} +{% endif %} + +{% if vault_telemetry_enabled | bool -%} +telemetry { + {% if vault_statsite_address is defined -%} + statsite_address = "{{vault_statsite_address}}" + {% endif -%} + {% if vault_statsd_address is defined -%} + statsd_address = "{{vault_statsd_address}}" + {% endif -%} + {% if vault_prometheus_retention_time is defined -%} + prometheus_retention_time = "{{ vault_prometheus_retention_time }}" + {% endif -%} + {% if vault_telemetry_disable_hostname is defined -%} + disable_hostname = {{vault_telemetry_disable_hostname | bool | lower }} + {% endif %} + +} +{% endif %}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/vault/templates/vault_service_registration_consul.hcl.j2 b/fdio.infra.ansible/roles/vault/templates/vault_service_registration_consul.hcl.j2 new file mode 100644 index 0000000000..cd5da1ffb6 --- /dev/null +++ b/fdio.infra.ansible/roles/vault/templates/vault_service_registration_consul.hcl.j2 @@ -0,0 +1,22 @@ +service_registration "consul" { + address = "{{ vault_service_registration_consul_address }}" + check_timeout = "{{ vault_service_registration_consul_check_timeout }}" + disable_registration = "{{ vault_service_registration_consul_disable_registration | bool | lower }}" + scheme = "{{ vault_service_registration_consul_scheme }}" + service = "{{ vault_service_registration_consul_service }}" + service_tags = "{{ vault_service_registration_consul_service_tags }}" + {% if vault_service_registration_consul_service_address is defined and vault_service_registration_consul_service_address -%} + service_address = "{{ vault_service_registration_consul_service_address }}" + {% endif -%} + {% if vault_service_registration_consul_token is defined and vault_service_registration_consul_token -%} + token = "{{ vault_service_registration_consul_token }}" + {% endif -%} + {% if vault_service_registration_consul_scheme == "https" -%} + tls_ca_file="{{ vault_service_registration_consul_tls_config_path }}/{{ vault_service_registration_consul_tls_ca_file }}" + tls_cert_file = "{{ vault_service_registration_consul_tls_config_path }}/{{ vault_service_registration_consul_tls_cert_file }}" + tls_key_file = "{{ vault_service_registration_consul_tls_config_path }}/{{ vault_service_registration_consul_tls_key_file }}" + tls_min_version = "{{ vault_service_registration_consul_tls_min_version }}" + tls_skip_verify = "{{ vault_service_registration_consul_tls_skip_verify }}" + {% endif %} + +}
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/vault/templates/vault_systemd.service.j2 b/fdio.infra.ansible/roles/vault/templates/vault_systemd.service.j2 new file mode 100644 index 0000000000..5d2ca78b2e --- /dev/null +++ b/fdio.infra.ansible/roles/vault/templates/vault_systemd.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Vault +Documentation=https://www.vaultproject.io/docs/ +Requires=network-online.target +After=network-online.target + +[Service] +User={{ vault_user }} +Group={{ vault_group }} +ProtectSystem=full +ProtectHome=read-only +PrivateTmp=yes +PrivateDevices=yes +NoNewPrivileges=yes +ExecReload=/bin/kill -HUP $MAINPID +ExecStart={{ vault_bin_dir }}/vault {{ vault_node_role }} -config={{ vault_config_dir }} +KillMode=process +KillSignal=SIGINT +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +StartLimitInterval=60 +StartLimitBurst=3 +LimitNOFILE=524288 +LimitNPROC=524288 +LimitMEMLOCK=infinity +LimitCORE=0 + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/vault/vars/main.yaml b/fdio.infra.ansible/roles/vault/vars/main.yaml new file mode 100644 index 0000000000..2b16a63fdf --- /dev/null +++ b/fdio.infra.ansible/roles/vault/vars/main.yaml @@ -0,0 +1,5 @@ +--- +# file: roles/vault/vars/main.yaml + +vault_node_client: "{{ (vault_node_role == 'client') or (vault_node_role == 'both') }}" +vault_node_server: "{{ (vault_node_role == 'server') or (vault_node_role == 'both') }}" diff --git a/fdio.infra.ansible/roles/vpp/defaults/main.yaml b/fdio.infra.ansible/roles/vpp/defaults/main.yaml new file mode 100644 index 0000000000..00c56859d0 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp/defaults/main.yaml @@ -0,0 +1,28 @@ +--- +# file: roles/vpp/defaults/main.yaml + +packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}" + +packages_base: + - "gdb" + - "libtool" + - "lxc" + - "pkg-config" + - "screen" + +packages_by_distro: + ubuntu: + jammy: + - "build-essential" + - "libglib2.0-dev" + - "libmbedcrypto7" + - "libmbedtls14" + - "libmbedx509-1" + - "libnuma-dev" + - "libpixman-1-dev" + +packages_by_arch: + aarch64: + - [] + x86_64: + - [] diff --git a/fdio.infra.ansible/roles/vpp/tasks/main.yaml b/fdio.infra.ansible/roles/vpp/tasks/main.yaml new file mode 100644 index 0000000000..cea06b764d --- /dev/null +++ b/fdio.infra.ansible/roles/vpp/tasks/main.yaml @@ -0,0 +1,27 @@ +--- +# file: roles/vpp/tasks/main.yaml + +- name: Inst - Update Package Cache (APT) + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: + - ansible_distribution|lower == 'ubuntu' + tags: + - vpp-inst-prerequisites + +- name: Inst - Prerequisites + ansible.builtin.package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + tags: + - vpp-inst-prerequisites + +- name: Conf - sysctl + ansible.builtin.file: + src: "/dev/null" + dest: "/etc/sysctl.d/80-vpp.conf" + state: "link" + become: true + tags: + - vpp-conf-sysctl diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh new file mode 100644 index 0000000000..cd04d61251 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 PANTHEON.tech and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to +# blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' ')) +# Add I350 Gigabit Network Connection 1521 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' ')) +# Add MT27800 Family [ConnectX-5] 1017 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' ')) +# Add MT2892 Family [ConnectX-6 Dx] 101d to whitelist. +PCI_WHITELIST+=($(lspci -Dmmd ':101d:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:01:00.0"]=0 +PF_INDICES["0000:01:00.1"]=1 +PF_INDICES["0003:02:00.0"]=0 +PF_INDICES["0003:02:00.1"]=1 +# Mellanox CX6 +PF_INDICES["0001:01:00.0"]=2 +PF_INDICES["0001:01:00.1"]=2
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh new file mode 100644 index 0000000000..91c93ab882 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add Intel Corporation Ethernet Controller 10G X550T to blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' ')) +# Add Intel Corporation Ethernet Controller E810-C for 100GbE QSFP to whitelist. +PCI_WHITELIST+=($(lspci -Dmmd ':1592:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:18:00.0"]=0 +PF_INDICES["0000:18:00.1"]=1 +PF_INDICES["0000:18:00.2"]=2 +PF_INDICES["0000:18:00.3"]=3 +PF_INDICES["0000:86:00.0"]=4 +PF_INDICES["0000:3b:00.0"]=0 +PF_INDICES["0000:3b:00.1"]=1 +PF_INDICES["0000:3b:00.2"]=2 +PF_INDICES["0000:3b:00.3"]=3 +PF_INDICES["0000:af:00.0"]=4 diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh new file mode 100644 index 0000000000..74593b24d4 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add Intel Corporation Ethernet Controller 10G X550T to blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' ')) +# Add Intel Corporation Ethernet Controller E810-C for 100GbE QSFP to whitelist. +PCI_WHITELIST+=($(lspci -Dmmd ':1592:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:2a:00.0"]=0 +PF_INDICES["0000:2a:00.1"]=1 +PF_INDICES["0000:2a:00.2"]=2 +PF_INDICES["0000:2a:00.3"]=3 +PF_INDICES["0000:bd:00.0"]=4 +PF_INDICES["0000:3d:00.0"]=0 +PF_INDICES["0000:3d:00.1"]=1 +PF_INDICES["0000:3d:00.2"]=2 +PF_INDICES["0000:3d:00.3"]=3 +PF_INDICES["0000:e1:00.0"]=4 + diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh new file mode 100644 index 0000000000..6c56752ad0 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 PANTHEON.tech and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to +# blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' ')) +# Add I350 Gigabit Network Connection 1521 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' ')) +# Add MT27800 Family [ConnectX-5] 1017 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:05:00.0"]=0 +PF_INDICES["0000:05:00.1"]=1 +PF_INDICES["0000:91:00.0"]=0 +PF_INDICES["0000:91:00.1"]=1 diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service new file mode 100644 index 0000000000..996792ab9b --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service @@ -0,0 +1,12 @@ +[Unit] +Description=CSIT Initialize SR-IOV VFs +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=True +ExecStart=/usr/local/bin/csit-initialize-vfs.sh start +ExecStop=/usr/local/bin/csit-initialize-vfs.sh stop + +[Install] +WantedBy=default.target diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh new file mode 100644 index 0000000000..afa84ae15a --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CSIT SRIOV VF initialization and isolation. + +set -euo pipefail + +SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh" + +# Initilize whitelisted NICs with maximum number of VFs. +pci_idx=0 +for pci_addr in ${PCI_WHITELIST[@]}; do + if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then + pci_path="/sys/bus/pci/devices/${pci_addr}" + # SR-IOV initialization + case "${1:-start}" in + "start" ) + if [ $(< "${pci_path}"/sriov_totalvfs) -gt 128 ] + then + sriov_totalvfs=128 + else + sriov_totalvfs=$(< "${pci_path}"/sriov_totalvfs) + fi + ;; + "stop" ) + sriov_totalvfs=0 + ;; + esac + echo ${sriov_totalvfs} > "${pci_path}"/sriov_numvfs + # SR-IOV 802.1Q isolation + case "${1:-start}" in + "start" ) + pf=$(basename "${pci_path}"/net/*) + for vf in $(seq "${sriov_totalvfs}"); do + # PCI address index in array (pairing siblings). + if [[ -n ${PF_INDICES[@]} ]] + then + vlan_pf_idx=${PF_INDICES[$pci_addr]} + else + vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) )) + fi + # 802.1Q base offset. + vlan_bs_off=1100 + # 802.1Q PF PCI address offset. + vlan_pf_off=$(( vlan_pf_idx * 100 + vlan_bs_off )) + # 802.1Q VF PCI address offset. + vlan_vf_off=$(( vlan_pf_off + vf - 1 )) + # VLAN string. + vlan_str="vlan ${vlan_vf_off}" + # MAC string. + mac5="$(printf '%x' ${pci_idx})" + mac6="$(printf '%x' $(( vf - 1 )))" + mac_str="mac ba:dc:0f:fe:${mac5}:${mac6}" + # Set 802.1Q VLAN id and MAC address + ip link set ${pf} vf $(( vf - 1 )) ${mac_str} ${vlan_str} + ip link set ${pf} vf $(( vf - 1 )) trust on + ip link set ${pf} vf $(( vf - 1 )) spoof off + sleep .5 + done + pci_idx=$(( pci_idx + 1 )) + ;; + esac + fi +done diff --git a/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml new file mode 100644 index 0000000000..3ac80cc16e --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml @@ -0,0 +1,21 @@ +--- +# file: handlers/main.yaml + +- name: "Start csit-initialize-vfs.service" + ansible.builtin.systemd: + enabled: true + state: "started" + name: "csit-initialize-vfs.service" + tags: + - start-vf-service + +- name: "Update GRUB" + ansible.builtin.command: "update-grub" + tags: + - update-grub + +- name: "Reboot server" + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml new file mode 100644 index 0000000000..91916456af --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml @@ -0,0 +1,139 @@ +--- +# file: tasks/main.yaml + +- name: "Load Kernel Modules On Startup (vfio-pci)" + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "vfio-pci" + tags: + - load-kernel-modules + +- name: "Disable IPv6 Router Advertisement" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.accept_ra" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 MLDv1 interval" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.mldv1_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 MLDv2 interval" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.mldv2_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 Autoconf" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.autoconf" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 MC Forwarding" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.mc_forwarding" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv4 IGMPv2 interval" + ansible.builtin.sysctl: + name: "net.ipv4.conf.default.igmpv2_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv4 IGMPv3 interval" + ansible.builtin.sysctl: + name: "net.ipv4.conf.default.igmpv3_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Copy csit-initialize-vfs.sh" + ansible.builtin.copy: + src: "files/csit-initialize-vfs.sh" + dest: "/usr/local/bin/" + owner: "root" + group: "root" + mode: 0744 + tags: + - copy-vf-script + +- name: "Copy csit-initialize-vfs-data.sh" + ansible.builtin.copy: + src: "files/{{ vfs_data_file }}" + dest: "/usr/local/bin/csit-initialize-vfs-data.sh" + owner: "root" + group: "root" + mode: 0744 + tags: copy-vf-data-script + when: + - vfs_data_file is defined + +- name: "Copy Default csit-initialize-vfs-data.sh" + ansible.builtin.copy: + src: "files/csit-initialize-vfs-default.sh" + dest: "/usr/local/bin/csit-initialize-vfs-data.sh" + owner: "root" + group: "root" + mode: 0744 + tags: copy-vf-data-script + when: + - vfs_data_file is not defined + +- name: "Start csit-initialize-vfs.service" + ansible.builtin.copy: + src: "files/csit-initialize-vfs.service" + dest: "/etc/systemd/system/" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Start csit-initialize-vfs.service" + tags: + - start-vf-service + +- ansible.builtin.meta: "flush_handlers" + +- name: "Set Hugepages In GRUB" + ansible.builtin.lineinfile: + path: "/etc/default/grub" + state: "present" + regexp: "^GRUB_CMDLINE_LINUX=" + line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value %}{{key}}={{value}} {% else %}{{key}} {% endif %}{% endfor %}\"" + notify: + - "Update GRUB" + tags: + - set-grub |