feat(pxe): Nvidia GRCoper-241014

Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch>
Change-Id: I41af1e40e66e5a429a274a6e7af40986273f1210

13 files changed, 6 insertions, 453 deletions

diff --git a/fdio.infra.ansible/inventories/lf_inventory/hosts b/fdio.infra.ansible/inventories/lf_inventory/hosts
index c5fe4a57fe..137b79bb33 100644
--- a/fdio.infra.ansible/inventories/lf_inventory/hosts
+++ b/fdio.infra.ansible/inventories/lf_inventory/hosts
@@ -40,12 +40,6 @@ all:
         10.30.51.93: #s93-t39-sut1 - snowridge
         10.30.51.94: #s94-t39-sut2 - snowridge
         10.32.8.24: #s60-t210-sut1 - epyc
-    vpp_device:
-      hosts:
-        10.30.51.30: #s30-t15-sut1 - sapphirerapids
-        10.30.51.31: #s31-t16-sut1 - sapphirerapids
-        10.30.51.70: #s70-t13-sut1 - altra
-        10.30.51.71: #s71-t14-sut1 - altra
     nomad:
       hosts:
         10.30.51.21: #s21-nomad - sapphirerapids
@@ -56,8 +50,12 @@ all:
         10.30.51.26: #s26-nomad - skylake
         10.30.51.27: #s27-nomad - skylake
         10.30.51.28: #s28-nomad - skylake
+        10.30.51.30: #s30-nomad - sapphirerapids
+        10.30.51.31: #s31-nomad - sapphirerapids
         10.30.51.50: #s50-nomad - skylake
         10.30.51.51: #s51-nomad - skylake
+        10.30.51.70: #s70-nomad - altra
+        10.30.51.71: #s71-nomad - altra
         10.30.51.91: #s58-nomad - neoverse n1
         10.30.51.92: #s59-nomad - neoverse n1
     dev:
diff --git a/fdio.infra.ansible/roles/calibration/defaults/main.yaml b/fdio.infra.ansible/roles/calibration/defaults/main.yaml
index 302f6d9663..28cecf26d3 100644
--- a/fdio.infra.ansible/roles/calibration/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/calibration/defaults/main.yaml
@@ -1,5 +1,5 @@
 ---
-# file: roles/calibration/defaults/main.yaml
+# file: defaults/main.yaml
 
 # Packages to install.
 packages: "{{ packages_base + packages_by_distro[ansible_distribution|lower][ansible_distribution_release] + packages_by_arch[ansible_machine] }}"
diff --git a/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml b/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml
index ca4e75d268..e01245556a 100644
--- a/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml
+++ b/fdio.infra.ansible/roles/calibration/tasks/aarch64.yaml
@@ -1,2 +1,2 @@
 ---
-# file: roles/calibration/tasks/aarch64.yaml
+# file: tasks/aarch64.yaml
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh
deleted file mode 100644
index cd04d61251..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2023 PANTHEON.tech and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to
-# blacklist.
-PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' '))
-# Add I350 Gigabit Network Connection 1521 to blacklist.
-PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' '))
-# Add MT27800 Family [ConnectX-5] 1017 to blacklist.
-PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' '))
-
-# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist.
-PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' '))
-# Add MT2892 Family [ConnectX-6 Dx] 101d to whitelist.
-PCI_WHITELIST+=($(lspci -Dmmd ':101d:0200' | cut -f1 -d' '))
-
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-
-declare -A PF_INDICES
-# Intel NICs
-PF_INDICES["0000:01:00.0"]=0
-PF_INDICES["0000:01:00.1"]=1
-PF_INDICES["0003:02:00.0"]=0
-PF_INDICES["0003:02:00.1"]=1
-# Mellanox CX6
-PF_INDICES["0001:01:00.0"]=2
-PF_INDICES["0001:01:00.1"]=2
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
deleted file mode 100644
index 91c93ab882..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2021 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
-PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
-
-# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
-PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
-# Add Intel Corporation Ethernet Controller E810-C for 100GbE QSFP to whitelist.
-PCI_WHITELIST+=($(lspci -Dmmd ':1592:0200' | cut -f1 -d' '))
-
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-
-declare -A PF_INDICES
-# Intel NICs
-PF_INDICES["0000:18:00.0"]=0
-PF_INDICES["0000:18:00.1"]=1
-PF_INDICES["0000:18:00.2"]=2
-PF_INDICES["0000:18:00.3"]=3
-PF_INDICES["0000:86:00.0"]=4
-PF_INDICES["0000:3b:00.0"]=0
-PF_INDICES["0000:3b:00.1"]=1
-PF_INDICES["0000:3b:00.2"]=2
-PF_INDICES["0000:3b:00.3"]=3
-PF_INDICES["0000:af:00.0"]=4
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh
deleted file mode 100644
index 74593b24d4..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2024 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
-PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
-
-# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
-PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
-# Add Intel Corporation Ethernet Controller E810-C for 100GbE QSFP to whitelist.
-PCI_WHITELIST+=($(lspci -Dmmd ':1592:0200' | cut -f1 -d' '))
-
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-
-declare -A PF_INDICES
-# Intel NICs
-PF_INDICES["0000:2a:00.0"]=0
-PF_INDICES["0000:2a:00.1"]=1
-PF_INDICES["0000:2a:00.2"]=2
-PF_INDICES["0000:2a:00.3"]=3
-PF_INDICES["0000:bd:00.0"]=4
-PF_INDICES["0000:3d:00.0"]=0
-PF_INDICES["0000:3d:00.1"]=1
-PF_INDICES["0000:3d:00.2"]=2
-PF_INDICES["0000:3d:00.3"]=3
-PF_INDICES["0000:e1:00.0"]=4
-
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
deleted file mode 100644
index 6c56752ad0..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2021 PANTHEON.tech and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to
-# blacklist.
-PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' '))
-# Add I350 Gigabit Network Connection 1521 to blacklist.
-PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' '))
-# Add MT27800 Family [ConnectX-5] 1017 to blacklist.
-PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' '))
-
-# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist.
-PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' '))
-
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-
-declare -A PF_INDICES
-# Intel NICs
-PF_INDICES["0000:05:00.0"]=0
-PF_INDICES["0000:05:00.1"]=1
-PF_INDICES["0000:91:00.0"]=0
-PF_INDICES["0000:91:00.1"]=1
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service
deleted file mode 100644
index 996792ab9b..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service
+++ /dev/null
@@ -1,12 +0,0 @@
-[Unit]
-Description=CSIT Initialize SR-IOV VFs
-After=network.target
-
-[Service]
-Type=oneshot
-RemainAfterExit=True
-ExecStart=/usr/local/bin/csit-initialize-vfs.sh start
-ExecStop=/usr/local/bin/csit-initialize-vfs.sh stop
-
-[Install]
-WantedBy=default.target
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh
deleted file mode 100644
index afa84ae15a..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2021 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# CSIT SRIOV VF initialization and isolation.
-
-set -euo pipefail
-
-SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
-source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh"
-
-# Initilize whitelisted NICs with maximum number of VFs.
-pci_idx=0
-for pci_addr in ${PCI_WHITELIST[@]}; do
-    if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then
-        pci_path="/sys/bus/pci/devices/${pci_addr}"
-        # SR-IOV initialization
-        case "${1:-start}" in
-            "start" )
-                if [ $(< "${pci_path}"/sriov_totalvfs) -gt 128 ]
-                then
-                    sriov_totalvfs=128
-                else
-                    sriov_totalvfs=$(< "${pci_path}"/sriov_totalvfs)
-                fi
-                ;;
-            "stop" )
-                sriov_totalvfs=0
-                ;;
-        esac
-        echo ${sriov_totalvfs} > "${pci_path}"/sriov_numvfs
-        # SR-IOV 802.1Q isolation
-        case "${1:-start}" in
-            "start" )
-                pf=$(basename "${pci_path}"/net/*)
-                for vf in $(seq "${sriov_totalvfs}"); do
-                    # PCI address index in array (pairing siblings).
-                    if [[ -n ${PF_INDICES[@]} ]]
-                    then
-                        vlan_pf_idx=${PF_INDICES[$pci_addr]}
-                    else
-                        vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) ))
-                    fi
-                    # 802.1Q base offset.
-                    vlan_bs_off=1100
-                    # 802.1Q PF PCI address offset.
-                    vlan_pf_off=$(( vlan_pf_idx * 100 + vlan_bs_off ))
-                    # 802.1Q VF PCI address offset.
-                    vlan_vf_off=$(( vlan_pf_off + vf - 1 ))
-                    # VLAN string.
-                    vlan_str="vlan ${vlan_vf_off}"
-                    # MAC string.
-                    mac5="$(printf '%x' ${pci_idx})"
-                    mac6="$(printf '%x' $(( vf - 1 )))"
-                    mac_str="mac ba:dc:0f:fe:${mac5}:${mac6}"
-                    # Set 802.1Q VLAN id and MAC address
-                    ip link set ${pf} vf $(( vf - 1 )) ${mac_str} ${vlan_str}
-                    ip link set ${pf} vf $(( vf - 1 )) trust on
-                    ip link set ${pf} vf $(( vf - 1 )) spoof off
-                    sleep .5
-                done
-                pci_idx=$(( pci_idx + 1 ))
-                ;;
-        esac
-    fi
-done
diff --git a/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml
deleted file mode 100644
index 3ac80cc16e..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-# file: handlers/main.yaml
-
-- name: "Start csit-initialize-vfs.service"
-  ansible.builtin.systemd:
-    enabled: true
-    state: "started"
-    name: "csit-initialize-vfs.service"
-  tags:
-    - start-vf-service
-
-- name: "Update GRUB"
-  ansible.builtin.command: "update-grub"
-  tags:
-    - update-grub
-
-- name: "Reboot server"
-  ansible.builtin.reboot:
-    reboot_timeout: 3600
-  tags:
-    - reboot-server
diff --git a/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml
deleted file mode 100644
index 91916456af..0000000000
--- a/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml
+++ /dev/null
@@ -1,139 +0,0 @@
----
-# file: tasks/main.yaml
-
-- name: "Load Kernel Modules On Startup (vfio-pci)"
-  ansible.builtin.lineinfile:
-    path: "/etc/modules"
-    state: "present"
-    line: "{{ item }}"
-  with_items:
-    - "vfio-pci"
-  tags:
-    - load-kernel-modules
-
-- name: "Disable IPv6 Router Advertisement"
-  ansible.builtin.sysctl:
-    name: "net.ipv6.conf.default.accept_ra"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Disable IPv6 MLDv1 interval"
-  ansible.builtin.sysctl:
-    name: "net.ipv6.conf.default.mldv1_unsolicited_report_interval"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Disable IPv6 MLDv2 interval"
-  ansible.builtin.sysctl:
-    name: "net.ipv6.conf.default.mldv2_unsolicited_report_interval"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Disable IPv6 Autoconf"
-  ansible.builtin.sysctl:
-    name: "net.ipv6.conf.default.autoconf"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Disable IPv6 MC Forwarding"
-  ansible.builtin.sysctl:
-    name: "net.ipv6.conf.default.mc_forwarding"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Disable IPv4 IGMPv2 interval"
-  ansible.builtin.sysctl:
-    name: "net.ipv4.conf.default.igmpv2_unsolicited_report_interval"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Disable IPv4 IGMPv3 interval"
-  ansible.builtin.sysctl:
-    name: "net.ipv4.conf.default.igmpv3_unsolicited_report_interval"
-    value: "0"
-    state: "present"
-    sysctl_file: "/etc/sysctl.d/90-csit.conf"
-    reload: "yes"
-  tags:
-    - set-sysctl
-
-- name: "Copy csit-initialize-vfs.sh"
-  ansible.builtin.copy:
-    src: "files/csit-initialize-vfs.sh"
-    dest: "/usr/local/bin/"
-    owner: "root"
-    group: "root"
-    mode: 0744
-  tags:
-    - copy-vf-script
-
-- name: "Copy csit-initialize-vfs-data.sh"
-  ansible.builtin.copy:
-    src: "files/{{ vfs_data_file }}"
-    dest: "/usr/local/bin/csit-initialize-vfs-data.sh"
-    owner: "root"
-    group: "root"
-    mode: 0744
-  tags: copy-vf-data-script
-  when:
-    - vfs_data_file is defined
-
-- name: "Copy Default csit-initialize-vfs-data.sh"
-  ansible.builtin.copy:
-    src: "files/csit-initialize-vfs-default.sh"
-    dest: "/usr/local/bin/csit-initialize-vfs-data.sh"
-    owner: "root"
-    group: "root"
-    mode: 0744
-  tags: copy-vf-data-script
-  when:
-    - vfs_data_file is not defined
-
-- name: "Start csit-initialize-vfs.service"
-  ansible.builtin.copy:
-    src: "files/csit-initialize-vfs.service"
-    dest: "/etc/systemd/system/"
-    owner: "root"
-    group: "root"
-    mode: 0644
-  notify:
-    - "Start csit-initialize-vfs.service"
-  tags:
-    - start-vf-service
-
-- ansible.builtin.meta: "flush_handlers"
-
-- name: "Set Hugepages In GRUB"
-  ansible.builtin.lineinfile:
-    path: "/etc/default/grub"
-    state: "present"
-    regexp: "^GRUB_CMDLINE_LINUX="
-    line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value %}{{key}}={{value}} {% else %}{{key}} {% endif %}{% endfor %}\""
-  notify:
-    - "Update GRUB"
-  tags:
-    - set-grub
diff --git a/fdio.infra.ansible/site.yaml b/fdio.infra.ansible/site.yaml
index 45a090344e..eee7940d54 100644
--- a/fdio.infra.ansible/site.yaml
+++ b/fdio.infra.ansible/site.yaml
@@ -15,10 +15,6 @@
     - sut_azure
     - sut_openstasck
 
-- import_playbook: vpp_device.yaml
-  tags:
-    - vpp-device
-
 - import_playbook: nomad.yaml
   tags:
     - nomad
diff --git a/fdio.infra.ansible/vpp_device.yaml b/fdio.infra.ansible/vpp_device.yaml
deleted file mode 100644
index 61c8016d0a..0000000000
--- a/fdio.infra.ansible/vpp_device.yaml
+++ /dev/null
@@ -1,44 +0,0 @@
----
-# file: vpp_device.yaml
-
-- hosts: vpp_device
-  remote_user: localadmin
-  become: true
-  become_user: root
-  gather_facts: false
-  pre_tasks:
-    - name: Gathering Facts
-      gather_facts:
-      tags:
-        - always
-  roles:
-    - role: user_add
-      tags: user_add
-    - role: baremetal
-      tags: baremetal
-    - role: common
-      tags: common
-#    - role: kernel
-#      tags: kernel
-    - role: intel
-      tags: intel
-    - role: mellanox
-      tags: mellanox
-    - role: docker
-      tags: docker
-    - role: docker_images
-      tags: docker_images
-    - role: nomad
-      tags: nomad
-    - role: consul
-      tags: consul
-    - role: prometheus_exporter
-      tags: prometheus_exporter
-    - role: jenkins_job_health_exporter
-      tags: jenkins_job_health_exporter
-    - role: vpp_device
-      tags: vpp_device
-    - role: kernel_vm
-      tags: kernel_vm
-    - role: cleanup
-      tags: cleanup