diff options
Diffstat (limited to 'fdio.infra.ansible/roles/vpp_device')
8 files changed, 397 insertions, 0 deletions
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh new file mode 100644 index 0000000000..cd04d61251 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-alt.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 PANTHEON.tech and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to +# blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' ')) +# Add I350 Gigabit Network Connection 1521 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' ')) +# Add MT27800 Family [ConnectX-5] 1017 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' ')) +# Add MT2892 Family [ConnectX-6 Dx] 101d to whitelist. +PCI_WHITELIST+=($(lspci -Dmmd ':101d:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:01:00.0"]=0 +PF_INDICES["0000:01:00.1"]=1 +PF_INDICES["0003:02:00.0"]=0 +PF_INDICES["0003:02:00.1"]=1 +# Mellanox CX6 +PF_INDICES["0001:01:00.0"]=2 +PF_INDICES["0001:01:00.1"]=2
\ No newline at end of file diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh new file mode 100644 index 0000000000..91c93ab882 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add Intel Corporation Ethernet Controller 10G X550T to blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' ')) +# Add Intel Corporation Ethernet Controller E810-C for 100GbE QSFP to whitelist. +PCI_WHITELIST+=($(lspci -Dmmd ':1592:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:18:00.0"]=0 +PF_INDICES["0000:18:00.1"]=1 +PF_INDICES["0000:18:00.2"]=2 +PF_INDICES["0000:18:00.3"]=3 +PF_INDICES["0000:86:00.0"]=4 +PF_INDICES["0000:3b:00.0"]=0 +PF_INDICES["0000:3b:00.1"]=1 +PF_INDICES["0000:3b:00.2"]=2 +PF_INDICES["0000:3b:00.3"]=3 +PF_INDICES["0000:af:00.0"]=4 diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh new file mode 100644 index 0000000000..74593b24d4 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-spr.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright (c) 2024 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add Intel Corporation Ethernet Controller 10G X550T to blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' ')) +# Add Intel Corporation Ethernet Controller E810-C for 100GbE QSFP to whitelist. +PCI_WHITELIST+=($(lspci -Dmmd ':1592:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:2a:00.0"]=0 +PF_INDICES["0000:2a:00.1"]=1 +PF_INDICES["0000:2a:00.2"]=2 +PF_INDICES["0000:2a:00.3"]=3 +PF_INDICES["0000:bd:00.0"]=4 +PF_INDICES["0000:3d:00.0"]=0 +PF_INDICES["0000:3d:00.1"]=1 +PF_INDICES["0000:3d:00.2"]=2 +PF_INDICES["0000:3d:00.3"]=3 +PF_INDICES["0000:e1:00.0"]=4 + diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh new file mode 100644 index 0000000000..6c56752ad0 --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 PANTHEON.tech and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to +# blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' ')) +# Add I350 Gigabit Network Connection 1521 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' ')) +# Add MT27800 Family [ConnectX-5] 1017 to blacklist. +PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +# Intel NICs +PF_INDICES["0000:05:00.0"]=0 +PF_INDICES["0000:05:00.1"]=1 +PF_INDICES["0000:91:00.0"]=0 +PF_INDICES["0000:91:00.1"]=1 diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service new file mode 100644 index 0000000000..996792ab9b --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service @@ -0,0 +1,12 @@ +[Unit] +Description=CSIT Initialize SR-IOV VFs +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=True +ExecStart=/usr/local/bin/csit-initialize-vfs.sh start +ExecStop=/usr/local/bin/csit-initialize-vfs.sh stop + +[Install] +WantedBy=default.target diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh new file mode 100644 index 0000000000..afa84ae15a --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CSIT SRIOV VF initialization and isolation. + +set -euo pipefail + +SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh" + +# Initilize whitelisted NICs with maximum number of VFs. +pci_idx=0 +for pci_addr in ${PCI_WHITELIST[@]}; do + if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then + pci_path="/sys/bus/pci/devices/${pci_addr}" + # SR-IOV initialization + case "${1:-start}" in + "start" ) + if [ $(< "${pci_path}"/sriov_totalvfs) -gt 128 ] + then + sriov_totalvfs=128 + else + sriov_totalvfs=$(< "${pci_path}"/sriov_totalvfs) + fi + ;; + "stop" ) + sriov_totalvfs=0 + ;; + esac + echo ${sriov_totalvfs} > "${pci_path}"/sriov_numvfs + # SR-IOV 802.1Q isolation + case "${1:-start}" in + "start" ) + pf=$(basename "${pci_path}"/net/*) + for vf in $(seq "${sriov_totalvfs}"); do + # PCI address index in array (pairing siblings). + if [[ -n ${PF_INDICES[@]} ]] + then + vlan_pf_idx=${PF_INDICES[$pci_addr]} + else + vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) )) + fi + # 802.1Q base offset. + vlan_bs_off=1100 + # 802.1Q PF PCI address offset. + vlan_pf_off=$(( vlan_pf_idx * 100 + vlan_bs_off )) + # 802.1Q VF PCI address offset. + vlan_vf_off=$(( vlan_pf_off + vf - 1 )) + # VLAN string. + vlan_str="vlan ${vlan_vf_off}" + # MAC string. + mac5="$(printf '%x' ${pci_idx})" + mac6="$(printf '%x' $(( vf - 1 )))" + mac_str="mac ba:dc:0f:fe:${mac5}:${mac6}" + # Set 802.1Q VLAN id and MAC address + ip link set ${pf} vf $(( vf - 1 )) ${mac_str} ${vlan_str} + ip link set ${pf} vf $(( vf - 1 )) trust on + ip link set ${pf} vf $(( vf - 1 )) spoof off + sleep .5 + done + pci_idx=$(( pci_idx + 1 )) + ;; + esac + fi +done diff --git a/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml new file mode 100644 index 0000000000..3ac80cc16e --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml @@ -0,0 +1,21 @@ +--- +# file: handlers/main.yaml + +- name: "Start csit-initialize-vfs.service" + ansible.builtin.systemd: + enabled: true + state: "started" + name: "csit-initialize-vfs.service" + tags: + - start-vf-service + +- name: "Update GRUB" + ansible.builtin.command: "update-grub" + tags: + - update-grub + +- name: "Reboot server" + ansible.builtin.reboot: + reboot_timeout: 3600 + tags: + - reboot-server diff --git a/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml new file mode 100644 index 0000000000..91916456af --- /dev/null +++ b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml @@ -0,0 +1,139 @@ +--- +# file: tasks/main.yaml + +- name: "Load Kernel Modules On Startup (vfio-pci)" + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "vfio-pci" + tags: + - load-kernel-modules + +- name: "Disable IPv6 Router Advertisement" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.accept_ra" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 MLDv1 interval" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.mldv1_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 MLDv2 interval" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.mldv2_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 Autoconf" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.autoconf" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv6 MC Forwarding" + ansible.builtin.sysctl: + name: "net.ipv6.conf.default.mc_forwarding" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv4 IGMPv2 interval" + ansible.builtin.sysctl: + name: "net.ipv4.conf.default.igmpv2_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Disable IPv4 IGMPv3 interval" + ansible.builtin.sysctl: + name: "net.ipv4.conf.default.igmpv3_unsolicited_report_interval" + value: "0" + state: "present" + sysctl_file: "/etc/sysctl.d/90-csit.conf" + reload: "yes" + tags: + - set-sysctl + +- name: "Copy csit-initialize-vfs.sh" + ansible.builtin.copy: + src: "files/csit-initialize-vfs.sh" + dest: "/usr/local/bin/" + owner: "root" + group: "root" + mode: 0744 + tags: + - copy-vf-script + +- name: "Copy csit-initialize-vfs-data.sh" + ansible.builtin.copy: + src: "files/{{ vfs_data_file }}" + dest: "/usr/local/bin/csit-initialize-vfs-data.sh" + owner: "root" + group: "root" + mode: 0744 + tags: copy-vf-data-script + when: + - vfs_data_file is defined + +- name: "Copy Default csit-initialize-vfs-data.sh" + ansible.builtin.copy: + src: "files/csit-initialize-vfs-default.sh" + dest: "/usr/local/bin/csit-initialize-vfs-data.sh" + owner: "root" + group: "root" + mode: 0744 + tags: copy-vf-data-script + when: + - vfs_data_file is not defined + +- name: "Start csit-initialize-vfs.service" + ansible.builtin.copy: + src: "files/csit-initialize-vfs.service" + dest: "/etc/systemd/system/" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Start csit-initialize-vfs.service" + tags: + - start-vf-service + +- ansible.builtin.meta: "flush_handlers" + +- name: "Set Hugepages In GRUB" + ansible.builtin.lineinfile: + path: "/etc/default/grub" + state: "present" + regexp: "^GRUB_CMDLINE_LINUX=" + line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value %}{{key}}={{value}} {% else %}{{key}} {% endif %}{% endfor %}\"" + notify: + - "Update GRUB" + tags: + - set-grub |