aboutsummaryrefslogtreecommitdiffstats
path: root/fdio.infra.ansible/roles/vpp_device
diff options
context:
space:
mode:
Diffstat (limited to 'fdio.infra.ansible/roles/vpp_device')
-rw-r--r--fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh22
-rw-r--r--fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh34
-rw-r--r--fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service12
-rw-r--r--fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh73
-rw-r--r--fdio.infra.ansible/roles/vpp_device/handlers/main.yaml21
-rw-r--r--fdio.infra.ansible/roles/vpp_device/tasks/main.yaml92
6 files changed, 254 insertions, 0 deletions
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
new file mode 100644
index 0000000000..d0fc772037
--- /dev/null
+++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
+PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
+PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
+
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
new file mode 100644
index 0000000000..6c56752ad0
--- /dev/null
+++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 PANTHEON.tech and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to
+# blacklist.
+PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' '))
+# Add I350 Gigabit Network Connection 1521 to blacklist.
+PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' '))
+# Add MT27800 Family [ConnectX-5] 1017 to blacklist.
+PCI_BLACKLIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist.
+PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' '))
+
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
+
+declare -A PF_INDICES
+# Intel NICs
+PF_INDICES["0000:05:00.0"]=0
+PF_INDICES["0000:05:00.1"]=1
+PF_INDICES["0000:91:00.0"]=0
+PF_INDICES["0000:91:00.1"]=1
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service
new file mode 100644
index 0000000000..996792ab9b
--- /dev/null
+++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=CSIT Initialize SR-IOV VFs
+After=network.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=True
+ExecStart=/usr/local/bin/csit-initialize-vfs.sh start
+ExecStop=/usr/local/bin/csit-initialize-vfs.sh stop
+
+[Install]
+WantedBy=default.target
diff --git a/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh
new file mode 100644
index 0000000000..393e997d65
--- /dev/null
+++ b/fdio.infra.ansible/roles/vpp_device/files/csit-initialize-vfs.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CSIT SRIOV VF initialization and isolation.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh"
+
+# Initilize whitelisted NICs with maximum number of VFs.
+pci_idx=0
+for pci_addr in ${PCI_WHITELIST[@]}; do
+ if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then
+ pci_path="/sys/bus/pci/devices/${pci_addr}"
+ # SR-IOV initialization
+ case "${1:-start}" in
+ "start" )
+ sriov_totalvfs=$(< "${pci_path}"/sriov_totalvfs)
+ ;;
+ "stop" )
+ sriov_totalvfs=0
+ ;;
+ esac
+ echo ${sriov_totalvfs} > "${pci_path}"/sriov_numvfs
+ # SR-IOV 802.1Q isolation
+ case "${1:-start}" in
+ "start" )
+ pf=$(basename "${pci_path}"/net/*)
+ for vf in $(seq "${sriov_totalvfs}"); do
+ # PCI address index in array (pairing siblings).
+ if [[ -n ${PF_INDICES[@]} ]]
+ then
+ vlan_pf_idx=${PF_INDICES[$pci_addr]}
+ else
+ vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) ))
+ fi
+ # 802.1Q base offset.
+ vlan_bs_off=1100
+ # 802.1Q PF PCI address offset.
+ vlan_pf_off=$(( vlan_pf_idx * 100 + vlan_bs_off ))
+ # 802.1Q VF PCI address offset.
+ vlan_vf_off=$(( vlan_pf_off + vf - 1 ))
+ # VLAN string.
+ vlan_str="vlan ${vlan_vf_off}"
+ # MAC string.
+ mac5="$(printf '%x' ${pci_idx})"
+ mac6="$(printf '%x' $(( vf - 1 )))"
+ mac_str="mac ba:dc:0f:fe:${mac5}:${mac6}"
+ # Set 802.1Q VLAN id and MAC address
+ ip link set ${pf} vf $(( vf - 1 )) ${mac_str} ${vlan_str}
+ ip link set ${pf} vf $(( vf - 1 )) trust on
+ ip link set ${pf} vf $(( vf - 1 )) spoof off
+ done
+ pci_idx=$(( pci_idx + 1 ))
+ ;;
+ esac
+ rmmod i40evf
+ modprobe i40evf
+ fi
+done
diff --git a/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml
new file mode 100644
index 0000000000..ee9d368638
--- /dev/null
+++ b/fdio.infra.ansible/roles/vpp_device/handlers/main.yaml
@@ -0,0 +1,21 @@
+---
+# file: roles/vpp_device/handlers/main.yaml
+
+- name: Start csit-initialize-vfs.service
+ systemd:
+ enabled: yes
+ state: started
+ name: csit-initialize-vfs.service
+ tags:
+ - start-vf-service
+
+- name: Update GRUB
+ command: update-grub
+ tags:
+ - update-grub
+
+- name: Reboot server
+ reboot:
+ reboot_timeout: 3600
+ tags:
+ - reboot-server
diff --git a/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml
new file mode 100644
index 0000000000..33b551715f
--- /dev/null
+++ b/fdio.infra.ansible/roles/vpp_device/tasks/main.yaml
@@ -0,0 +1,92 @@
+---
+# file: roles/vpp_device/tasks/main.yaml
+
+- name: VPP_device - Load Kernel Modules By Default
+ lineinfile:
+ path: "/etc/modules"
+ state: "present"
+ line: "{{ item }}"
+ with_items:
+ - "vfio-pci"
+ tags:
+ - load-kernel-modules
+
+- name: VPP_device - Disable ipv6 router advertisement
+ sysctl:
+ name: "net.ipv6.conf.default.accept_ra"
+ value: "0"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - set-sysctl
+
+- name: VPP_device - Disable ipv6 router advertisement
+ sysctl:
+ name: "net.ipv6.conf.all.accept_ra"
+ value: "0"
+ state: "present"
+ sysctl_file: "/etc/sysctl.d/90-csit.conf"
+ reload: "yes"
+ tags:
+ - set-sysctl
+
+- name: VPP_device - Copy csit-initialize-vfs.sh
+ copy:
+ src: "files/csit-initialize-vfs.sh"
+ dest: "/usr/local/bin/"
+ owner: "root"
+ group: "root"
+ mode: "744"
+ tags:
+ - copy-vf-script
+
+- name: VPP_device - Copy csit-initialize-vfs-data.sh
+ copy:
+ src: "files/{{ vfs_data_file }}"
+ dest: "/usr/local/bin/csit-initialize-vfs-data.sh"
+ owner: "root"
+ group: "root"
+ mode: "744"
+ tags: copy-vf-data-script
+ when:
+ - vfs_data_file is defined
+
+- name: VPP_device - Copy default csit-initialize-vfs-data.sh
+ copy:
+ src: "files/csit-initialize-vfs-default.sh"
+ dest: "/usr/local/bin/csit-initialize-vfs-data.sh"
+ owner: "root"
+ group: "root"
+ mode: "744"
+ tags: copy-vf-data-script
+ when:
+ - vfs_data_file is not defined
+
+- name: VPP_device - Start csit-initialize-vfs.service
+ copy:
+ src: "files/csit-initialize-vfs.service"
+ dest: "/etc/systemd/system/"
+ owner: "root"
+ group: "root"
+ mode: "644"
+ notify:
+ - "Start csit-initialize-vfs.service"
+ tags:
+ - start-vf-service
+
+- meta: flush_handlers
+
+- name: VPP_device - Set hugepages in GRUB
+ lineinfile:
+ path: "/etc/default/grub"
+ state: "present"
+ regexp: "^GRUB_CMDLINE_LINUX="
+ line: "GRUB_CMDLINE_LINUX=\"{% for key, value in grub.items() %}{% if value %}{{key}}={{value}} {% else %}{{key}} {% endif %}{% endfor %}\""
+ notify:
+ - "Update GRUB"
+ - "Reboot server"
+ tags:
+ - set-grub
+
+- meta: flush_handlers