CSIT-1477: add 1n_tx2 VPP Device

- add 1n_tx2 testbed
- update VF reservation to meet 1n_tx2 testbed needs
- update ansible with 1n_tx2

Change-Id: Ia075a913d4859f537fd0e6bff731ea88aff01dd9
Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>

9 files changed, 138 insertions, 12 deletions

diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
new file mode 100644
index 0000000000..7d917c5e8b
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
@@ -0,0 +1,7 @@
+---
+# file: host_vars/10.30.51.69.yaml
+
+hostname: "s27-t13-sut1"
+vfs_data_file: "csit-initialize-vfs-tx2.sh"
+grub:
+  nr_hugepages: 57344
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
index 09bfc98ed9..80655bab27 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
@@ -51,6 +51,11 @@ all:
             10.32.8.11: null #s29-t26-sut1
             10.32.8.12: null #s30-t35-sut1
             10.32.8.13: null #s31-t35-sut2
+    thunderx2:
+      children:
+        vpp_device:
+          hosts:
+            10.30.51.69: null #s27-t13-sut1
     taishan:
       children:
 #        tg:
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64 b/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64
new file mode 100644
index 0000000000..38a2cc38e0
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64
@@ -0,0 +1,34 @@
+# If you change this file, run 'update-grub' afterwards to update
+# /boot/grub/grub.cfg.
+# For full documentation of the options in this file, see:
+#   info -f grub -n 'Simple configuration'
+
+GRUB_DEFAULT=0
+GRUB_TIMEOUT=10
+GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
+GRUB_CMDLINE_LINUX_DEFAULT="console=ttyAMA0,115200n8"
+GRUB_CMDLINE_LINUX=""
+
+# Uncomment to enable BadRAM filtering, modify to suit your needs
+# This works with Linux (no patch required) and with any kernel that obtains
+# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
+#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"
+
+# Uncomment to disable graphical terminal (grub-pc only)
+#GRUB_TERMINAL=console
+GRUB_TERMINAL=serial
+GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+
+# The resolution used on graphical terminal
+# note that you can use only modes which your graphic card supports via VBE
+# you can see them in real GRUB with the command `vbeinfo'
+#GRUB_GFXMODE=640x480
+
+# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
+#GRUB_DISABLE_LINUX_UUID=true
+
+# Uncomment to disable generation of recovery mode menu entries
+#GRUB_DISABLE_RECOVERY="true"
+
+# Uncomment to get a beep at grub start
+#GRUB_INIT_TUNE="480 440 1"
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/grub b/resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64
index d4e27b3f0f..d4e27b3f0f 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/files/grub
+++ b/resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 43e83105ce..96e3f83e1b 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -77,7 +77,7 @@
 
 - name: Copy grub file
   template:
-   src: 'files/grub'
+   src: 'files/grub_{{ ansible_machine }}'
    dest: '/etc/default/grub'
    owner: 'root'
    group: 'root'
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
new file mode 100644
index 0000000000..becc18f1a8
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
+PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
+PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
+
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
new file mode 100644
index 0000000000..591c95a2be
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 PANTHEON.tech and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to
+# blacklist.
+PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist.
+PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' '))
+
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
+
+declare -A PF_INDICES
+PF_INDICES["0000:05:00.0"]=0
+PF_INDICES["0000:05:00.1"]=1
+PF_INDICES["0000:08:00.0"]=2
+PF_INDICES["0000:08:00.1"]=1
+PF_INDICES["0000:85:00.0"]=0
+PF_INDICES["0000:85:00.1"]=2
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
index 06b9923128..365497865e 100644
--- a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -17,18 +17,13 @@
 
 set -euo pipefail
 
-# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-pci_blacklist=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
-
-# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-pci_whitelist=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
+SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh"
 
 # Initilize whitelisted NICs with maximum number of VFs.
 pci_idx=0
-for pci_addr in ${pci_whitelist[@]}; do
-    if ! [[ ${pci_blacklist[*]} =~ "${pci_addr}" ]]; then
+for pci_addr in ${PCI_WHITELIST[@]}; do
+    if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then
         pci_path="/sys/bus/pci/devices/${pci_addr}"
         # SR-IOV initialization
         case "${1:-start}" in
@@ -46,7 +41,12 @@ for pci_addr in ${pci_whitelist[@]}; do
                 pf=$(basename "${pci_path}"/net/*)
                 for vf in $(seq "${sriov_totalvfs}"); do
                     # PCI address index in array (pairing siblings).
-                    vlan_pf_idx=$(( pci_idx % (${#pci_whitelist[@]} / 2) ))
+                    if [[ -n ${PF_INDICES[@]} ]]
+                    then
+                        vlan_pf_idx=${PF_INDICES[$pci_addr]}
+                    else
+                        vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) ))
+                    fi
                     # 802.1Q base offset.
                     vlan_bs_off=1100
                     # 802.1Q PF PCI address offset.
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
index 93c5d36bc2..46ca1d746b 100644
--- a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
@@ -1,6 +1,13 @@
 ---
 # file: roles/vpp_device/tasks/main.yaml
 
+- name: Load vfio-pci by default
+  lineinfile:
+    path: '/etc/modules'
+    state: 'present'
+    line: 'vfio-pci'
+  tags: load-vfio-pci
+
 - name: Copy csit-initialize-vfs.sh
   copy:
    src: 'files/csit-initialize-vfs.sh'
@@ -10,6 +17,26 @@
    mode: '744'
   tags: copy-vf-script
 
+- name: Copy csit-initialize-vfs-data.sh
+  copy:
+   src: 'files/{{ vfs_data_file }}'
+   dest: '/usr/local/bin/csit-initialize-vfs-data.sh'
+   owner: 'root'
+   group: 'root'
+   mode: '744'
+  tags: copy-vf-data-script
+  when: vfs_data_file is defined
+
+- name: Copy default csit-initialize-vfs-data.sh
+  copy:
+   src: 'files/csit-initialize-vfs-default.sh'
+   dest: '/usr/local/bin/csit-initialize-vfs-data.sh'
+   owner: 'root'
+   group: 'root'
+   mode: '744'
+  tags: copy-vf-data-script
+  when: vfs_data_file is not defined
+
 - name: Start csit-initialize-vfs.service
   copy:
    src: 'files/csit-initialize-vfs.service'