CSIT-1292 Update Ansible playbooks with vpp_device tasks

Change-Id: I22d1fe2fc27d6adb39eba2578e68d06992a3f976
Signed-off-by: Peter Mikus <pmikus@cisco.com>

7 files changed, 122 insertions, 3 deletions

diff --git a/resources/tools/testbed-setup/ansible/production b/resources/tools/testbed-setup/ansible/production
index 8e5ff62564..f2364031b6 100644
--- a/resources/tools/testbed-setup/ansible/production
+++ b/resources/tools/testbed-setup/ansible/production
@@ -41,7 +41,7 @@ all:
             10.30.51.56: null #s9-t24-sut1
             10.30.51.58: null #s14-t32-sut1
             10.30.51.59: null #s15-t32-sut2
-        vppdevice:
+        vpp_device:
           hosts:
             10.30.51.50: null #s1-t11-sut1
             10.30.51.51: null #s2-t12-sut1
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.service b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.service
new file mode 100644
index 0000000000..996792ab9b
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=CSIT Initialize SR-IOV VFs
+After=network.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=True
+ExecStart=/usr/local/bin/csit-initialize-vfs.sh start
+ExecStop=/usr/local/bin/csit-initialize-vfs.sh stop
+
+[Install]
+WantedBy=default.target
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
new file mode 100644
index 0000000000..666839c707
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CSIT SRIOV VF initialization and isolation.
+
+set -euo pipefail
+
+# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
+pci_blacklist=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
+pci_whitelist=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
+
+# Initilize whitelisted NICs with maximum number of VFs.
+pci_idx=0
+for pci_addr in ${pci_whitelist[@]}; do
+    if ! [[ ${pci_blacklist[*]} =~ "${pci_addr}" ]]; then
+        pci_path="/sys/bus/pci/devices/${pci_addr}"
+        # SR-IOV initialization
+        case "${1:-start}" in
+            "start" )
+                sriov_totalvfs=$(< "${pci_path}"/sriov_totalvfs)
+                ;;
+            "stop" )
+                sriov_totalvfs=0
+                ;;
+        esac
+        echo ${sriov_totalvfs} > "${pci_path}"/sriov_numvfs
+        # SR-IOV 802.1Q isolation
+        case "${1:-start}" in
+            "start" )
+                pf=$(basename "${pci_path}"/net/*)
+                for vf in $(seq "${sriov_totalvfs}"); do
+                    # PCI address index in array (pairing siblings).
+                    vlan_pf_idx=$(( pci_idx % (${#pci_whitelist[@]} / 2) ))
+                    # 802.1Q base offset.
+                    vlan_bs_off=1100
+                    # 802.1Q PF PCI address offset.
+                    vlan_pf_off=$(( vlan_pf_idx * 100 + vlan_bs_off ))
+                    # 802.1Q VF PCI address offset.
+                    vlan_vf_off=$(( vlan_pf_off + vf - 1 ))
+                    # VLAN string.
+                    vlan_str="vlan ${vlan_vf_off}"
+                    # MAC string.
+                    mac5="$(printf '%x' ${pci_idx})"
+                    mac6="$(printf '%x' $(( vf - 1 )))"
+                    mac_str="mac ba:dc:0f:fe:${mac5}:${mac6}"
+                    # Set 802.1Q VLAN id and MAC address
+                    ip link set ${pf} vf $(( vf - 1 )) ${mac_str} ${vlan_str}
+                done
+                pci_idx=$(( pci_idx + 1 ))
+                ;;
+        esac
+    fi
+done
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml
new file mode 100644
index 0000000000..e2add5cecf
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml
@@ -0,0 +1,7 @@
+---
+# file: roles/vpp_device/handlers/main.yaml
+
+- name: Enable csit-initialize-vfs.service
+  command: systemctl enable csit-initialize-vfs.service
+  tags: enable-vf-service
+
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
new file mode 100644
index 0000000000..d1ede0cfbb
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
@@ -0,0 +1,21 @@
+---
+# file: roles/vpp_device/tasks/main.yaml
+
+- name: Copy csit-initialize-vfs.sh
+  copy:
+   src: 'files/csit-initialize-vfs.sh'
+   dest: '/usr/local/bin/'
+   owner: 'root'
+   group: 'root'
+   mode: '744'
+  tags: copy-vf-script
+
+- name: Copy csit-initialize-vfs.service
+  copy:
+   src: 'files/csit-initialize-vfs.service'
+   dest: '/etc/systemd/system/'
+   owner: 'root'
+   group: 'root'
+   mode: '644'
+  notify: ['Enable csit-initialize-vfs.service']
+  tags: copy-vf-service
diff --git a/resources/tools/testbed-setup/ansible/site.yaml b/resources/tools/testbed-setup/ansible/site.yaml
index 4a8a7f045a..1f59505168 100644
--- a/resources/tools/testbed-setup/ansible/site.yaml
+++ b/resources/tools/testbed-setup/ansible/site.yaml
@@ -7,8 +7,8 @@
 - import_playbook: sut.yaml
   tags: sut
 
-#- import_playbook: vppdevice.yaml
-#  tags: vppdevice
+- import_playbook: vpp_device.yaml
+  tags: vpp-device
 
 #- import_playbook: virl.yaml
 #  tags: virl
diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml
new file mode 100644
index 0000000000..7f0f6eb91a
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml
@@ -0,0 +1,10 @@
+---
+# file: vpp_device.yaml
+
+- hosts: vpp_device
+  remote_user: testuser
+  become: yes
+  become_user: root
+  roles:
+    - common
+    - vpp_device