vpp_device: updates for 1n-tx2 testbeds

One ThunderX2 9975 server (.69) was replaced with two ThunderX2 9980
(.70, .71) servers. Move the .69 server under ansible perf section in
anticipation of repurposing it for that purpose. Update the ansible
scripts with .70 and .71 config and rename port names in device.sh lib
to reflect the NIC differences between .69 and .70 (and .71).

Change-Id: I88b75648735243e5559175d3192ffcc8fc70071c
Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>

8 files changed, 128 insertions, 42 deletions

diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh
index 1819c457bb..a94244e52c 100644
--- a/resources/libraries/bash/function/device.sh
+++ b/resources/libraries/bash/function/device.sh
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -276,16 +276,12 @@ function get_available_interfaces () {
             pci_id="0x154c"
             tg_netdev=(enp5s2 enp5s3 enp5s4 enp5s5
                        enp5s6 enp5s7 enp5s8 enp5s9)
-            tg_netdev+=(enp8s2 enp8s3 enp8s4 enp8s5
-                        enp8s6 enp8s7 enp8s8 enp8s9)
-            tg_netdev+=(enp8s10 enp8s11 enp8s12 enp8s13
-                        enp8s14 enp8s15 enp8s16 enp8s17)
-            dut1_netdev=(enp133s2 enp133s3 enp133s4 enp133s5
-                         enp133s6 enp133s7 enp133s8 enp133s9)
-            dut1_netdev+=(enp133s10 enp133s11 enp133s12 enp133s13
-                          enp133s14 enp133s15 enp133s16 enp133s17)
-            dut1_netdev+=(enp5s10 enp5s11 enp5s12 enp5s13
-                          enp5s14 enp5s15 enp5s16 enp5s17)
+            tg_netdev+=(enp5s10 enp5s11 enp5s12 enp5s13
+                        enp5s14 enp5s15 enp5s16 enp5s17)
+            dut1_netdev=(enp145s2 enp145s3 enp145s4 enp145s5
+                         enp145s6 enp145s7 enp145s8 enp145s9)
+            dut1_netdev+=(enp145s10 enp145s11 enp145s12 enp145s13
+                          enp145s14 enp145s15 enp145s16 enp145s17)
             ;;
        "1n_vbox")
             # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
index f330a19640..dfbe5a876b 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
@@ -2,10 +2,18 @@
 # file: host_vars/10.30.51.69.yaml
 
 hostname: "s27-t13-sut1"
-inventory_ipmi_hostname: "10.30.50.69"
-vfs_data_file: "csit-initialize-vfs-tx2.sh"
 grub:
+  isolcpus: "1-111,113-223"
+  nohz_full: "1-111,113-223"
+  rcu_nocbs: "1-111,113-223"
+sysctl:
+  kernel:
+  watchdog_cpumask: "0,112"
+  vm:
   nr_hugepages: 57344
+  max_map_count: 20000
+
+inventory_ipmi_hostname: "10.30.50.69"
 cpu_microarchitecture: "thunderx2"
 
 # User management.
@@ -22,24 +30,3 @@ users:
     ssh_key:
       - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
       - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXUbbhesOpvPF+KI8nq4TXvEO/Un1aU/ehZ9clCyw9C40xjDkX2BlcX8WeHxFHe7fjFaCd07Vg73rn/3M9uNDnDxvjH1GQ0twvi3iBTO4PkHBBfGF9qnE8MYzno6FvlsVKLuUuPbfm8kbOQ+ZDfdXq6gdtXh0hSYYkqC1heNPCNsqaakkB99Edyle+Ot0V7cpW+Yo2wo98KuX/cgUEhVoA8QnNVE7zaWcjSXBZEteoA4gLpAbV6p67/d6H/2ykHTidBViYTEsHco56tJoA4nTPuAupDOLBcWXgF5TAN6z1aCn2JA1DDfniLakgrZ5oVj2qHhUmbxQAtnKQfHADjqzV jlinkes@jlinkes"
-
-# Nomad settings.
-nomad_certificates:
-  - src: "{{ vault_nomad_v1_ca_file }}"
-    dest: "{{ nomad_ca_file }}"
-  - src: "{{ vault_nomad_v1_cert_file }}"
-    dest: "{{ nomad_cert_file }}"
-  - src: "{{ vault_nomad_v1_key_file }}"
-    dest: "{{ nomad_key_file }}"
-nomad_datacenter: "yul1"
-nomad_name: "{{ hostname }}-{{ ansible_architecture }}"
-nomad_node_role: "client"
-nomad_node_class: "csitarm"
-nomad_options:
-  driver.raw_exec.enable: 1
-  docker.cleanup.image: false
-  docker.privileged.enabled: true
-  docker.volumes.enabled: true
-  driver.whitelist: "docker,raw_exec,exec"
-nomad_servers: [ "10.30.51.32:4647", "10.30.51.33:4647" ]
-nomad_cpu_total_compute: "40000"
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml
new file mode 100644
index 0000000000..da603c28ce
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.70.yaml
@@ -0,0 +1,45 @@
+---
+# file: host_vars/10.30.51.70.yaml
+
+hostname: "s55-t36-sut1"
+inventory_ipmi_hostname: "10.30.50.70"
+vfs_data_file: "csit-initialize-vfs-tx2.sh"
+grub:
+  nr_hugepages: 65536
+cpu_microarchitecture: "thunderx2"
+
+# User management.
+users:
+  - username: localadmin
+    groups: [adm, sudo]
+    password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXUbbhesOpvPF+KI8nq4TXvEO/Un1aU/ehZ9clCyw9C40xjDkX2BlcX8WeHxFHe7fjFaCd07Vg73rn/3M9uNDnDxvjH1GQ0twvi3iBTO4PkHBBfGF9qnE8MYzno6FvlsVKLuUuPbfm8kbOQ+ZDfdXq6gdtXh0hSYYkqC1heNPCNsqaakkB99Edyle+Ot0V7cpW+Yo2wo98KuX/cgUEhVoA8QnNVE7zaWcjSXBZEteoA4gLpAbV6p67/d6H/2ykHTidBViYTEsHco56tJoA4nTPuAupDOLBcWXgF5TAN6z1aCn2JA1DDfniLakgrZ5oVj2qHhUmbxQAtnKQfHADjqzV jlinkes@jlinkes"
+  - username: testuser
+    groups: [adm, sudo]
+    password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXUbbhesOpvPF+KI8nq4TXvEO/Un1aU/ehZ9clCyw9C40xjDkX2BlcX8WeHxFHe7fjFaCd07Vg73rn/3M9uNDnDxvjH1GQ0twvi3iBTO4PkHBBfGF9qnE8MYzno6FvlsVKLuUuPbfm8kbOQ+ZDfdXq6gdtXh0hSYYkqC1heNPCNsqaakkB99Edyle+Ot0V7cpW+Yo2wo98KuX/cgUEhVoA8QnNVE7zaWcjSXBZEteoA4gLpAbV6p67/d6H/2ykHTidBViYTEsHco56tJoA4nTPuAupDOLBcWXgF5TAN6z1aCn2JA1DDfniLakgrZ5oVj2qHhUmbxQAtnKQfHADjqzV jlinkes@jlinkes"
+
+# Nomad settings.
+nomad_certificates:
+  - src: "{{ vault_nomad_v1_ca_file }}"
+    dest: "{{ nomad_ca_file }}"
+  - src: "{{ vault_nomad_v1_cert_file }}"
+    dest: "{{ nomad_cert_file }}"
+  - src: "{{ vault_nomad_v1_key_file }}"
+    dest: "{{ nomad_key_file }}"
+nomad_datacenter: "yul1"
+nomad_name: "{{ hostname }}-{{ ansible_architecture }}"
+nomad_node_role: "client"
+nomad_node_class: "csitarm"
+nomad_options:
+  driver.raw_exec.enable: 1
+  docker.cleanup.image: false
+  docker.privileged.enabled: true
+  docker.volumes.enabled: true
+  driver.whitelist: "docker,raw_exec,exec"
+nomad_servers: [ "10.30.51.32:4647", "10.30.51.33:4647" ]
+nomad_cpu_total_compute: "40000"
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml
new file mode 100644
index 0000000000..b5fb9cc4ec
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.71.yaml
@@ -0,0 +1,45 @@
+---
+# file: host_vars/10.30.51.71.yaml
+
+hostname: "s56-t37-sut1"
+inventory_ipmi_hostname: "10.30.50.71"
+vfs_data_file: "csit-initialize-vfs-tx2.sh"
+grub:
+  nr_hugepages: 65536
+cpu_microarchitecture: "thunderx2"
+
+# User management.
+users:
+  - username: localadmin
+    groups: [adm, sudo]
+    password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXUbbhesOpvPF+KI8nq4TXvEO/Un1aU/ehZ9clCyw9C40xjDkX2BlcX8WeHxFHe7fjFaCd07Vg73rn/3M9uNDnDxvjH1GQ0twvi3iBTO4PkHBBfGF9qnE8MYzno6FvlsVKLuUuPbfm8kbOQ+ZDfdXq6gdtXh0hSYYkqC1heNPCNsqaakkB99Edyle+Ot0V7cpW+Yo2wo98KuX/cgUEhVoA8QnNVE7zaWcjSXBZEteoA4gLpAbV6p67/d6H/2ykHTidBViYTEsHco56tJoA4nTPuAupDOLBcWXgF5TAN6z1aCn2JA1DDfniLakgrZ5oVj2qHhUmbxQAtnKQfHADjqzV jlinkes@jlinkes"
+  - username: testuser
+    groups: [adm, sudo]
+    password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXUbbhesOpvPF+KI8nq4TXvEO/Un1aU/ehZ9clCyw9C40xjDkX2BlcX8WeHxFHe7fjFaCd07Vg73rn/3M9uNDnDxvjH1GQ0twvi3iBTO4PkHBBfGF9qnE8MYzno6FvlsVKLuUuPbfm8kbOQ+ZDfdXq6gdtXh0hSYYkqC1heNPCNsqaakkB99Edyle+Ot0V7cpW+Yo2wo98KuX/cgUEhVoA8QnNVE7zaWcjSXBZEteoA4gLpAbV6p67/d6H/2ykHTidBViYTEsHco56tJoA4nTPuAupDOLBcWXgF5TAN6z1aCn2JA1DDfniLakgrZ5oVj2qHhUmbxQAtnKQfHADjqzV jlinkes@jlinkes"
+
+# Nomad settings.
+nomad_certificates:
+  - src: "{{ vault_nomad_v1_ca_file }}"
+    dest: "{{ nomad_ca_file }}"
+  - src: "{{ vault_nomad_v1_cert_file }}"
+    dest: "{{ nomad_cert_file }}"
+  - src: "{{ vault_nomad_v1_key_file }}"
+    dest: "{{ nomad_key_file }}"
+nomad_datacenter: "yul1"
+nomad_name: "{{ hostname }}-{{ ansible_architecture }}"
+nomad_node_role: "client"
+nomad_node_class: "csitarm"
+nomad_options:
+  driver.raw_exec.enable: 1
+  docker.cleanup.image: false
+  docker.privileged.enabled: true
+  docker.volumes.enabled: true
+  driver.whitelist: "docker,raw_exec,exec"
+nomad_servers: [ "10.30.51.32:4647", "10.30.51.33:4647" ]
+nomad_cpu_total_compute: "40000"
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
index b01053a5bf..ac1c46eb65 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
@@ -40,11 +40,13 @@ all:
         10.32.8.18: #s33-t27-sut1 - cascadelake
         10.32.8.20: #s35-t28-sut1 - cascadelake
         10.32.8.22: #s37-t29-sut1 - cascadelake
+        10.30.51.69: #s27-t13-sut1 - thunderx2 9975
     vpp_device:
       hosts:
         10.30.51.50: #s1-t11-sut1 - skylake
         10.30.51.51: #s2-t12-sut1 - skylake
-        10.30.51.69: #s27-t13-sut1 - thunderx2
+        10.30.51.70: #s55-t36-sut1 - thunderx2 9980
+        10.30.51.71: #s56-t37-sut1 - thunderx2 9980
     nomad:
       hosts:
         10.30.51.28: #s41-nomad - haswell
diff --git a/resources/tools/testbed-setup/ansible/roles/common/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/defaults/main.yaml
index f54bfce228..13db641423 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/defaults/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/defaults/main.yaml
@@ -38,7 +38,9 @@ packages_by_arch:
   aarch64:
     - "gfortran"
     - "libblas-dev"
+    - "libffi-dev"
     - "liblapack-dev"
+    - "libssl-dev"
   x86_64:
     - []
 
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index c3d16d8b9c..e60b32ae2a 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -35,7 +35,6 @@
       - "dill==0.2.8.2"
       - "numpy==1.17.3"
       - "hdrhistogram==0.6.1"
-      - "pandas==0.25.3"
       - "plotly==4.1.1"
       - "PTable==0.9.2"
       - "Sphinx==2.2.1"
@@ -76,9 +75,10 @@
   tags:
     - install-pip
 
-- name: Install CSIT PIP requirements - SciPy workaround
+- name: Install CSIT PIP requirements - Pandas and SciPy workaround
   pip:
     name:
+      - "pandas==0.25.3"
       - "scipy==1.1.0"
   tags:
     - install-pip
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
index 591c95a2be..b900efcf7c 100644
--- a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2019 PANTHEON.tech and/or its affiliates.
+# Copyright (c) 2020 PANTHEON.tech and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,16 +16,25 @@
 # Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to
 # blacklist.
 PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' '))
+# Add I350 Gigabit Network Connection 1521 to blacklist.
+PCI_BLACKLIST+=($(lspci -Dmmd ':1521:0200' | cut -f1 -d' '))
 
 # Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist.
 PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' '))
+# Add MT27800 Family [ConnectX-5] 1017 to whitelist.
+PCI_WHITELIST+=($(lspci -Dmmd ':1017:0200' | cut -f1 -d' '))
 
 # See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
 
 declare -A PF_INDICES
+# Intel NICs
 PF_INDICES["0000:05:00.0"]=0
 PF_INDICES["0000:05:00.1"]=1
-PF_INDICES["0000:08:00.0"]=2
-PF_INDICES["0000:08:00.1"]=1
-PF_INDICES["0000:85:00.0"]=0
-PF_INDICES["0000:85:00.1"]=2
+PF_INDICES["0000:91:00.0"]=0
+PF_INDICES["0000:91:00.1"]=1
+
+# Mellanox NICs
+PF_INDICES["0000:0b:00.0"]=2
+PF_INDICES["0000:0b:00.1"]=3
+PF_INDICES["0000:9a:00.0"]=2
+PF_INDICES["0000:9a:00.1"]=3