aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjuraj.linkes <juraj.linkes@pantheon.tech>2019-01-31 15:32:24 +0100
committerPeter Mikus <pmikus@cisco.com>2019-07-11 19:16:47 +0000
commit46efda9bd3a32cfe9944cb421ae3e2a3978bf9fc (patch)
treeb26ab38c18a0eb3105dbd3687331a5120a4fcb3a
parentf8452762b3dd6370792a5891b1010c263140bae6 (diff)
CSIT-1477: add 1n_tx2 VPP Device
- add 1n_tx2 testbed - update VF reservation to meet 1n_tx2 testbed needs - update ansible with 1n_tx2 Change-Id: Ia075a913d4859f537fd0e6bff731ea88aff01dd9 Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>
-rw-r--r--resources/libraries/bash/function/common.sh12
-rw-r--r--resources/libraries/bash/function/device.sh88
-rw-r--r--resources/libraries/python/Constants.py3
-rw-r--r--resources/libraries/python/ContainerUtils.py3
-rw-r--r--resources/libraries/robot/shared/container.robot3
-rw-r--r--resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml7
-rw-r--r--resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts5
-rw-r--r--resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch6434
-rw-r--r--resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64 (renamed from resources/tools/testbed-setup/ansible/roles/common/files/grub)0
-rw-r--r--resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml2
-rw-r--r--resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh22
-rw-r--r--resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh31
-rw-r--r--resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh22
-rw-r--r--resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml27
14 files changed, 202 insertions, 57 deletions
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 078ed70197..f1f02850e9 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -46,7 +46,7 @@ function activate_docker_topology () {
device_image="$(< ${CSIT_DIR}/${IMAGE_VER_FILE})"
case_text="${NODENESS}_${FLAVOR}"
case "${case_text}" in
- "1n_skx")
+ "1n_skx" | "1n_tx2")
# We execute reservation over csit-shim-dcr (ssh) which runs sourced
# script's functions. Env variables are read from ssh output
# back to localhost for further processing.
@@ -319,7 +319,7 @@ function deactivate_docker_topology () {
case_text="${NODENESS}_${FLAVOR}"
case "${case_text}" in
- "1n_skx")
+ "1n_skx" | "1n_tx2")
hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3) || die
ssh="ssh root@${hostname} -p 6022"
env_vars=$(env | grep CSIT_ | tr '\n' ' ' ) || die
@@ -436,6 +436,10 @@ function get_test_code () {
NODENESS="1n"
FLAVOR="skx"
;;
+ *"1n-tx2"*)
+ NODENESS="1n"
+ FLAVOR="tx2"
+ ;;
*"2n-skx"*)
NODENESS="2n"
FLAVOR="skx"
@@ -824,7 +828,7 @@ function select_vpp_device_tags () {
TAGS=()
- # We will prefix with perftest to prevent running other tests
+ # We will prefix with devicetest to prevent running other tests
# (e.g. Functional).
prefix="devicetestAND"
if [[ "${TEST_CODE}" == "vpp-"* ]]; then
@@ -895,7 +899,7 @@ function select_topology () {
TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
TOPOLOGIES_TAGS="2_node_single_link_topo"
;;
- "1n_skx")
+ "1n_skx" | "1n_tx2")
TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template )
TOPOLOGIES_TAGS="2_node_single_link_topo"
;;
diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh
index 0eda008abb..5d33af342e 100644
--- a/resources/libraries/bash/function/device.sh
+++ b/resources/libraries/bash/function/device.sh
@@ -270,7 +270,24 @@ function get_available_interfaces () {
tg_netdev=(enp24)
dut1_netdev=(enp59)
;;
- "1n_vbox")
+ "1n_tx2")
+ # Add Intel Corporation XL710/X710 Virtual Function to the
+ # whitelist.
+ pci_id="0x154c"
+ tg_netdev=(enp5s2 enp5s3 enp5s4 enp5s5
+ enp5s6 enp5s7 enp5s8 enp5s9)
+ tg_netdev+=(enp8s2 enp8s3 enp8s4 enp8s5
+ enp8s6 enp8s7 enp8s8 enp8s9)
+ tg_netdev+=(enp8s10 enp8s11 enp8s12 enp8s13
+ enp8s14 enp8s15 enp8s16 enp8s17)
+ dut1_netdev=(enp133s2 enp133s3 enp133s4 enp133s5
+ enp133s6 enp133s7 enp133s8 enp133s9)
+ dut1_netdev+=(enp133s10 enp133s11 enp133s12 enp133s13
+ enp133s14 enp133s15 enp133s16 enp133s17)
+ dut1_netdev+=(enp5s10 enp5s11 enp5s12 enp5s13
+ enp5s14 enp5s15 enp5s16 enp5s17)
+ ;;
+ "1n_vbox")
# Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
# whitelist.
pci_id="0x100f"
@@ -281,7 +298,7 @@ function get_available_interfaces () {
die "Unknown specification: ${case_text}!"
esac
- net_path="/sys/bus/pci/devices/*/net/*"
+ device_count=2
# TG side of connections.
TG_NETDEVS=()
@@ -294,47 +311,40 @@ function get_available_interfaces () {
DUT1_NETMACS=()
DUT1_DRIVERS=()
- # Following code is filtering available VFs represented by network device
- # name. Only allowed VFs PCI IDs are used.
- for netdev in \
- $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';');
+ # Find the first ${device_count} number of available TG Linux network
+ # VF device names. Only allowed VF PCI IDs are filtered.
+ for netdev in ${tg_netdev[@]}
do
- if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then
- # We will filter to TG/DUT1 side of connection (this can be in
- # future overriden by more advanced conditions for mapping).
- for sub in ${tg_netdev[@]}; do
- if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
- tg_side+=(${netdev})
- fi
- done
- for sub in ${dut1_netdev[@]}; do
- if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
- dut1_side+=(${netdev})
- fi
- done
+ for netdev_path in $(grep -l "${pci_id}" \
+ /sys/class/net/${netdev}*/device/device \
+ 2> /dev/null)
+ do
+ if [[ ${#TG_NETDEVS[@]} -lt ${device_count} ]]; then
+ tg_netdev_name=$(dirname ${netdev_path})
+ tg_netdev_name=$(dirname ${tg_netdev_name})
+ TG_NETDEVS+=($(basename ${tg_netdev_name}))
+ else
+ break
+ fi
+ done
+ if [[ ${#TG_NETDEVS[@]} -eq ${device_count} ]]; then
+ break
fi
done
- case "${case_text}" in
- "1n_skx")
- # Pick up first two DUT1 interfaces binded to i40evf.
- for netdev in "${dut1_side[@]::2}"; do
- DUT1_NETDEVS+=(${netdev})
- done
- # Corresponding TG interfaces will be same ID.SUB_ID, but on
- # opposite linked device.
- for netdev in "${DUT1_NETDEVS[@]}"; do
- TG_NETDEVS+=(${netdev/$dut1_netdev/$tg_netdev})
- done
- ;;
- *)
- for netdev in "${tg_side[@]::2}"; do
- TG_NETDEVS+=(${netdev})
- done
- for netdev in "${dut1_side[@]::2}"; do
- DUT1_NETDEVS+=(${netdev})
- done
- esac
+ i=0
+ for netdev in "${TG_NETDEVS[@]}"; do
+ # Find the index of selected tg netdev among tg_netdevs
+ # e.g. enp8s5f7 is a vf of netdev enp8s5 with index 11
+ # and the corresponding dut1 netdev is enp133s13.
+ while [[ "${netdev}" != "${tg_netdev[$i]}"* ]]; do
+ ((i++))
+ done
+ # Rename tg netdev to dut1 netdev
+ # e.g. enp8s5f7 -> enp133s13f7
+ DUT1_NETDEVS+=(${netdev/${tg_netdev[$i]}/${dut1_netdev[$i]}})
+ # Don't need to reset i, all netdevs are sorted.
+ done
for NETDEV in "${TG_NETDEVS[@]}"; do
get_pci_addr
diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index cb66a5d8d5..877fc25012 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -67,6 +67,9 @@ class Constants(object):
# QEMU VM DPDK path
QEMU_VM_DPDK = '/opt/dpdk-19.02'
+ # Docker container SUT image
+ DOCKER_SUT_IMAGE_UBUNTU = 'snergster/csit-sut:latest'
+
# TRex install version
TREX_INSTALL_VERSION = '2.54'
diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py
index 7d04b06fba..228648921a 100644
--- a/resources/libraries/python/ContainerUtils.py
+++ b/resources/libraries/python/ContainerUtils.py
@@ -784,7 +784,8 @@ class Docker(ContainerEngine):
return
if not self.container.image:
- setattr(self.container, 'image', 'snergster/csit-sut:latest')
+ setattr(self.container, 'image',
+ Constants.DOCKER_SUT_IMAGE_UBUNTU)
cmd = 'docker pull {image}'.format(image=self.container.image)
diff --git a/resources/libraries/robot/shared/container.robot b/resources/libraries/robot/shared/container.robot
index 6067a81c4e..1d646a6f22 100644
--- a/resources/libraries/robot/shared/container.robot
+++ b/resources/libraries/robot/shared/container.robot
@@ -56,12 +56,13 @@
| | | ${root}= | Run Keyword If | ${dut1_uuid_length}
| | | ... | Get Docker Mergeddir | ${nodes['DUT1']} | ${dut1_uuid}
| | | ... | ELSE | Set Variable | ${EMPTY}
+| | | ${node_arch}= | Get Node Arch | ${nodes['${dut}']}
| | | ${mnt}= | Create List
| | | ... | ${root}/tmp/:/mnt/host/
| | | ... | ${root}/dev/vfio/:/dev/vfio/
| | | ... | ${root}/usr/bin/vpp:/usr/bin/vpp
| | | ... | ${root}/usr/bin/vppctl:/usr/bin/vppctl
-| | | ... | ${root}/usr/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/
+| | | ... | ${root}/usr/lib/${node_arch}-linux-gnu/:/usr/lib/${node_arch}-linux-gnu/
| | | ... | ${root}/usr/share/vpp/:/usr/share/vpp/
| | | ${nf_cpus}= | Set Variable | ${None}
| | | ${nf_cpus}= | Run Keyword If | ${pinning}
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
new file mode 100644
index 0000000000..7d917c5e8b
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml
@@ -0,0 +1,7 @@
+---
+# file: host_vars/10.30.51.69.yaml
+
+hostname: "s27-t13-sut1"
+vfs_data_file: "csit-initialize-vfs-tx2.sh"
+grub:
+ nr_hugepages: 57344
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
index 09bfc98ed9..80655bab27 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
@@ -51,6 +51,11 @@ all:
10.32.8.11: null #s29-t26-sut1
10.32.8.12: null #s30-t35-sut1
10.32.8.13: null #s31-t35-sut2
+ thunderx2:
+ children:
+ vpp_device:
+ hosts:
+ 10.30.51.69: null #s27-t13-sut1
taishan:
children:
# tg:
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64 b/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64
new file mode 100644
index 0000000000..38a2cc38e0
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64
@@ -0,0 +1,34 @@
+# If you change this file, run 'update-grub' afterwards to update
+# /boot/grub/grub.cfg.
+# For full documentation of the options in this file, see:
+# info -f grub -n 'Simple configuration'
+
+GRUB_DEFAULT=0
+GRUB_TIMEOUT=10
+GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
+GRUB_CMDLINE_LINUX_DEFAULT="console=ttyAMA0,115200n8"
+GRUB_CMDLINE_LINUX=""
+
+# Uncomment to enable BadRAM filtering, modify to suit your needs
+# This works with Linux (no patch required) and with any kernel that obtains
+# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
+#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"
+
+# Uncomment to disable graphical terminal (grub-pc only)
+#GRUB_TERMINAL=console
+GRUB_TERMINAL=serial
+GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+
+# The resolution used on graphical terminal
+# note that you can use only modes which your graphic card supports via VBE
+# you can see them in real GRUB with the command `vbeinfo'
+#GRUB_GFXMODE=640x480
+
+# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
+#GRUB_DISABLE_LINUX_UUID=true
+
+# Uncomment to disable generation of recovery mode menu entries
+#GRUB_DISABLE_RECOVERY="true"
+
+# Uncomment to get a beep at grub start
+#GRUB_INIT_TUNE="480 440 1"
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/grub b/resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64
index d4e27b3f0f..d4e27b3f0f 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/files/grub
+++ b/resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 43e83105ce..96e3f83e1b 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -77,7 +77,7 @@
- name: Copy grub file
template:
- src: 'files/grub'
+ src: 'files/grub_{{ ansible_machine }}'
dest: '/etc/default/grub'
owner: 'root'
group: 'root'
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
new file mode 100644
index 0000000000..becc18f1a8
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
+PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
+PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
+
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
new file mode 100644
index 0000000000..591c95a2be
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 PANTHEON.tech and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to
+# blacklist.
+PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' '))
+
+# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist.
+PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' '))
+
+# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
+
+declare -A PF_INDICES
+PF_INDICES["0000:05:00.0"]=0
+PF_INDICES["0000:05:00.1"]=1
+PF_INDICES["0000:08:00.0"]=2
+PF_INDICES["0000:08:00.1"]=1
+PF_INDICES["0000:85:00.0"]=0
+PF_INDICES["0000:85:00.1"]=2
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
index 06b9923128..365497865e 100644
--- a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -17,18 +17,13 @@
set -euo pipefail
-# Add Intel Corporation Ethernet Controller 10G X550T to blacklist.
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-pci_blacklist=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' '))
-
-# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist.
-# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
-pci_whitelist=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' '))
+SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh"
# Initilize whitelisted NICs with maximum number of VFs.
pci_idx=0
-for pci_addr in ${pci_whitelist[@]}; do
- if ! [[ ${pci_blacklist[*]} =~ "${pci_addr}" ]]; then
+for pci_addr in ${PCI_WHITELIST[@]}; do
+ if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then
pci_path="/sys/bus/pci/devices/${pci_addr}"
# SR-IOV initialization
case "${1:-start}" in
@@ -46,7 +41,12 @@ for pci_addr in ${pci_whitelist[@]}; do
pf=$(basename "${pci_path}"/net/*)
for vf in $(seq "${sriov_totalvfs}"); do
# PCI address index in array (pairing siblings).
- vlan_pf_idx=$(( pci_idx % (${#pci_whitelist[@]} / 2) ))
+ if [[ -n ${PF_INDICES[@]} ]]
+ then
+ vlan_pf_idx=${PF_INDICES[$pci_addr]}
+ else
+ vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) ))
+ fi
# 802.1Q base offset.
vlan_bs_off=1100
# 802.1Q PF PCI address offset.
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
index 93c5d36bc2..46ca1d746b 100644
--- a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml
@@ -1,6 +1,13 @@
---
# file: roles/vpp_device/tasks/main.yaml
+- name: Load vfio-pci by default
+ lineinfile:
+ path: '/etc/modules'
+ state: 'present'
+ line: 'vfio-pci'
+ tags: load-vfio-pci
+
- name: Copy csit-initialize-vfs.sh
copy:
src: 'files/csit-initialize-vfs.sh'
@@ -10,6 +17,26 @@
mode: '744'
tags: copy-vf-script
+- name: Copy csit-initialize-vfs-data.sh
+ copy:
+ src: 'files/{{ vfs_data_file }}'
+ dest: '/usr/local/bin/csit-initialize-vfs-data.sh'
+ owner: 'root'
+ group: 'root'
+ mode: '744'
+ tags: copy-vf-data-script
+ when: vfs_data_file is defined
+
+- name: Copy default csit-initialize-vfs-data.sh
+ copy:
+ src: 'files/csit-initialize-vfs-default.sh'
+ dest: '/usr/local/bin/csit-initialize-vfs-data.sh'
+ owner: 'root'
+ group: 'root'
+ mode: '744'
+ tags: copy-vf-data-script
+ when: vfs_data_file is not defined
+
- name: Start csit-initialize-vfs.service
copy:
src: 'files/csit-initialize-vfs.service'