diff options
Diffstat (limited to 'resources')
| -rw-r--r-- | resources/libraries/bash/entry/bootstrap_verify_perf.sh | 2 | ||||
| -rwxr-xr-x | resources/libraries/bash/entry/bootstrap_vpp_device.sh | 46 | ||||
| -rw-r--r-- | resources/libraries/bash/entry/per_patch_perf.sh | 4 | ||||
| -rw-r--r-- | resources/libraries/bash/function/common.sh | 220 | ||||
| -rw-r--r-- | resources/libraries/bash/function/device.sh | 570 |
5 files changed, 826 insertions, 16 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index 74edc4cea8..1a0d638da1 100644 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -34,8 +34,8 @@ source "${BASH_FUNCTION_DIR}/common.sh" || { } source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed." common_dirs || die -get_test_tag_string || die get_test_code "${1-}" || die +get_test_tag_string || die select_topology || die gather_build || die check_download_dir || die diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh new file mode 100755 index 0000000000..6d389d6baf --- /dev/null +++ b/resources/libraries/bash/entry/bootstrap_vpp_device.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# Assumptions: +# + There is a directory holding CSIT code to use (this script is there). +# + At least one of the following is true: +# ++ JOB_NAME environment variable is set, +# ++ or this entry script has access to arguments. +# Consequences (and specific assumptions) are multiple, +# examine tree of functions for current description. + +# "set -eu" handles failures from the following two lines. +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed." +common_dirs || die +get_test_code "${1-}" || die +get_test_tag_string || die +select_topology || die +gather_build || die +check_download_dir || die +activate_virtualenv "${CSIT_DIR}" || die +activate_docker_topology || die +select_vpp_device_tags || die +compose_pybot_arguments || die +run_pybot || die +copy_archives || die +die_on_pybot_error || die diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh index abb1001652..73d4f69500 100644 --- a/resources/libraries/bash/entry/per_patch_perf.sh +++ b/resources/libraries/bash/entry/per_patch_perf.sh @@ -46,10 +46,10 @@ prepare_build_parent || die build_vpp_ubuntu_amd64 "PARENT" || die prepare_test || die ## Replace previous 4 lines with this to speed up testing. -#download_builds "https://jenkins.fd.io/sandbox/job/vpp-csit-verify-perf-master-2n-skx/2/artifact/*zip*/archive.zip" || die +#download_builds "REPLACE_WITH_URL" || die initialize_csit_dirs || die -get_test_tag_string || die get_test_code "${1-}" || die +get_test_tag_string || die set_perpatch_dut || die select_topology || die activate_virtualenv "${VPP_DIR}" || die diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 69c70935c7..db3d86592e 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -21,6 +21,71 @@ set -exuo pipefail # the code might become more readable (but longer). +function activate_docker_topology () { + # Create virtual vpp-device topology. Output of the function is topology + # file describing created environment saved to a file. + # + # Variables read: + # - BASH_FUNCTION_DIR - Path to existing directory this file is located in. + # - TOPOLOGIES - Available topologies. + # - NODENESS - Node multiplicity of desired testbed. + # - FLAVOR - Node flavor string, usually describing the processor. + # Variables set: + # - WORKING_TOPOLOGY - Path to topology file. + + set -exuo pipefail + + source "${BASH_FUNCTION_DIR}/device.sh" || { + die "Source failed!" + } + + device_image="$(< ${CSIT_DIR}/VPP_DEVICE_IMAGE)" + case_text="${NODENESS}_${FLAVOR}" + case "${case_text}" in + "1n_skx") + # We execute reservation over csit-shim-dcr (ssh) which runs sourced + # script's functions. Env variables are read from ssh output + # back to localhost for further processing. + hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3) + ssh="ssh root@${hostname} -p 6022" + run="activate_wrapper ${NODENESS} ${FLAVOR} ${device_image}" + env_vars=$(${ssh} "$(declare -f); ${run}") || { + die "Topology reservation via shim-dcr failed!" + } + set -a + source <(echo "$env_vars" | grep -v /usr/bin/docker) || { + die "Source failed!" + } + set +a + ;; + "1n_vbox") + # We execute reservation on localhost. Sourced script automatially + # sets environment variables for further processing. + activate_wrapper "${NODENESS}" "${FLAVOR}" "${device_image}" || die + ;; + *) + die "Unknown specification: ${case_text}!" + esac + + trap 'deactivate_docker_topology' EXIT || { + die "Trap attempt failed, please cleanup manually. Aborting!" + } + + # Replace all variables in template with those in environment. + source <(echo 'cat <<EOF >topo.yml'; cat ${TOPOLOGIES[0]}; echo EOF;) || { + die "Topology file create failed!" + } + + WORKING_TOPOLOGY="/tmp/topology.yaml" + mv topo.yml "${WORKING_TOPOLOGY}" || { + die "Topology move failed!" + } + cat ${WORKING_TOPOLOGY} | grep -v password || { + die "Topology read failed!" + } +} + + function activate_virtualenv () { set -exuo pipefail @@ -154,13 +219,28 @@ function compose_pybot_arguments () { # - DUT - CSIT test/ subdirectory, set while processing tags. # - TAGS - Array variable holding selected tag boolean expressions. # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology. + # - TEST_CODE - The test selection string from environment or argument. # Variables set: # - PYBOT_ARGS - String holding part of all arguments for pybot. # - EXPANDED_TAGS - Array of strings pybot arguments compiled from tags. # No explicit check needed with "set -u". - PYBOT_ARGS=("--loglevel" "TRACE" "--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}") - PYBOT_ARGS+=("--suite" "tests.${DUT}.perf") + PYBOT_ARGS=("--loglevel" "TRACE") + PYBOT_ARGS+=("--variable" "TOPOLOGY_PATH:${WORKING_TOPOLOGY}") + + case "${TEST_CODE}" in + *"device"*) + PYBOT_ARGS+=("--suite" "tests.${DUT}.device") + ;; + *"func"*) + PYBOT_ARGS+=("--suite" "tests.${DUT}.func") + ;; + *"perf"*) + PYBOT_ARGS+=("--suite" "tests.${DUT}.perf") + ;; + *) + die "Unknown specification: ${TEST_CODE}" + esac EXPANDED_TAGS=() for tag in "${TAGS[@]}"; do @@ -197,6 +277,38 @@ function copy_archives () { } +function deactivate_docker_topology () { + # Deactivate virtual vpp-device topology by removing containers. + # + # Variables read: + # - NODENESS - Node multiplicity of desired testbed. + # - FLAVOR - Node flavor string, usually describing the processor. + + set -exuo pipefail + + case_text="${NODENESS}_${FLAVOR}" + case "${case_text}" in + "1n_skx") + hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3) + ssh="ssh root@${hostname} -p 6022" + env_vars="$(env | grep CSIT_ | tr '\n' ' ' )" + ${ssh} "$(declare -f); deactivate_wrapper ${env_vars}" || { + die "Topology cleanup via shim-dcr failed!" + } + ;; + "1n_vbox") + enter_mutex || die + clean_environment || { + die "Topology cleanup locally failed!" + } + exit_mutex || die + ;; + *) + die "Unknown specification: ${case_text}!" + esac +} + + function die () { # Print the message to standard error end exit with error code specified # by the second argument. @@ -251,6 +363,14 @@ function get_test_code () { fi case "${TEST_CODE}" in + *"1n-vbox"*) + NODENESS="1n" + FLAVOR="vbox" + ;; + *"1n-skx"*) + NODENESS="1n" + FLAVOR="skx" + ;; *"2n-skx"*) NODENESS="2n" FLAVOR="skx" @@ -275,6 +395,7 @@ function get_test_tag_string () { # Variables read: # - GERRIT_EVENT_TYPE - Event type set by gerrit, can be unset. # - GERRIT_EVENT_COMMENT_TEXT - Comment text, read for "comment-added" type. + # - TEST_CODE - The test selection string from environment or argument. # Variables set: # - TEST_TAG_STRING - The string following "perftest" in gerrit comment, # or empty. @@ -283,12 +404,27 @@ function get_test_tag_string () { trigger="" if [[ "${GERRIT_EVENT_TYPE-}" == "comment-added" ]]; then - # On parsing error, ${trigger} stays empty. - trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \ - | grep -oE '(perftest$|perftest[[:space:]].+$)')" || true + case "${TEST_CODE}" in + *"device"*) + # On parsing error, ${trigger} stays empty. + trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \ + | grep -oE '(devicetest$|devicetest[[:space:]].+$)')" \ + || true + # Set test tags as string. + TEST_TAG_STRING="${trigger#$"devicetest"}" + ;; + *"perf"*) + # On parsing error, ${trigger} stays empty. + trigger="$(echo "${GERRIT_EVENT_COMMENT_TEXT}" \ + | grep -oE '(perftest$|perftest[[:space:]].+$)')" \ + || true + # Set test tags as string. + TEST_TAG_STRING="${trigger#$"perftest"}" + ;; + *) + die "Unknown specification: ${TEST_CODE}" + esac fi - # Set test tags as string. - TEST_TAG_STRING="${trigger#$"perftest"}" } @@ -565,6 +701,52 @@ function select_tags () { } +function select_vpp_device_tags () { + + set -exuo pipefail + + # Variables read: + # - TEST_CODE - String affecting test selection, usually jenkins job name. + # - TEST_TAG_STRING - String selecting tags, from gerrit comment. + # Can be unset. + # Variables set: + # - TAGS - Array of processed tag boolean expressions. + + case "${TEST_CODE}" in + # Select specific performance tests based on jenkins job type variable. + * ) + if [[ -z "${TEST_TAG_STRING-}" ]]; then + # If nothing is specified, we will run pre-selected tests by + # following tags. Items of array will be concatenated by OR + # in Robot Framework. + test_tag_array=() + else + # If trigger contains tags, split them into array. + test_tag_array=(${TEST_TAG_STRING//:/ }) + fi + ;; + esac + + TAGS=() + + # We will prefix with perftest to prevent running other tests + # (e.g. Functional). + prefix="devicetestAND" + if [[ "${TEST_CODE}" == "vpp-"* ]]; then + # Automatic prefixing for VPP jobs to limit testing. + prefix="${prefix}" + fi + for tag in "${test_tag_array[@]}"; do + if [[ ${tag} == "!"* ]]; then + # Exclude tags are not prefixed. + TAGS+=("${tag}") + else + TAGS+=("${prefix}${tag}") + fi + done +} + + function select_topology () { set -exuo pipefail @@ -573,7 +755,7 @@ function select_topology () { # - NODENESS - Node multiplicity of testbed, either "2n" or "3n". # - FLAVOR - Node flavor string, currently either "hsw" or "skx". # - CSIT_DIR - Path to existing root of local CSIT git repository. - # - TOPOLOGIES_DIR - Path to existing directory with available tpologies. + # - TOPOLOGIES_DIR - Path to existing directory with available topologies. # Variables set: # - TOPOLOGIES - Array of paths to suitable topology yaml files. # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology. @@ -582,13 +764,17 @@ function select_topology () { case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in - "3n_hsw") + "1n_vbox") TOPOLOGIES=( - "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml" - "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml" - "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml" + "${TOPOLOGIES_DIR}/vpp_device.template" ) - TOPOLOGIES_TAGS="3_node_single_link_topo" + TOPOLOGIES_TAGS="2_node_single_link_topo" + ;; + "1n_skx") + TOPOLOGIES=( + "${TOPOLOGIES_DIR}/vpp_device.template" + ) + TOPOLOGIES_TAGS="2_node_single_link_topo" ;; "2n_skx") TOPOLOGIES=( @@ -606,6 +792,14 @@ function select_topology () { ) TOPOLOGIES_TAGS="3_node_*_link_topo" ;; + "3n_hsw") + TOPOLOGIES=( + "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed1.yaml" + "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed2.yaml" + "${TOPOLOGIES_DIR}/lf_3n_hsw_testbed3.yaml" + ) + TOPOLOGIES_TAGS="3_node_single_link_topo" + ;; *) # No falling back to 3n_hsw default, that should have been done # by the function which has set NODENESS and FLAVOR. diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh new file mode 100644 index 0000000000..e4ed319c7f --- /dev/null +++ b/resources/libraries/bash/function/device.sh @@ -0,0 +1,570 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# This library defines functions used by multiple entry scripts. +# Keep functions ordered alphabetically, please. + +function activate_wrapper () { + # Acts as wrapper for activate docker topology. + # + # Variables read: + # - ${1} - Node multiplicity of desired testbed. + # - ${2} - Node flavor string, usually describing the processor. + # - ${3} - CSIT-SUT-DCR image name and version. + + set -exuo pipefail + + enter_mutex || die + get_available_interfaces "${1}" "${2}" || die + start_topology_containers "${3}" || die + bind_interfaces_to_containers || die + set_env_variables || die + print_env_variables || die + exit_mutex || die +} + + +function bind_interfaces_to_containers () { + # Bind linux network interface to container and create symlink for PCI + # address in container. + # + # Variables read: + # - DCR_UUIDS - Docker Container UUIDs. + # - DCR_CPIDS - Docker Container PIDs (namespaces). + # - DUT1_NETDEVS - List of network devices allocated to DUT1 container. + # - PCI_ADDR - PCI address of network device. + # - TG_NETDEVS - List of network devices allocated to TG container. + # Variables set: + # - NETDEV - Linux network interface. + + set -exuo pipefail + + for NETDEV in "${TG_NETDEVS[@]}"; do + get_pci_addr || die + link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || { + die "Reading symlink for PCI address failed!" + } + cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}" + + sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || { + die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!" + } + docker exec "${DCR_UUIDS[tg]}" ${cmd} || { + die "Linking PCI address in container failed!" + } + done + for NETDEV in "${DUT1_NETDEVS[@]}"; do + get_pci_addr || die + link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || { + die "Reading symlink for PCI address failed!" + } + cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}" + + sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || { + die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!" + } + docker exec "${DCR_UUIDS[dut1]}" ${cmd} || { + die "Linking PCI address in container failed!" + } + done +} + + +function bind_interfaces_to_driver () { + # Bind network interface specified by parameter to driver specified by + # parameter. + # + # Variables read: + # - ADDR - PCI address of network interface. + # - DRIVER - Kernel driver. + + pci_path="/sys/bus/pci/devices/${ADDR}" + drv_path="/sys/bus/pci/drivers/${DRIVER}" + vd="$(cat ${pci_path}/vendor ${pci_path}/device)" || { + die "Failed to retrieve interface details!" + } + set +e + echo ${vd} | sudo tee ${drv_path}/new_id + set -e + echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || { + die "Failed to unbind interface ${ADDR}!" + } + echo ${ADDR} | sudo tee ${drv_path}/bind || { + die "Failed to bind interface ${ADDR}!" + } +} + + +function clean_environment () { + # Cleanup environment by removing topology containers and binding + # interfaces back to original driver. + # + # Variables read: + # - DCR_UUIDS - Docker Container UUIDs. + # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container. + # - TG_PCIDEVS - List of PCI addresses of devices of TG container. + # Variables set: + # - ADDR - PCI address of network interface. + # - DRIVER - Kernel driver. + + set -exuo pipefail + + # Kill docker containers. + docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!" + + # Rebind interfaces back to kernel drivers. + for ADDR in ${TG_PCIDEVS[@]}; do + DRIVER="${TG_DRIVERS[0]}" + bind_interfaces_to_driver || die + done + for ADDR in ${DUT1_PCIDEVS[@]}; do + DRIVER="${DUT1_DRIVERS[0]}" + bind_interfaces_to_driver || die + done +} + + +function clean_environment_on_exit () { + # Cleanup environment by removing topology containers and binding + # interfaces back to original driver only if exit code is not 0. + # This function acts as workaround as 'set -eu' does not trigger ERR trap. + + if [ $? -ne 0 ]; then + clean_environment || die + fi +} + + +function deactivate_wrapper () { + # Acts as wrapper for deactivate docker topology. + # + # Variables read: + # - ${@} - CSIT environment variables. + + set -exuo pipefail + + enter_mutex || die + read_env_variables "${@}" || die + clean_environment || die + exit_mutex || die +} + + +function die () { + # Print the message to standard error end exit with error code specified + # by the second argument. + # + # Hardcoded values: + # - The default error message. + # Arguments: + # - ${1} - The whole error message, be sure to quote. Optional + # - ${2} - the code to exit with, default: 1. + + set -x + set +eu + warn "${1:-Unspecified run-time error occurred!}" + exit "${2:-1}" +} + + +function enter_mutex () { + # Enter mutual exclusion for protecting execution from starvation and + # deadlock. + + set -exuo pipefail + + mutex_timeout=3600 + mutex_file="/tmp/mutex_file" + + # Create mutex. + exec {lock_fd}>${mutex_file} || { + die "Mutex enter failed!" + } + flock --timeout "${mutex_timeout}" "${lock_fd}" || { + die "Calling flock() failed!" + } + # ---------------------- + # Enter mutex succeeded. + warn "Mutex enter succeeded for PID $$." +} + + +function exit_mutex () { + # Exit mutual exclusion. + + set -exuo pipefail + + # --------------------- + # Remove mutex so we are not blocking others anymore. + flock -u "${lock_fd}" || { + die "Mutex destroy failed!" + } + warn "Mutex leave succeeded for PID $$." +} + + +function get_available_interfaces () { + # Find and get available Virtual functions. + # + # Arguments: + # - ${1} - Node flavor string, usually describing the processor and node + # multiplicity of desired testbed, separated by underscore. + # Variables set: + # - DUT1_NETDEVS - List of network devices allocated to DUT1 container. + # - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container. + # - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container. + # - DUT1_DRIVERS - List of interface drivers to DUT1 container. + # - TG_NETDEVS - List of network devices allocated to TG container. + # - TG_PCIDEVS - List of PCI addresses allocated to TG container. + # - TG_NETMACS - List of MAC addresses allocated to TG container. + # - TG_DRIVERS - List of interface drivers to TG container. + + set -exuo pipefail + + # Following code is specifing VFs ID based on nodeness and flavor. + # As there is great variability in hardware configuration outside LF, + # from bootstrap architecure point of view these are considered as flavors. + # Anyone can override flavor for its own machine and add condition here. + # See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + case_text="${1}_${2}" + case "${case_text}" in + "1n_skx") + # Add Intel Corporation XL710/X710 Virtual Function to the + # whitelist. + pci_id="0x154c" + tg_netdev=(enp24) + dut1_netdev=(enp59) + ;; + "1n_vbox") + # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the + # whitelist. + pci_id="0x100f" + tg_netdev=(eth1 eth2) + dut1_netdev=(eth3 eth4) + ;; + *) + die "Unknown specification: ${case_text}!" + esac + + net_path="/sys/bus/pci/devices/*/net/*" + + # TG side of connections. + TG_NETDEVS=() + TG_PCIDEVS=() + TG_NETMACS=() + TG_DRIVERS=() + # DUT1 side of connections. + DUT1_NETDEVS=() + DUT1_PCIDEVS=() + DUT1_NETMACS=() + DUT1_DRIVERS=() + + # Following code is filtering available VFs represented by network device + # name. Only allowed VFs PCI IDs are used. + for netdev in \ + $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';'); + do + if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then + # We will filter to TG/DUT1 side of connection (this can be in + # future overriden by more advanced conditions for mapping). + for sub in ${tg_netdev[@]}; do + if [[ "${netdev#*$sub}" != "${netdev}" ]]; then + tg_side+=(${netdev}) + fi + done + for sub in ${dut1_netdev[@]}; do + if [[ "${netdev#*$sub}" != "${netdev}" ]]; then + dut1_side+=(${netdev}) + fi + done + fi + done + + for netdev in "${tg_side[@]::2}"; do + TG_NETDEVS+=(${netdev}) + done + for netdev in "${dut1_side[@]::2}"; do + DUT1_NETDEVS+=(${netdev}) + done + + for NETDEV in "${TG_NETDEVS[@]}"; do + get_pci_addr + get_mac_addr + get_krn_driver + TG_PCIDEVS+=(${PCI_ADDR}) + TG_NETMACS+=(${MAC_ADDR}) + TG_DRIVERS+=(${KRN_DRIVER}) + done + for NETDEV in "${DUT1_NETDEVS[@]}"; do + get_pci_addr + get_mac_addr + get_krn_driver + DUT1_PCIDEVS+=(${PCI_ADDR}) + DUT1_NETMACS+=(${MAC_ADDR}) + DUT1_DRIVERS+=(${KRN_DRIVER}) + done + + # We need at least two interfaces for TG/DUT1 for building topology. + if [ "${#TG_NETDEVS[@]}" -ne 2 ] || [ "${#DUT1_NETDEVS[@]}" -ne 2 ]; then + die "Not enough linux network interfaces found!" + fi + if [ "${#TG_PCIDEVS[@]}" -ne 2 ] || [ "${#DUT1_PCIDEVS[@]}" -ne 2 ]; then + die "Not enough pci interfaces found!" + fi +} + + +function get_krn_driver () { + # Get kernel driver from linux network device name. + # + # Variables read: + # - PCI_ADDR - PCI address of network device. + # Variables set: + # - KRN_DRIVER - Kernel driver of network device. + + set -exuo pipefail + + pci_path="/sys/bus/pci/devices/${PCI_ADDR}" + KRN_DRIVER="$(basename $(readlink -f ${pci_path}/driver))" || { + die "Failed to get kernel driver of PCI interface!" + } +} + + +function get_mac_addr () { + # Get MAC address from linux network device name. + # + # Variables read: + # - NETDEV - Linux network device name. + # Variables set: + # - MAC_ADDR - MAC address of network device. + + set -exuo pipefail + + if [ -d /sys/class/net/${NETDEV}/device ]; then + MAC_ADDR="$(</sys/class/net/${NETDEV}/address)" || { + die "Failed to get MAC address of linux network interface!" + } + fi +} + + +function get_pci_addr () { + # Get PCI address in <domain>:<bus:<device>.<func> format from linux network + # device name. + # + # Variables read: + # - NETDEV - Linux network device name. + # Variables set: + # - PCI_ADDR - PCI address of network device. + + set -exuo pipefail + + if [ -d /sys/class/net/${NETDEV}/device ]; then + PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || { + die "Failed to get PCI address of linux network interface!" + } + fi + if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then + die "PCI device ${NETDEV} doesn't exist!" + fi +} + + +function installed () { + + set -exuo pipefail + + # Check if the given utility is installed. Fail if not installed. + # + # Arguments: + # - ${1} - Utility to check. + # Returns: + # - 0 - If command is installed. + # - 1 - If command is not installed. + + command -v "${1}" +} + + +function print_env_variables () { + # Get environment variables prefixed by CSIT_. + + set -exuo pipefail + + env | grep CSIT_ +} + + +function read_env_variables () { + # Read environment variables from parameters. + # + # Arguments: + # - ${@} - Variables passed as an argument. + + set -exuo pipefail + + for param in "$@"; do + export "${param}" + done + declare -gA DCR_UUIDS + DCR_UUIDS+=([tg]="${CSIT_TG_UUID}") + DCR_UUIDS+=([dut1]="${CSIT_DUT1_UUID}") + TG_PCIDEVS=("${CSIT_TG_INTERFACES_PORT1_PCI}") + TG_DRIVERS=("${CSIT_TG_INTERFACES_PORT1_DRV}") + TG_PCIDEVS+=("${CSIT_TG_INTERFACES_PORT2_PCI}") + TG_DRIVERS+=("${CSIT_TG_INTERFACES_PORT2_DRV}") + DUT1_PCIDEVS=("${CSIT_DUT1_INTERFACES_PORT1_PCI}") + DUT1_DRIVERS=("${CSIT_DUT1_INTERFACES_PORT1_DRV}") + DUT1_PCIDEVS+=("${CSIT_DUT1_INTERFACES_PORT2_PCI}") + DUT1_DRIVERS+=("${CSIT_DUT1_INTERFACES_PORT2_DRV}") +} + + +function set_env_variables () { + # Set environment variables. + # + # Variables read: + # - DCR_UUIDS - Docker Container UUIDs. + # - DCR_PORTS - Docker Container's SSH ports. + # - DUT1_NETMACS - List of network devices MAC addresses of DUT1 container. + # - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container. + # - DUT1_DRIVERS - List of interface drivers to DUT1 container. + # - TG_NETMACS - List of network devices MAC addresses of TG container. + # - TG_PCIDEVS - List of PCI addresses of devices of TG container. + # - TG_DRIVERS - List of interface drivers to TG container. + + set -exuo pipefail + + set -a + CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || { + die "Reading hostname IP address failed!" + } + CSIT_TG_PORT="${DCR_PORTS[tg]#*:}" + CSIT_TG_UUID="${DCR_UUIDS[tg]}" + CSIT_TG_ARCH="$(uname -i)" || { + die "Reading machine architecture failed!" + } + CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || { + die "Reading hostname IP address failed!" + } + CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}" + CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}" + CSIT_DUT1_ARCH="$(uname -i)" || { + die "Reading machine architecture failed!" + } + CSIT_TG_INTERFACES_PORT1_MAC="${TG_NETMACS[0]}" + CSIT_TG_INTERFACES_PORT1_PCI="${TG_PCIDEVS[0]}" + CSIT_TG_INTERFACES_PORT1_DRV="${TG_DRIVERS[0]}" + CSIT_TG_INTERFACES_PORT2_MAC="${TG_NETMACS[1]}" + CSIT_TG_INTERFACES_PORT2_PCI="${TG_PCIDEVS[1]}" + CSIT_TG_INTERFACES_PORT2_DRV="${TG_DRIVERS[1]}" + CSIT_DUT1_INTERFACES_PORT1_MAC="${DUT1_NETMACS[0]}" + CSIT_DUT1_INTERFACES_PORT1_PCI="${DUT1_PCIDEVS[0]}" + CSIT_DUT1_INTERFACES_PORT1_DRV="${DUT1_DRIVERS[0]}" + CSIT_DUT1_INTERFACES_PORT2_MAC="${DUT1_NETMACS[1]}" + CSIT_DUT1_INTERFACES_PORT2_PCI="${DUT1_PCIDEVS[1]}" + CSIT_DUT1_INTERFACES_PORT2_DRV="${DUT1_DRIVERS[1]}" + set +a +} + + +function start_topology_containers () { + # Starts csit-sut-dcr docker containers for TG/DUT1. + # + # Variables read: + # - CSIT_DIR - Path to existing root of local CSIT git repository. + # Variables set: + # - DCR_UUIDS - Docker Container UUIDs. + # - DCR_PORTS - Docker Container SSH TCP ports. + # - DCR_CPIDS - Docker Container PIDs (namespaces). + + set -exuo pipefail + + if ! installed docker; then + die "Docker not present. Please install before continue!" + fi + + # If the IMAGE is not already loaded then docker run will pull the IMAGE, + # and all image dependencies, before it starts the container. + dcr_image="${1}" + # Run the container in the background and print the new container ID. + dcr_stc_params="--detach=true " + # Give extended privileges to this container. A "privileged" container is + # given access to all devices and able to run nested containers. + dcr_stc_params+="--privileged " + # Publish all exposed ports to random ports on the host interfaces. + dcr_stc_params+="--publish-all " + # Automatically remove the container when it exits. + dcr_stc_params+="--rm " + # Size of /dev/shm. + dcr_stc_params+="--shm-size 512M " + # Override access to PCI bus by attaching a filesystem mount to the + # container. + dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices " + # Mount vfio to be able to bind to see binded interfaces. We cannot use + # --device=/dev/vfio as this does not see newly binded interfaces. + dcr_stc_params+="--volume /dev/vfio:/dev/vfio " + + # Docker Container UUIDs. + declare -gA DCR_UUIDS + # Docker Container SSH TCP ports. + declare -gA DCR_PORTS + # Docker Container PIDs (namespaces). + declare -gA DCR_CPIDS + + # Run TG and DUT1. As initial version we do support only 2-node. + params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image}) + DCR_UUIDS+=([tg]="$(docker run "${params[@]}")") || { + die "Failed to start TG docker container!" + } + params=(${dcr_stc_params} --name csit-dut1-$(uuidgen) ${dcr_image}) + DCR_UUIDS+=([dut1]="$(docker run "${params[@]}")") || { + die "Failed to start DUT1 docker container!" + } + + trap 'clean_environment_on_exit' EXIT || { + die "Trap attempt failed, please cleanup manually. Aborting!" + } + + # Get Containers TCP ports. + params=(${DCR_UUIDS[tg]}) + DCR_PORTS+=([tg]="$(docker port "${params[@]}")") || { + die "Failed to get port of TG docker container!" + } + params=(${DCR_UUIDS[dut1]}) + DCR_PORTS+=([dut1]="$(docker port "${params[@]}")") || { + die "Failed to get port of DUT1 docker container!" + } + + # Get Containers PIDs. + params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]}) + DCR_CPIDS+=([tg]="$(docker inspect "${params[@]}")") || { + die "Failed to get PID of TG docker container!" + } + params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]}) + DCR_CPIDS+=([dut1]="$(docker inspect "${params[@]}")") || { + die "Failed to get PID of DUT1 docker container!" + } +} + +function warn () { + # Print the message to standard error. + # + # Arguments: + # - ${@} - The text of the message. + + echo "$@" >&2 +} |