diff options
11 files changed, 135 insertions, 239 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh b/resources/libraries/bash/entry/bootstrap_setup_testbed.sh deleted file mode 100755 index 9dd61cfc9d..0000000000 --- a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -exuo pipefail - -# Assumptions: -# + There is a directory holding CSIT code to use (this script is there). -# ++ This entry script has access to arguments: -# ++ ${@} = <linux_ip> <mgmt_ip> <mgmt_username> <mgmt_pass> -# Consequences (and specific assumptions) are multiple, -# examine tree of functions for current description. - -# "set -eu" handles failures from the following two lines. -BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" -BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" -source "${BASH_FUNCTION_DIR}/common.sh" || { - echo "Source failed." >&2 - exit 1 -} -source "${BASH_FUNCTION_DIR}/setup.sh" || die "Source failed." -common_dirs || die -pxe_host "${@}" || die -ansible_host || die diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index aad2969ecc..cfe94686cb 100644 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -31,6 +31,7 @@ source "${BASH_FUNCTION_DIR}/common.sh" || { exit 1 } source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed." +source "${BASH_FUNCTION_DIR}/ansible.sh" || die "Source failed." common_dirs || die get_test_code "${1-}" || die get_test_tag_string || die @@ -39,6 +40,7 @@ gather_build || die check_download_dir || die activate_virtualenv || die reserve_testbed || die +ansible_hosts "run-jitter-tool" || die select_tags || die compose_pybot_arguments || die run_pybot || die diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh new file mode 100644 index 0000000000..a9314b4cd2 --- /dev/null +++ b/resources/libraries/bash/function/ansible.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + + +function ansible_hosts () { + # Run ansible playbook on hosts in working topology file. Ansible scope is + # determined by tags passed as parameters to this function. + # + # Variable read: + # - ${WORKING_TOPOLOGY} - Reserved working topology. + # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located. + + set -exuo pipefail + + if ! installed sshpass; then + sudo apt-get update -y || die "apt-get update failed!" + sudo apt-get install -y sshpass || die "Install sshpass failed!" + fi + + if ! installed ansible-playbook; then + # TODO: Consider moving to requirements.txt? + pip install ansible==2.7.8 || die "Install ansible via PIP failed!" + fi + + hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || { + die "Failed to read hosts from working topology!" + } + pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!" + ansible-playbook \ + --vault-password-file=vault_pass \ + --extra-vars '@vault.yml' \ + --inventory inventories/lf_inventory/hosts site.yaml \ + --limit "$(echo ${hosts[@]//\"})" \ + --tags "$(echo $@)" || die "Failed to run ansible on host!" + popd || die "Popd failed!" +} + +function installed () { + + set -exuo pipefail + + # Check if the given utility is installed. Fail if not installed. + # + # Arguments: + # - ${1} - Utility to check. + # Returns: + # - 0 - If command is installed. + # - 1 - If command is not installed. + + command -v "${1}" +}
\ No newline at end of file diff --git a/resources/libraries/bash/function/setup.sh b/resources/libraries/bash/function/setup.sh deleted file mode 100644 index e481c539b4..0000000000 --- a/resources/libraries/bash/function/setup.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2018 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -exuo pipefail - - -function ansible_host () { - # Setup host via ansible playbook. - # - # Variable read: - # - ${HOST} - Server production IP address (Linux). - - set -exuo pipefail - - pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!" - ssh-copy-id -o StrictHostKeyChecking=no testuser@"${HOST}" - ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' \ - --inventory lf_inventory site.yaml --limit ${HOST} || { - die "Failed to ansible host!" - } - popd || die "Popd failed!" -} - - -function installed () { - # Check if the given utility is installed. Fail if not installed. - # - # Arguments: - # - ${1} - Utility to check. - # Returns: - # - 0 - If command is installed. - # - 1 - If command is not installed. - - set -exuo pipefail - - command -v "${1}" -} - - -function pxe_cimc () { - # Reboot server with next boot set to PXE, disables PXE after server is UP - # to prevent endless loop. - # - # Variable read: - # - ${HOST} - Server production IP address (Linux). - # - ${MGMT} - Server management IP address (IPMI/CIMC). - # - ${USER} - CIMC user. - # - ${PASS} - CIMC pass. - - set -exuo pipefail - - pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!" - ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -pxe || { - die "Failed to send the PXE reboot command!" - } - for i in $(seq 1 500); do - warn "Waiting for server to become reachable ... " || die - if pingable "${HOST}"; then - ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -hdd || { - die "Failed to send the HDD command!" - } - ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || { - die "Failed to remove obsolete SSH key!" - } - warn "Server reachable, PXE running!" || die - break - fi - done - popd || die "Popd failed!" -} - - -function pxe_host () { - # Reboot host into PXE mode and detect once it is up. - # - # Variable read: - # - ${@} - All script arguments from command line. - # Variable set: - # - ${HOST} - Server production IP address (Linux). - # - ${MGMT} - Server management IP address (IPMI/CIMC). - # - ${USER} - User. - # - ${PASS} - Pass. - - set -exuo pipefail - - if ! installed ipmitool; then - die "ipmitool not present. Please install before continue!" - fi - - HOST=${1} - MGMT=${2} - USER=${3} - PASS=${4} - - ipmi="ipmitool -I lanplus -H ${MGMT} -U ${USER} -P ${PASS} chassis status" - cimc="./cimc.py -u ${USER} -p ${PASS} ${MGMT} --mac-table" - pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!" - if ${ipmi}; then - warn "This is IMPI managed server!" - pxe_supermicro || die - elif ${cimc}; then - warn "This is CIMC managed server!" - pxe_cimc || die - else - die "Unknown management console!" - fi - popd || die "Popd failed!" - - wait_for_ssh || die -} - - -function pxe_supermicro () { - # Reboot server with next boot set to PXE. - # - # Variable read: - # - ${HOST} - Server production IP address (Linux). - # - ${MGMT} - Server management IP address (IPMI/CIMC). - # - ${USER} - IPMI user. - # - ${PASS} - IPMI pass. - - set -exuo pipefail - - if ! installed ipmitool; then - die "ipmitool not present. Please install before continue!" - fi - - cmd="chassis bootdev pxe" - ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || { - die "Failed to send the ${cmd} command!" - } - cmd="power reset" - ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || { - die "Failed to send the ${cmd} command!" - } - for i in $(seq 1 500); do - warn "Waiting for server to become reachable ... " || die - if pingable "${HOST}"; then - ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || { - die "Failed to remove obsolete SSH key!" - } - warn "Server reachable, PXE running!" || die - break - fi - done -} - - -function pingable () { - # Check if the host is reachable over ping. Fail if not reachable. - # - # Arguments: - # - ${1} - Host to check. - # Returns: - # - 0 - If host is reachable over ping. - # - 1 - If host is not reachable over ping. - - set -exuo pipefail - - ping -q -c 1 "${1}" 2>&1 > /dev/null -} - - -function wait_for_ssh () { - # Check if the host is reachable over SSH. Fail if not reachable. - # - # Variable read: - # - ${HOST} - Server production IP address (Linux). - - set -exuo pipefail - - while ! ssh "${HOST}" - do - warn "SSH not yet reachable, trying again!" - sleep 2 - done - warn "SSH reachable!" -} diff --git a/resources/tools/testbed-setup/README.rst b/resources/tools/testbed-setup/README.rst index 14871d1d29..01be10b5d3 100644 --- a/resources/tools/testbed-setup/README.rst +++ b/resources/tools/testbed-setup/README.rst @@ -98,12 +98,6 @@ Set CIMC/IPMI address, username, password and hostname an BIOS. Bootstrap the host `````````````````` -Convenient way to re-stage host via script: - -.. code-block:: bash - - sudo ./bootstrap_setup_testbed.sh <linux_ip> <mgmt_ip> <username> <pass> - Optional: CIMC - From PXE boostrap server ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml index 4d96a39496..452a12c6fc 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml @@ -40,3 +40,9 @@ kubernetes_channel: 'main' kubernetes_version: '1.11.0-00' kubernetes_repository: 'deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_channel }}' kubernetes_apt_package_name: '{{ kubernetes_version }}' + +# Calibration settings. +jitter: + directory: '/tmp/pma_tools' + core: 7 + iterations: 30 diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml new file mode 100644 index 0000000000..b70694ad31 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml @@ -0,0 +1,33 @@ +--- +# file: roles/calibration/tasks/main.yaml + +- name: Ubuntu specific + import_tasks: ubuntu.yaml + when: ansible_distribution|lower == 'ubuntu' + +- name: Run Spectre Meltdown checker + raw: 'spectre-meltdown-checker --no-color' + register: spectre_meltdown_output + tags: run-spectre-meltdown-checker + +- debug: var=spectre_meltdown_output.stdout_lines + tags: run-spectre-meltdown-checker + +- name: Clone Jitter tool + git: + repo: 'https://gerrit.fd.io/r/pma_tools' + dest: '{{ jitter.directory }}' + tags: run-jitter-tool + +- name: Compile Jitter tool + raw: 'cd {{ jitter.directory }}/jitter && make' + tags: run-jitter-tool + +- name: Run Jitter tool + raw: '{{ jitter.directory }}/jitter/jitter -c {{ jitter.core }} -i {{ jitter.iterations }} -f' + become: yes + register: jitter_output + tags: run-jitter-tool + +- debug: var=jitter_output.stdout_lines + tags: run-jitter-tool diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml new file mode 100644 index 0000000000..bb4ad1e8c7 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml @@ -0,0 +1,9 @@ +--- +# file: roles/calibration/tasks/ubuntu.yaml + +- name: Install calibration dependencies + apt: + name: 'spectre-meltdown-checker' + state: 'present' + install_recommends: False + tags: install-csit-dependencies diff --git a/resources/tools/testbed-setup/ansible/sut.yaml b/resources/tools/testbed-setup/ansible/sut.yaml index 1f9a4cbaab..9b7a66656d 100644 --- a/resources/tools/testbed-setup/ansible/sut.yaml +++ b/resources/tools/testbed-setup/ansible/sut.yaml @@ -6,6 +6,11 @@ become: yes become_user: root roles: - - common - - sut - - tg_sut + - role: common + tags: common + - role: tg + tags: tg + - role: tg_sut + tags: tg_sut + - role: calibration + tags: calibration diff --git a/resources/tools/testbed-setup/ansible/tg.yaml b/resources/tools/testbed-setup/ansible/tg.yaml index a652dc5c80..16d2e0fd40 100644 --- a/resources/tools/testbed-setup/ansible/tg.yaml +++ b/resources/tools/testbed-setup/ansible/tg.yaml @@ -6,6 +6,11 @@ become: yes become_user: root roles: - - common - - tg - - tg_sut + - role: common + tags: common + - role: tg + tags: tg + - role: tg_sut + tags: tg_sut + - role: calibration + tags: calibration diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml index 7f0f6eb91a..3d4894136f 100644 --- a/resources/tools/testbed-setup/ansible/vpp_device.yaml +++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml @@ -6,5 +6,7 @@ become: yes become_user: root roles: - - common - - vpp_device + - role: common + tags: common + - role: vpp_device + tags: vpp_device |