diff options
| -rwxr-xr-x | resources/libraries/bash/entry/bootstrap_setup_testbed.sh | 35 | ||||
| -rw-r--r-- | resources/libraries/bash/function/setup.sh | 190 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/README.md | 37 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/production | 30 | ||||
| -rwxr-xr-x[-rw-r--r--] | resources/tools/testbed-setup/cimc/cimc.py | 0 |
5 files changed, 258 insertions, 34 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh b/resources/libraries/bash/entry/bootstrap_setup_testbed.sh new file mode 100755 index 0000000000..9dd61cfc9d --- /dev/null +++ b/resources/libraries/bash/entry/bootstrap_setup_testbed.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# Assumptions: +# + There is a directory holding CSIT code to use (this script is there). +# ++ This entry script has access to arguments: +# ++ ${@} = <linux_ip> <mgmt_ip> <mgmt_username> <mgmt_pass> +# Consequences (and specific assumptions) are multiple, +# examine tree of functions for current description. + +# "set -eu" handles failures from the following two lines. +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/setup.sh" || die "Source failed." +common_dirs || die +pxe_host "${@}" || die +ansible_host || die diff --git a/resources/libraries/bash/function/setup.sh b/resources/libraries/bash/function/setup.sh new file mode 100644 index 0000000000..0b863aa17a --- /dev/null +++ b/resources/libraries/bash/function/setup.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + + +function ansible_host () { + # Setup host via ansible playbook. + # + # Variable read: + # - ${HOST} - Server production IP address (Linux). + + set -exuo pipefail + + pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!" + ssh-copy-id -o StrictHostKeyChecking=no testuser@"${HOST}" + ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' \ + --inventory production site.yaml --limit ${HOST} || { + die "Failed to ansible host!" + } + popd || die "Popd failed!" +} + + +function installed () { + # Check if the given utility is installed. Fail if not installed. + # + # Arguments: + # - ${1} - Utility to check. + # Returns: + # - 0 - If command is installed. + # - 1 - If command is not installed. + + set -exuo pipefail + + command -v "${1}" +} + + +function pxe_cimc () { + # Reboot server with next boot set to PXE, disables PXE after server is UP + # to prevent endless loop. + # + # Variable read: + # - ${HOST} - Server production IP address (Linux). + # - ${MGMT} - Server management IP address (IPMI/CIMC). + # - ${USER} - CIMC user. + # - ${PASS} - CIMC pass. + + set -exuo pipefail + + pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!" + ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -pxe || { + die "Failed to send the PXE reboot command!" + } + for i in $(seq 1 500); do + warn "Waiting for server to become reachable ... " || die + if pingable "${HOST}"; then + ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -hdd || { + die "Failed to send the HDD command!" + } + ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || { + die "Failed to remove obsolete SSH key!" + } + warn "Server reachable, PXE running!" || die + break + fi + done + popd || die "Popd failed!" +} + + +function pxe_host () { + # Reboot host into PXE mode and detect once it is up. + # + # Variable read: + # - ${@} - All script arguments from command line. + # Variable set: + # - ${HOST} - Server production IP address (Linux). + # - ${MGMT} - Server management IP address (IPMI/CIMC). + # - ${USER} - User. + # - ${PASS} - Pass. + + set -exuo pipefail + + if ! installed ipmitool; then + die "ipmitool not present. Please install before continue!" + fi + + HOST=${1} + MGMT=${2} + USER=${3} + PASS=${4} + + ipmi="ipmitool -I lanplus -H ${MGMT} -U ${USER} -P ${PASS} chassis status" + cimc="./cimc.py -u ${USER} -p ${PASS} ${MGMT} --mac-table" + pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!" + if ${ipmi}; then + warn "This is IMPI managed server!" + pxe_supermicro || die + elif ${cimc}; then + warn "This is CIMC managed server!" + pxe_cimc || die + else + die "Unknown management console!" + fi + popd || die "Popd failed!" + + wait_for_ssh || die +} + + +function pxe_supermicro () { + # Reboot server with next boot set to PXE. + # + # Variable read: + # - ${HOST} - Server production IP address (Linux). + # - ${MGMT} - Server management IP address (IPMI/CIMC). + # - ${USER} - IPMI user. + # - ${PASS} - IPMI pass. + + set -exuo pipefail + + if ! installed ipmitool; then + die "ipmitool not present. Please install before continue!" + fi + + cmd="chassis bootdev pxe" + ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || { + die "Failed to send the ${cmd} command!" + } + cmd="power reset" + ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || { + die "Failed to send the ${cmd} command!" + } + for i in $(seq 1 500); do + warn "Waiting for server to become reachable ... " || die + if pingable "${HOST}"; then + ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || { + die "Failed to remove obsolete SSH key!" + } + warn "Server reachable, PXE running!" || die + break + fi + done +} + + +function pingable () { + # Check if the host is reachable over ping. Fail if not reachable. + # + # Arguments: + # - ${1} - Host to check. + # Returns: + # - 0 - If host is reachable over ping. + # - 1 - If host is not reachable over ping. + + set -exuo pipefail + + ping -q -c 1 "${1}" 2>&1 > /dev/null +} + + +function wait_for_ssh () { + # Check if the host is reachable over SSH. Fail if not reachable. + # + # Variable read: + # - ${HOST} - Server production IP address (Linux). + + set -exuo pipefail + + while ! ssh "${HOST}" + do + warn "SSH not yet reachable, trying again!" + sleep 2 + done + warn "SSH reachable!" +} diff --git a/resources/tools/testbed-setup/README.md b/resources/tools/testbed-setup/README.md index ad9b400126..b4f376b96e 100644 --- a/resources/tools/testbed-setup/README.md +++ b/resources/tools/testbed-setup/README.md @@ -32,9 +32,6 @@ is available on the PXE bootstrap server in ~testuser/host-setup. - `sudo service isc-dhcp-server restart` - `cd ~testuser/host-setup` - `sudo mkdir /mnt/cdrom` - - Ubuntu Xenial - - `wget 'http://releases.ubuntu.com/16.04.2/ubuntu-16.04.2-server-amd64.iso'` - - `sudo mount -o loop ubuntu-16.04.2-server-amd64.iso /mnt/cdrom/` - Ubuntu Bionic - `wget 'http://cdimage.ubuntu.com/ubuntu/releases/18.04/release/ubuntu-18.04-server-amd64.iso'` - `sudo mount -o loop ubuntu-18.04-server-amd64.iso /mnt/cdrom/` @@ -65,31 +62,31 @@ is available on the PXE bootstrap server in ~testuser/host-setup. ### Bootstrap the host -Optional: From PXE boostrap server in case of installing Haswell +Convenient way to re-stage host via script: + + `sudo ./bootstrap_setup_testbed.sh <linux_ip> <mgmt_ip> <username> <pass>` + +Optional: CIMC - From PXE boostrap server - - `cd resources/tools/testbed-setup/cimc` - Initialize args.ip: Power-Off, reset BIOS defaults, Enable console redir, get LOM MAC addr - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d -i` - Adjust BIOS settings - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d -s '<biosVfIntelHyperThreadingTech rn="Intel-HyperThreading-Tech" vpIntelHyperThreadingTech="disabled" />' -s '<biosVfEnhancedIntelSpeedStepTech rn="Enhanced-Intel-SpeedStep-Tech" vpEnhancedIntelSpeedStepTech="disabled" />' -s '<biosVfIntelTurboBoostTech rn="Intel-Turbo-Boost-Tech" vpIntelTurboBoostTech="disabled" />'` - - add MAC address to DHCP (/etc/dhcp/dhcpd.conf) - - Reboot server with boot from PXE (restart immediately) - - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d -pxe` - -Optional: If RAID is not created on Haswells. Execute while Ubuntu install is running - - - create RAID array. Reboot if needed. + - Add MAC address to DHCP (/etc/dhcp/dhcpd.conf) + - If RAID is not created in CIMC. Create RAID array. Reboot. - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d --wipe` - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d -r -rl 1 -rs <disk size> -rd '[1,2]'` Alternatively, create the RAID array manually. - - - Set the next boot from HDD (without restart) + - Reboot server with boot from PXE (restart immediately) + - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d -pxe` + - Set the next boot from HDD (without restart) Execute while Ubuntu install is running. - `./cimc.py -u admin -p Cisco1234 $CIMC_ADDRESS -d -hdd` -Optional: If installing Skylake machine +Optional: IPMI - From PXE boostrap server - Get MAC address of LAN0 - `ipmitool -U ADMIN -H $HOST_ADDRESS raw 0x30 0x21 | tail -c 18` + - Add MAC address to DHCP (/etc/dhcp/dhcpd.conf) - Reboot into PXE for next boot only - `ipmitool -I lanplus -H $HOST_ADDRESS -U ADMIN chassis bootdev pxe` - `ipmitool -I lanplus -H $HOST_ADDRESS -U ADMIN power reset` @@ -97,14 +94,15 @@ Optional: If installing Skylake machine - `ipmitool -I lanplus -H $HOST_ADDRESS -U ADMIN sol activate` - `ipmitool -I lanplus -H $HOST_ADDRESS -U ADMIN sol deactivate` - When installation is finished: - Copy ssh keys for no pass access: `ssh-copy-id 10.30.51.x` - Clone CSIT actual repo: `git clone https://gerrit.fd.io/r/csit` - Go to ansible directory: `cd csit/resources/tools/testbed-setup/ansible` - - Edit production file and uncomment servers that are supposed to be installed: `ansible-playbook --ask-become-pass --inventory production site.yaml --list-hosts` - - Run ansible on selected hosts: `ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' --inventory production site.yaml` + - Edit production file and uncomment servers that are supposed to be + installed. + - Run ansible on selected hosts: + `ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' --inventory production site.yaml` For non-VIRL hosts, stop here. @@ -158,4 +156,5 @@ Back on the PXE bootstrap server: `ansible-playbook 04-disk-image.yaml` -The VIRL host should now be operational. Test, and when ready, create a ~jenkins-in/status file with the appropriate status. +The VIRL host should now be operational. Test, and when ready, create a +~jenkins-in/status file with the appropriate status. diff --git a/resources/tools/testbed-setup/ansible/production b/resources/tools/testbed-setup/ansible/production index f2364031b6..465796546e 100644 --- a/resources/tools/testbed-setup/ansible/production +++ b/resources/tools/testbed-setup/ansible/production @@ -1,20 +1,20 @@ all: children: -# haswell: -# children: -# tg: -# hosts: -# 10.30.51.16: null #t1-tg1 -# 10.30.51.20: null #t2-tg1 -# 10.30.51.24: null #t3-tg1 -# sut: -# hosts: -# 10.30.51.17: null #t1-sut1 -# 10.30.51.18: null #t1-sut2 -# 10.30.51.21: null #t2-sut1 -# 10.30.51.22: null #t2-sut2 -# 10.30.51.25: null #t3-sut1 -# 10.30.51.26: null #t3-sut2 + haswell: + children: + tg: + hosts: + 10.30.51.16: null #t1-tg1 + 10.30.51.20: null #t2-tg1 + 10.30.51.24: null #t3-tg1 + sut: + hosts: + 10.30.51.17: null #t1-sut1 + 10.30.51.18: null #t1-sut2 + 10.30.51.21: null #t2-sut1 + 10.30.51.22: null #t2-sut2 + 10.30.51.25: null #t3-sut1 + 10.30.51.26: null #t3-sut2 # virl: # WARNING, DO NOT ENABLE VIRL UNLESS YOU KNOW WHAT YOU ARE DOING # hosts: # 10.30.51.28: null #t4-virl1 diff --git a/resources/tools/testbed-setup/cimc/cimc.py b/resources/tools/testbed-setup/cimc/cimc.py index 2e0fc42a25..2e0fc42a25 100644..100755 --- a/resources/tools/testbed-setup/cimc/cimc.py +++ b/resources/tools/testbed-setup/cimc/cimc.py |