CSIT-1449 Ansible: Adding calibration support

- Remove obsolete bootstrap_setup.sh.
- Add calibration support into ansible.
- Inject Ansible run with calibration to every job.

Change-Id: Ib2d6cd0c04d3bce700b205bef34c3c88edb26e28
Signed-off-by: Peter Mikus <pmikus@cisco.com>

4 files changed, 67 insertions, 225 deletions

diff --git a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh b/resources/libraries/bash/entry/bootstrap_setup_testbed.sh
deleted file mode 100755
index 9dd61cfc9d..0000000000
--- a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -exuo pipefail
-
-# Assumptions:
-# + There is a directory holding CSIT code to use (this script is there).
-# ++ This entry script has access to arguments:
-# ++ ${@} = <linux_ip> <mgmt_ip> <mgmt_username> <mgmt_pass>
-# Consequences (and specific assumptions) are multiple,
-# examine tree of functions for current description.
-
-# "set -eu" handles failures from the following two lines.
-BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
-BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
-source "${BASH_FUNCTION_DIR}/common.sh" || {
-    echo "Source failed." >&2
-    exit 1
-}
-source "${BASH_FUNCTION_DIR}/setup.sh" || die "Source failed."
-common_dirs || die
-pxe_host "${@}" || die
-ansible_host || die
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index aad2969ecc..cfe94686cb 100644
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -31,6 +31,7 @@ source "${BASH_FUNCTION_DIR}/common.sh" || {
     exit 1
 }
 source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed."
+source "${BASH_FUNCTION_DIR}/ansible.sh" || die "Source failed."
 common_dirs || die
 get_test_code "${1-}" || die
 get_test_tag_string || die
@@ -39,6 +40,7 @@ gather_build || die
 check_download_dir || die
 activate_virtualenv || die
 reserve_testbed || die
+ansible_hosts "run-jitter-tool" || die
 select_tags || die
 compose_pybot_arguments || die
 run_pybot || die
diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh
new file mode 100644
index 0000000000..a9314b4cd2
--- /dev/null
+++ b/resources/libraries/bash/function/ansible.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+
+function ansible_hosts () {
+    # Run ansible playbook on hosts in working topology file. Ansible scope is
+    # determined by tags passed as parameters to this function.
+    #
+    # Variable read:
+    # - ${WORKING_TOPOLOGY} - Reserved working topology.
+    # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located.
+
+    set -exuo pipefail
+
+    if ! installed sshpass; then
+        sudo apt-get update -y || die "apt-get update failed!"
+        sudo apt-get install -y sshpass || die "Install sshpass failed!"
+    fi
+
+    if ! installed ansible-playbook; then
+        # TODO: Consider moving to requirements.txt?
+        pip install ansible==2.7.8 || die "Install ansible via PIP failed!"
+    fi
+
+    hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
+        die "Failed to read hosts from working topology!"
+    }
+    pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
+    ansible-playbook \
+        --vault-password-file=vault_pass \
+        --extra-vars '@vault.yml' \
+        --inventory inventories/lf_inventory/hosts site.yaml \
+        --limit "$(echo ${hosts[@]//\"})" \
+        --tags "$(echo $@)" || die "Failed to run ansible on host!"
+    popd || die "Popd failed!"
+}
+
+function installed () {
+
+    set -exuo pipefail
+
+    # Check if the given utility is installed. Fail if not installed.
+    #
+    # Arguments:
+    # - ${1} - Utility to check.
+    # Returns:
+    # - 0 - If command is installed.
+    # - 1 - If command is not installed.
+
+    command -v "${1}"
+}
\ No newline at end of file
diff --git a/resources/libraries/bash/function/setup.sh b/resources/libraries/bash/function/setup.sh
deleted file mode 100644
index e481c539b4..0000000000
--- a/resources/libraries/bash/function/setup.sh
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -exuo pipefail
-
-
-function ansible_host () {
-    # Setup host via ansible playbook.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-
-    set -exuo pipefail
-
-    pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
-    ssh-copy-id -o StrictHostKeyChecking=no testuser@"${HOST}"
-    ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' \
-        --inventory lf_inventory site.yaml --limit ${HOST} || {
-        die "Failed to ansible host!"
-    }
-    popd || die "Popd failed!"
-}
-
-
-function installed () {
-    # Check if the given utility is installed. Fail if not installed.
-    #
-    # Arguments:
-    # - ${1} - Utility to check.
-    # Returns:
-    # - 0 - If command is installed.
-    # - 1 - If command is not installed.
-
-    set -exuo pipefail
-
-    command -v "${1}"
-}
-
-
-function pxe_cimc () {
-    # Reboot server with next boot set to PXE, disables PXE after server is UP
-    # to prevent endless loop.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-    # - ${MGMT} - Server management IP address (IPMI/CIMC).
-    # - ${USER} - CIMC user.
-    # - ${PASS} - CIMC pass.
-
-    set -exuo pipefail
-
-    pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!"
-    ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -pxe || {
-        die "Failed to send the PXE reboot command!"
-    }
-    for i in $(seq 1 500); do
-        warn "Waiting for server to become reachable ... " || die
-        if pingable "${HOST}"; then
-            ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -hdd || {
-                die "Failed to send the HDD command!"
-            }
-            ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || {
-                die "Failed to remove obsolete SSH key!"
-            }
-            warn "Server reachable, PXE running!" || die
-            break
-        fi
-    done
-    popd || die "Popd failed!"
-}
-
-
-function pxe_host () {
-    # Reboot host into PXE mode and detect once it is up.
-    #
-    # Variable read:
-    # - ${@} - All script arguments from command line.
-    # Variable set:
-    # - ${HOST} - Server production IP address (Linux).
-    # - ${MGMT} - Server management IP address (IPMI/CIMC).
-    # - ${USER} - User.
-    # - ${PASS} - Pass.
-
-    set -exuo pipefail
-
-    if ! installed ipmitool; then
-        die "ipmitool not present. Please install before continue!"
-    fi
-
-    HOST=${1}
-    MGMT=${2}
-    USER=${3}
-    PASS=${4}
-
-    ipmi="ipmitool -I lanplus -H ${MGMT} -U ${USER} -P ${PASS} chassis status"
-    cimc="./cimc.py -u ${USER} -p ${PASS} ${MGMT} --mac-table"
-    pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!"
-    if ${ipmi}; then
-        warn "This is IMPI managed server!"
-        pxe_supermicro || die
-    elif ${cimc}; then
-        warn "This is CIMC managed server!"
-        pxe_cimc || die
-    else
-        die "Unknown management console!"
-    fi
-    popd || die "Popd failed!"
-
-    wait_for_ssh || die
-}
-
-
-function pxe_supermicro () {
-    # Reboot server with next boot set to PXE.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-    # - ${MGMT} - Server management IP address (IPMI/CIMC).
-    # - ${USER} - IPMI user.
-    # - ${PASS} - IPMI pass.
-
-    set -exuo pipefail
-
-    if ! installed ipmitool; then
-        die "ipmitool not present. Please install before continue!"
-    fi
-
-    cmd="chassis bootdev pxe"
-    ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || {
-        die "Failed to send the ${cmd} command!"
-    }
-    cmd="power reset"
-    ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || {
-        die "Failed to send the ${cmd} command!"
-    }
-    for i in $(seq 1 500); do
-        warn "Waiting for server to become reachable ... " || die
-        if pingable "${HOST}"; then
-            ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || {
-                die "Failed to remove obsolete SSH key!"
-            }
-            warn "Server reachable, PXE running!" || die
-            break
-        fi
-    done
-}
-
-
-function pingable () {
-    # Check if the host is reachable over ping. Fail if not reachable.
-    #
-    # Arguments:
-    # - ${1} - Host to check.
-    # Returns:
-    # - 0 - If host is reachable over ping.
-    # - 1 - If host is not reachable over ping.
-
-    set -exuo pipefail
-
-    ping -q -c 1 "${1}" 2>&1 > /dev/null
-}
-
-
-function wait_for_ssh () {
-    # Check if the host is reachable over SSH. Fail if not reachable.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-
-    set -exuo pipefail
-
-    while ! ssh "${HOST}"
-    do
-        warn "SSH not yet reachable, trying again!"
-        sleep 2
-    done
-    warn "SSH reachable!"
-}