aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2019-02-26 10:24:41 +0000
committerPeter Mikus <pmikus@cisco.com>2019-02-27 19:18:43 +0000
commit2045a26e54b45a5fd7635102ceda1662d84a2047 (patch)
tree7ca5a76399a569a3b5093f394615633a31385719
parent97063f15d83378f8c5989b7116eb7a542147455f (diff)
CSIT-1449 Ansible: Adding calibration support
- Remove obsolete bootstrap_setup.sh. - Add calibration support into ansible. - Inject Ansible run with calibration to every job. Change-Id: Ib2d6cd0c04d3bce700b205bef34c3c88edb26e28 Signed-off-by: Peter Mikus <pmikus@cisco.com>
-rwxr-xr-xresources/libraries/bash/entry/bootstrap_setup_testbed.sh35
-rw-r--r--resources/libraries/bash/entry/bootstrap_verify_perf.sh2
-rw-r--r--resources/libraries/bash/function/ansible.sh65
-rw-r--r--resources/libraries/bash/function/setup.sh190
-rw-r--r--resources/tools/testbed-setup/README.rst6
-rw-r--r--resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml6
-rw-r--r--resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml33
-rw-r--r--resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml9
-rw-r--r--resources/tools/testbed-setup/ansible/sut.yaml11
-rw-r--r--resources/tools/testbed-setup/ansible/tg.yaml11
-rw-r--r--resources/tools/testbed-setup/ansible/vpp_device.yaml6
11 files changed, 135 insertions, 239 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh b/resources/libraries/bash/entry/bootstrap_setup_testbed.sh
deleted file mode 100755
index 9dd61cfc9d..0000000000
--- a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -exuo pipefail
-
-# Assumptions:
-# + There is a directory holding CSIT code to use (this script is there).
-# ++ This entry script has access to arguments:
-# ++ ${@} = <linux_ip> <mgmt_ip> <mgmt_username> <mgmt_pass>
-# Consequences (and specific assumptions) are multiple,
-# examine tree of functions for current description.
-
-# "set -eu" handles failures from the following two lines.
-BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
-BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
-source "${BASH_FUNCTION_DIR}/common.sh" || {
- echo "Source failed." >&2
- exit 1
-}
-source "${BASH_FUNCTION_DIR}/setup.sh" || die "Source failed."
-common_dirs || die
-pxe_host "${@}" || die
-ansible_host || die
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index aad2969ecc..cfe94686cb 100644
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -31,6 +31,7 @@ source "${BASH_FUNCTION_DIR}/common.sh" || {
exit 1
}
source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed."
+source "${BASH_FUNCTION_DIR}/ansible.sh" || die "Source failed."
common_dirs || die
get_test_code "${1-}" || die
get_test_tag_string || die
@@ -39,6 +40,7 @@ gather_build || die
check_download_dir || die
activate_virtualenv || die
reserve_testbed || die
+ansible_hosts "run-jitter-tool" || die
select_tags || die
compose_pybot_arguments || die
run_pybot || die
diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh
new file mode 100644
index 0000000000..a9314b4cd2
--- /dev/null
+++ b/resources/libraries/bash/function/ansible.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+
+function ansible_hosts () {
+ # Run ansible playbook on hosts in working topology file. Ansible scope is
+ # determined by tags passed as parameters to this function.
+ #
+ # Variable read:
+ # - ${WORKING_TOPOLOGY} - Reserved working topology.
+ # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located.
+
+ set -exuo pipefail
+
+ if ! installed sshpass; then
+ sudo apt-get update -y || die "apt-get update failed!"
+ sudo apt-get install -y sshpass || die "Install sshpass failed!"
+ fi
+
+ if ! installed ansible-playbook; then
+ # TODO: Consider moving to requirements.txt?
+ pip install ansible==2.7.8 || die "Install ansible via PIP failed!"
+ fi
+
+ hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
+ die "Failed to read hosts from working topology!"
+ }
+ pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
+ ansible-playbook \
+ --vault-password-file=vault_pass \
+ --extra-vars '@vault.yml' \
+ --inventory inventories/lf_inventory/hosts site.yaml \
+ --limit "$(echo ${hosts[@]//\"})" \
+ --tags "$(echo $@)" || die "Failed to run ansible on host!"
+ popd || die "Popd failed!"
+}
+
+function installed () {
+
+ set -exuo pipefail
+
+ # Check if the given utility is installed. Fail if not installed.
+ #
+ # Arguments:
+ # - ${1} - Utility to check.
+ # Returns:
+ # - 0 - If command is installed.
+ # - 1 - If command is not installed.
+
+ command -v "${1}"
+} \ No newline at end of file
diff --git a/resources/libraries/bash/function/setup.sh b/resources/libraries/bash/function/setup.sh
deleted file mode 100644
index e481c539b4..0000000000
--- a/resources/libraries/bash/function/setup.sh
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -exuo pipefail
-
-
-function ansible_host () {
- # Setup host via ansible playbook.
- #
- # Variable read:
- # - ${HOST} - Server production IP address (Linux).
-
- set -exuo pipefail
-
- pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
- ssh-copy-id -o StrictHostKeyChecking=no testuser@"${HOST}"
- ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' \
- --inventory lf_inventory site.yaml --limit ${HOST} || {
- die "Failed to ansible host!"
- }
- popd || die "Popd failed!"
-}
-
-
-function installed () {
- # Check if the given utility is installed. Fail if not installed.
- #
- # Arguments:
- # - ${1} - Utility to check.
- # Returns:
- # - 0 - If command is installed.
- # - 1 - If command is not installed.
-
- set -exuo pipefail
-
- command -v "${1}"
-}
-
-
-function pxe_cimc () {
- # Reboot server with next boot set to PXE, disables PXE after server is UP
- # to prevent endless loop.
- #
- # Variable read:
- # - ${HOST} - Server production IP address (Linux).
- # - ${MGMT} - Server management IP address (IPMI/CIMC).
- # - ${USER} - CIMC user.
- # - ${PASS} - CIMC pass.
-
- set -exuo pipefail
-
- pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!"
- ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -pxe || {
- die "Failed to send the PXE reboot command!"
- }
- for i in $(seq 1 500); do
- warn "Waiting for server to become reachable ... " || die
- if pingable "${HOST}"; then
- ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -hdd || {
- die "Failed to send the HDD command!"
- }
- ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || {
- die "Failed to remove obsolete SSH key!"
- }
- warn "Server reachable, PXE running!" || die
- break
- fi
- done
- popd || die "Popd failed!"
-}
-
-
-function pxe_host () {
- # Reboot host into PXE mode and detect once it is up.
- #
- # Variable read:
- # - ${@} - All script arguments from command line.
- # Variable set:
- # - ${HOST} - Server production IP address (Linux).
- # - ${MGMT} - Server management IP address (IPMI/CIMC).
- # - ${USER} - User.
- # - ${PASS} - Pass.
-
- set -exuo pipefail
-
- if ! installed ipmitool; then
- die "ipmitool not present. Please install before continue!"
- fi
-
- HOST=${1}
- MGMT=${2}
- USER=${3}
- PASS=${4}
-
- ipmi="ipmitool -I lanplus -H ${MGMT} -U ${USER} -P ${PASS} chassis status"
- cimc="./cimc.py -u ${USER} -p ${PASS} ${MGMT} --mac-table"
- pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!"
- if ${ipmi}; then
- warn "This is IMPI managed server!"
- pxe_supermicro || die
- elif ${cimc}; then
- warn "This is CIMC managed server!"
- pxe_cimc || die
- else
- die "Unknown management console!"
- fi
- popd || die "Popd failed!"
-
- wait_for_ssh || die
-}
-
-
-function pxe_supermicro () {
- # Reboot server with next boot set to PXE.
- #
- # Variable read:
- # - ${HOST} - Server production IP address (Linux).
- # - ${MGMT} - Server management IP address (IPMI/CIMC).
- # - ${USER} - IPMI user.
- # - ${PASS} - IPMI pass.
-
- set -exuo pipefail
-
- if ! installed ipmitool; then
- die "ipmitool not present. Please install before continue!"
- fi
-
- cmd="chassis bootdev pxe"
- ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || {
- die "Failed to send the ${cmd} command!"
- }
- cmd="power reset"
- ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || {
- die "Failed to send the ${cmd} command!"
- }
- for i in $(seq 1 500); do
- warn "Waiting for server to become reachable ... " || die
- if pingable "${HOST}"; then
- ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || {
- die "Failed to remove obsolete SSH key!"
- }
- warn "Server reachable, PXE running!" || die
- break
- fi
- done
-}
-
-
-function pingable () {
- # Check if the host is reachable over ping. Fail if not reachable.
- #
- # Arguments:
- # - ${1} - Host to check.
- # Returns:
- # - 0 - If host is reachable over ping.
- # - 1 - If host is not reachable over ping.
-
- set -exuo pipefail
-
- ping -q -c 1 "${1}" 2>&1 > /dev/null
-}
-
-
-function wait_for_ssh () {
- # Check if the host is reachable over SSH. Fail if not reachable.
- #
- # Variable read:
- # - ${HOST} - Server production IP address (Linux).
-
- set -exuo pipefail
-
- while ! ssh "${HOST}"
- do
- warn "SSH not yet reachable, trying again!"
- sleep 2
- done
- warn "SSH reachable!"
-}
diff --git a/resources/tools/testbed-setup/README.rst b/resources/tools/testbed-setup/README.rst
index 14871d1d29..01be10b5d3 100644
--- a/resources/tools/testbed-setup/README.rst
+++ b/resources/tools/testbed-setup/README.rst
@@ -98,12 +98,6 @@ Set CIMC/IPMI address, username, password and hostname an BIOS.
Bootstrap the host
``````````````````
-Convenient way to re-stage host via script:
-
-.. code-block:: bash
-
- sudo ./bootstrap_setup_testbed.sh <linux_ip> <mgmt_ip> <username> <pass>
-
Optional: CIMC - From PXE boostrap server
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
index 4d96a39496..452a12c6fc 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
@@ -40,3 +40,9 @@ kubernetes_channel: 'main'
kubernetes_version: '1.11.0-00'
kubernetes_repository: 'deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_channel }}'
kubernetes_apt_package_name: '{{ kubernetes_version }}'
+
+# Calibration settings.
+jitter:
+ directory: '/tmp/pma_tools'
+ core: 7
+ iterations: 30
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
new file mode 100644
index 0000000000..b70694ad31
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
@@ -0,0 +1,33 @@
+---
+# file: roles/calibration/tasks/main.yaml
+
+- name: Ubuntu specific
+ import_tasks: ubuntu.yaml
+ when: ansible_distribution|lower == 'ubuntu'
+
+- name: Run Spectre Meltdown checker
+ raw: 'spectre-meltdown-checker --no-color'
+ register: spectre_meltdown_output
+ tags: run-spectre-meltdown-checker
+
+- debug: var=spectre_meltdown_output.stdout_lines
+ tags: run-spectre-meltdown-checker
+
+- name: Clone Jitter tool
+ git:
+ repo: 'https://gerrit.fd.io/r/pma_tools'
+ dest: '{{ jitter.directory }}'
+ tags: run-jitter-tool
+
+- name: Compile Jitter tool
+ raw: 'cd {{ jitter.directory }}/jitter && make'
+ tags: run-jitter-tool
+
+- name: Run Jitter tool
+ raw: '{{ jitter.directory }}/jitter/jitter -c {{ jitter.core }} -i {{ jitter.iterations }} -f'
+ become: yes
+ register: jitter_output
+ tags: run-jitter-tool
+
+- debug: var=jitter_output.stdout_lines
+ tags: run-jitter-tool
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml
new file mode 100644
index 0000000000..bb4ad1e8c7
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml
@@ -0,0 +1,9 @@
+---
+# file: roles/calibration/tasks/ubuntu.yaml
+
+- name: Install calibration dependencies
+ apt:
+ name: 'spectre-meltdown-checker'
+ state: 'present'
+ install_recommends: False
+ tags: install-csit-dependencies
diff --git a/resources/tools/testbed-setup/ansible/sut.yaml b/resources/tools/testbed-setup/ansible/sut.yaml
index 1f9a4cbaab..9b7a66656d 100644
--- a/resources/tools/testbed-setup/ansible/sut.yaml
+++ b/resources/tools/testbed-setup/ansible/sut.yaml
@@ -6,6 +6,11 @@
become: yes
become_user: root
roles:
- - common
- - sut
- - tg_sut
+ - role: common
+ tags: common
+ - role: tg
+ tags: tg
+ - role: tg_sut
+ tags: tg_sut
+ - role: calibration
+ tags: calibration
diff --git a/resources/tools/testbed-setup/ansible/tg.yaml b/resources/tools/testbed-setup/ansible/tg.yaml
index a652dc5c80..16d2e0fd40 100644
--- a/resources/tools/testbed-setup/ansible/tg.yaml
+++ b/resources/tools/testbed-setup/ansible/tg.yaml
@@ -6,6 +6,11 @@
become: yes
become_user: root
roles:
- - common
- - tg
- - tg_sut
+ - role: common
+ tags: common
+ - role: tg
+ tags: tg
+ - role: tg_sut
+ tags: tg_sut
+ - role: calibration
+ tags: calibration
diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml
index 7f0f6eb91a..3d4894136f 100644
--- a/resources/tools/testbed-setup/ansible/vpp_device.yaml
+++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml
@@ -6,5 +6,7 @@
become: yes
become_user: root
roles:
- - common
- - vpp_device
+ - role: common
+ tags: common
+ - role: vpp_device
+ tags: vpp_device