CSIT-1449 Ansible: Adding calibration support

- Remove obsolete bootstrap_setup.sh.
- Add calibration support into ansible.
- Inject Ansible run with calibration to every job.

Change-Id: Ib2d6cd0c04d3bce700b205bef34c3c88edb26e28
Signed-off-by: Peter Mikus <pmikus@cisco.com>

11 files changed, 135 insertions, 239 deletions

diff --git a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh b/resources/libraries/bash/entry/bootstrap_setup_testbed.sh
deleted file mode 100755
index 9dd61cfc9d..0000000000
--- a/resources/libraries/bash/entry/bootstrap_setup_testbed.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -exuo pipefail
-
-# Assumptions:
-# + There is a directory holding CSIT code to use (this script is there).
-# ++ This entry script has access to arguments:
-# ++ ${@} = <linux_ip> <mgmt_ip> <mgmt_username> <mgmt_pass>
-# Consequences (and specific assumptions) are multiple,
-# examine tree of functions for current description.
-
-# "set -eu" handles failures from the following two lines.
-BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
-BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
-source "${BASH_FUNCTION_DIR}/common.sh" || {
-    echo "Source failed." >&2
-    exit 1
-}
-source "${BASH_FUNCTION_DIR}/setup.sh" || die "Source failed."
-common_dirs || die
-pxe_host "${@}" || die
-ansible_host || die
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index aad2969ecc..cfe94686cb 100644
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -31,6 +31,7 @@ source "${BASH_FUNCTION_DIR}/common.sh" || {
     exit 1
 }
 source "${BASH_FUNCTION_DIR}/gather.sh" || die "Source failed."
+source "${BASH_FUNCTION_DIR}/ansible.sh" || die "Source failed."
 common_dirs || die
 get_test_code "${1-}" || die
 get_test_tag_string || die
@@ -39,6 +40,7 @@ gather_build || die
 check_download_dir || die
 activate_virtualenv || die
 reserve_testbed || die
+ansible_hosts "run-jitter-tool" || die
 select_tags || die
 compose_pybot_arguments || die
 run_pybot || die
diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh
new file mode 100644
index 0000000000..a9314b4cd2
--- /dev/null
+++ b/resources/libraries/bash/function/ansible.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+
+function ansible_hosts () {
+    # Run ansible playbook on hosts in working topology file. Ansible scope is
+    # determined by tags passed as parameters to this function.
+    #
+    # Variable read:
+    # - ${WORKING_TOPOLOGY} - Reserved working topology.
+    # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located.
+
+    set -exuo pipefail
+
+    if ! installed sshpass; then
+        sudo apt-get update -y || die "apt-get update failed!"
+        sudo apt-get install -y sshpass || die "Install sshpass failed!"
+    fi
+
+    if ! installed ansible-playbook; then
+        # TODO: Consider moving to requirements.txt?
+        pip install ansible==2.7.8 || die "Install ansible via PIP failed!"
+    fi
+
+    hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
+        die "Failed to read hosts from working topology!"
+    }
+    pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
+    ansible-playbook \
+        --vault-password-file=vault_pass \
+        --extra-vars '@vault.yml' \
+        --inventory inventories/lf_inventory/hosts site.yaml \
+        --limit "$(echo ${hosts[@]//\"})" \
+        --tags "$(echo $@)" || die "Failed to run ansible on host!"
+    popd || die "Popd failed!"
+}
+
+function installed () {
+
+    set -exuo pipefail
+
+    # Check if the given utility is installed. Fail if not installed.
+    #
+    # Arguments:
+    # - ${1} - Utility to check.
+    # Returns:
+    # - 0 - If command is installed.
+    # - 1 - If command is not installed.
+
+    command -v "${1}"
+}
\ No newline at end of file
diff --git a/resources/libraries/bash/function/setup.sh b/resources/libraries/bash/function/setup.sh
deleted file mode 100644
index e481c539b4..0000000000
--- a/resources/libraries/bash/function/setup.sh
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2018 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -exuo pipefail
-
-
-function ansible_host () {
-    # Setup host via ansible playbook.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-
-    set -exuo pipefail
-
-    pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
-    ssh-copy-id -o StrictHostKeyChecking=no testuser@"${HOST}"
-    ansible-playbook --vault-id vault_pass --extra-vars '@vault.yml' \
-        --inventory lf_inventory site.yaml --limit ${HOST} || {
-        die "Failed to ansible host!"
-    }
-    popd || die "Popd failed!"
-}
-
-
-function installed () {
-    # Check if the given utility is installed. Fail if not installed.
-    #
-    # Arguments:
-    # - ${1} - Utility to check.
-    # Returns:
-    # - 0 - If command is installed.
-    # - 1 - If command is not installed.
-
-    set -exuo pipefail
-
-    command -v "${1}"
-}
-
-
-function pxe_cimc () {
-    # Reboot server with next boot set to PXE, disables PXE after server is UP
-    # to prevent endless loop.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-    # - ${MGMT} - Server management IP address (IPMI/CIMC).
-    # - ${USER} - CIMC user.
-    # - ${PASS} - CIMC pass.
-
-    set -exuo pipefail
-
-    pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!"
-    ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -pxe || {
-        die "Failed to send the PXE reboot command!"
-    }
-    for i in $(seq 1 500); do
-        warn "Waiting for server to become reachable ... " || die
-        if pingable "${HOST}"; then
-            ./cimc.py -u "${USER}" -p "${PASS}" "${MGMT}" --debug -hdd || {
-                die "Failed to send the HDD command!"
-            }
-            ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || {
-                die "Failed to remove obsolete SSH key!"
-            }
-            warn "Server reachable, PXE running!" || die
-            break
-        fi
-    done
-    popd || die "Popd failed!"
-}
-
-
-function pxe_host () {
-    # Reboot host into PXE mode and detect once it is up.
-    #
-    # Variable read:
-    # - ${@} - All script arguments from command line.
-    # Variable set:
-    # - ${HOST} - Server production IP address (Linux).
-    # - ${MGMT} - Server management IP address (IPMI/CIMC).
-    # - ${USER} - User.
-    # - ${PASS} - Pass.
-
-    set -exuo pipefail
-
-    if ! installed ipmitool; then
-        die "ipmitool not present. Please install before continue!"
-    fi
-
-    HOST=${1}
-    MGMT=${2}
-    USER=${3}
-    PASS=${4}
-
-    ipmi="ipmitool -I lanplus -H ${MGMT} -U ${USER} -P ${PASS} chassis status"
-    cimc="./cimc.py -u ${USER} -p ${PASS} ${MGMT} --mac-table"
-    pushd "${TOOLS_DIR}"/testbed-setup/cimc || die "Pushd failed!"
-    if ${ipmi}; then
-        warn "This is IMPI managed server!"
-        pxe_supermicro || die
-    elif ${cimc}; then
-        warn "This is CIMC managed server!"
-        pxe_cimc || die
-    else
-        die "Unknown management console!"
-    fi
-    popd || die "Popd failed!"
-
-    wait_for_ssh || die
-}
-
-
-function pxe_supermicro () {
-    # Reboot server with next boot set to PXE.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-    # - ${MGMT} - Server management IP address (IPMI/CIMC).
-    # - ${USER} - IPMI user.
-    # - ${PASS} - IPMI pass.
-
-    set -exuo pipefail
-
-    if ! installed ipmitool; then
-        die "ipmitool not present. Please install before continue!"
-    fi
-
-    cmd="chassis bootdev pxe"
-    ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || {
-        die "Failed to send the ${cmd} command!"
-    }
-    cmd="power reset"
-    ipmitool -I lanplus -H "${MGMT}" -U "${USER}" -P "${PASS}" "${cmd}" || {
-        die "Failed to send the ${cmd} command!"
-    }
-    for i in $(seq 1 500); do
-        warn "Waiting for server to become reachable ... " || die
-        if pingable "${HOST}"; then
-            ssh-keygen -f "/home/testuser/.ssh/known_hosts" -R "${HOST}" || {
-                die "Failed to remove obsolete SSH key!"
-            }
-            warn "Server reachable, PXE running!" || die
-            break
-        fi
-    done
-}
-
-
-function pingable () {
-    # Check if the host is reachable over ping. Fail if not reachable.
-    #
-    # Arguments:
-    # - ${1} - Host to check.
-    # Returns:
-    # - 0 - If host is reachable over ping.
-    # - 1 - If host is not reachable over ping.
-
-    set -exuo pipefail
-
-    ping -q -c 1 "${1}" 2>&1 > /dev/null
-}
-
-
-function wait_for_ssh () {
-    # Check if the host is reachable over SSH. Fail if not reachable.
-    #
-    # Variable read:
-    # - ${HOST} - Server production IP address (Linux).
-
-    set -exuo pipefail
-
-    while ! ssh "${HOST}"
-    do
-        warn "SSH not yet reachable, trying again!"
-        sleep 2
-    done
-    warn "SSH reachable!"
-}
diff --git a/resources/tools/testbed-setup/README.rst b/resources/tools/testbed-setup/README.rst
index 14871d1d29..01be10b5d3 100644
--- a/resources/tools/testbed-setup/README.rst
+++ b/resources/tools/testbed-setup/README.rst
@@ -98,12 +98,6 @@ Set CIMC/IPMI address, username, password and hostname an BIOS.
 Bootstrap the host
 ``````````````````
 
-Convenient way to re-stage host via script:
-
-.. code-block:: bash
-
-   sudo ./bootstrap_setup_testbed.sh <linux_ip> <mgmt_ip> <username> <pass>
-
 Optional: CIMC - From PXE boostrap server
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
index 4d96a39496..452a12c6fc 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
@@ -40,3 +40,9 @@ kubernetes_channel: 'main'
 kubernetes_version: '1.11.0-00'
 kubernetes_repository: 'deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_channel }}'
 kubernetes_apt_package_name: '{{ kubernetes_version }}'
+
+# Calibration settings.
+jitter:
+  directory: '/tmp/pma_tools'
+  core: 7
+  iterations: 30
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
new file mode 100644
index 0000000000..b70694ad31
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
@@ -0,0 +1,33 @@
+---
+# file: roles/calibration/tasks/main.yaml
+
+- name: Ubuntu specific
+  import_tasks: ubuntu.yaml
+  when: ansible_distribution|lower == 'ubuntu'
+
+- name: Run Spectre Meltdown checker
+  raw: 'spectre-meltdown-checker --no-color'
+  register: spectre_meltdown_output
+  tags: run-spectre-meltdown-checker
+
+- debug: var=spectre_meltdown_output.stdout_lines
+  tags: run-spectre-meltdown-checker
+
+- name: Clone Jitter tool
+  git:
+    repo: 'https://gerrit.fd.io/r/pma_tools'
+    dest: '{{ jitter.directory }}'
+  tags: run-jitter-tool
+
+- name: Compile Jitter tool
+  raw: 'cd {{ jitter.directory }}/jitter && make'
+  tags: run-jitter-tool
+
+- name: Run Jitter tool
+  raw: '{{ jitter.directory }}/jitter/jitter -c {{ jitter.core }} -i {{ jitter.iterations }} -f'
+  become: yes
+  register: jitter_output
+  tags: run-jitter-tool
+
+- debug: var=jitter_output.stdout_lines
+  tags: run-jitter-tool
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml
new file mode 100644
index 0000000000..bb4ad1e8c7
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/ubuntu.yaml
@@ -0,0 +1,9 @@
+---
+# file: roles/calibration/tasks/ubuntu.yaml
+
+- name: Install calibration dependencies
+  apt:
+    name: 'spectre-meltdown-checker'
+    state: 'present'
+    install_recommends: False
+  tags: install-csit-dependencies
diff --git a/resources/tools/testbed-setup/ansible/sut.yaml b/resources/tools/testbed-setup/ansible/sut.yaml
index 1f9a4cbaab..9b7a66656d 100644
--- a/resources/tools/testbed-setup/ansible/sut.yaml
+++ b/resources/tools/testbed-setup/ansible/sut.yaml
@@ -6,6 +6,11 @@
   become: yes
   become_user: root
   roles:
-    - common
-    - sut
-    - tg_sut
+    - role: common
+      tags: common
+    - role: tg
+      tags: tg
+    - role: tg_sut
+      tags: tg_sut
+    - role: calibration
+      tags: calibration
diff --git a/resources/tools/testbed-setup/ansible/tg.yaml b/resources/tools/testbed-setup/ansible/tg.yaml
index a652dc5c80..16d2e0fd40 100644
--- a/resources/tools/testbed-setup/ansible/tg.yaml
+++ b/resources/tools/testbed-setup/ansible/tg.yaml
@@ -6,6 +6,11 @@
   become: yes
   become_user: root
   roles:
-    - common
-    - tg
-    - tg_sut
+    - role: common
+      tags: common
+    - role: tg
+      tags: tg
+    - role: tg_sut
+      tags: tg_sut
+    - role: calibration
+      tags: calibration
diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml
index 7f0f6eb91a..3d4894136f 100644
--- a/resources/tools/testbed-setup/ansible/vpp_device.yaml
+++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml
@@ -6,5 +6,7 @@
   become: yes
   become_user: root
   roles:
-    - common
-    - vpp_device
+    - role: common
+      tags: common
+    - role: vpp_device
+      tags: vpp_device