aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2020-01-24 14:56:57 +0000
committerPeter Mikus <pmikus@cisco.com>2020-02-04 15:22:38 +0000
commit2cf0755933dac3e95c10928347996bad3cbe0a42 (patch)
tree89a79eb7b4a27e47b60b4ac1e6bb6eb56903b781
parent35024b691ade4278f49d8a65d237c7112fa32bc2 (diff)
Ansible: Cleanup and speedup
Signed-off-by: Peter Mikus <pmikus@cisco.com> Change-Id: Ia2ce482798204ad426f93f334c97e28eb51139fd
-rw-r--r--resources/libraries/bash/entry/bootstrap_verify_perf.sh1
-rw-r--r--resources/libraries/bash/function/ansible.sh38
-rw-r--r--resources/libraries/bash/function/common.sh10
-rw-r--r--resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml2
-rw-r--r--resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml6
-rw-r--r--resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml41
-rw-r--r--resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml47
-rw-r--r--resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml112
8 files changed, 147 insertions, 110 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index 2329beba70..2098f789cf 100644
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -44,7 +44,6 @@ activate_virtualenv || die
generate_tests || die
archive_tests || die
reserve_and_cleanup_testbed || die
-ansible_hosts "calibration" || die
select_tags || die
compose_pybot_arguments || die
run_pybot || die
diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh
index 431acc7c5d..56665dfa1c 100644
--- a/resources/libraries/bash/function/ansible.sh
+++ b/resources/libraries/bash/function/ansible.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -16,10 +16,9 @@
set -exuo pipefail
-function ansible_hosts () {
+function ansible_adhoc () {
- # Run ansible playbook on hosts in working topology file. Ansible scope is
- # determined by tags passed as parameters to this function.
+ # Run ansible ad-hoc command module on hosts in working topology file.
#
# Variable read:
# - ${WORKING_TOPOLOGY} - Reserved working topology.
@@ -31,9 +30,33 @@ function ansible_hosts () {
die "Please install sshpass!"
fi
- if ! installed ansible-playbook; then
- # TODO: Consider moving to requirements.txt?
- pip install ansible==2.7.8 || die "Install ansible via PIP failed!"
+ hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
+ die "Failed to read hosts from working topology!"
+ }
+ pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
+ ANSIBLE_STDOUT_CALLBACK=yaml ansible \
+ --vault-password-file=vault_pass \
+ --extra-vars '@vault.yml' \
+ --inventory inventories/lf_inventory/hosts site.yaml \
+ --limit "$(echo ${hosts[@]//\"})" \
+ --module-name shell \
+ --args \"$(echo $@)\" || die "Failed to run ansible on host!"
+ popd || die "Popd failed!"
+}
+
+function ansible_playbook () {
+
+ # Run ansible playbook on hosts in working topology file. Ansible scope is
+ # determined by tags passed as parameters to this function.
+ #
+ # Variable read:
+ # - ${WORKING_TOPOLOGY} - Reserved working topology.
+ # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located.
+
+ set -exuo pipefail
+
+ if ! installed sshpass; then
+ die "Please install sshpass!"
fi
hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
@@ -49,6 +72,7 @@ function ansible_hosts () {
popd || die "Popd failed!"
}
+
function installed () {
# Check if the given utility is installed. Fail if not installed.
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 38ba60116a..f89f71dc65 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -561,7 +561,7 @@ function reserve_and_cleanup_testbed () {
# - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
# Functions called:
# - die - Print to stderr and exit.
- # - ansible_hosts - Perform an action using ansible, see ansible.sh
+ # - ansible_playbook - Perform an action using ansible, see ansible.sh
# Traps registered:
# - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
@@ -588,9 +588,9 @@ function reserve_and_cleanup_testbed () {
}
die "Trap attempt failed, unreserve succeeded. Aborting."
}
- # Cleanup check.
+ # Cleanup + calibration checks.
set +e
- ansible_hosts "cleanup"
+ ansible_playbook "cleanup, calibration"
result="$?"
set -e
if [[ "${result}" == "0" ]]; then
@@ -1033,7 +1033,7 @@ function untrap_and_unreserve_testbed () {
# - EXIT - Failure to untrap is reported, but ignored otherwise.
# Functions called:
# - die - Print to stderr and exit.
- # - ansible_hosts - Perform an action using ansible, see ansible.sh
+ # - ansible_playbook - Perform an action using ansible, see ansible.sh
set -xo pipefail
set +eu # We do not want to exit early in a "teardown" function.
@@ -1043,7 +1043,7 @@ function untrap_and_unreserve_testbed () {
set -eu
warn "Testbed looks unreserved already. Trap removal failed before?"
else
- ansible_hosts "cleanup" || true
+ ansible_playbook "cleanup" || true
python3 "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
}
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml
new file mode 100644
index 0000000000..ca4e75d268
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml
@@ -0,0 +1,2 @@
+---
+# file: roles/calibration/tasks/aarch64.yaml
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
index 34c987bdcb..c3c96ac499 100644
--- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
@@ -9,6 +9,6 @@
- debug: var=spectre_meltdown_output.stdout_lines
tags: run-spectre-meltdown-checker
-- name: x86 specific
- import_tasks: x86_64.yaml
- when: ansible_machine == 'x86_64'
+- name: Machine specifics
+ include_tasks: '{{ ansible_machine }}.yaml'
+ tags: run-jitter-tool
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
index 20306100e6..1cd64351a8 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
@@ -1,24 +1,29 @@
---
# file: roles/cleanup/tasks/kill_containers.yaml
-- name: Kill container - Get running Docker containers
- shell: "docker ps -aq"
- register: running_containers
- changed_when: no
- tags: kill-containers
+- name: Kill containers
+ block:
+ - name: Kill container - Get running Docker containers
+ shell: "docker ps -aq"
+ register: running_containers
+ changed_when: no
+ tags: kill-containers
-- name: Kill container - Remove all Docker containers
- shell: "docker rm --force {{ item }}"
- with_items: "{{ running_containers.stdout_lines }}"
- tags: kill-containers
+ - name: Kill container - Remove all Docker containers
+ shell: "docker rm --force {{ item }}"
+ with_items: "{{ running_containers.stdout_lines }}"
+ tags: kill-containers
-- name: Kill container - Get running LXC containers
- shell: "lxc-ls"
- register: running_containers
- changed_when: no
- tags: kill-containers
+ - name: Kill container - Get running LXC containers
+ shell: "lxc-ls"
+ register: running_containers
+ changed_when: no
+ tags: kill-containers
-- name: Kill container - Remove all LXC containers
- shell: "lxc-destroy --force -n {{ item }}"
- with_items: "{{ running_containers.stdout_lines }}"
- tags: kill-containers
+ - name: Kill container - Remove all LXC containers
+ shell: "lxc-destroy --force -n {{ item }}"
+ with_items: "{{ running_containers.stdout_lines }}"
+ tags: kill-containers
+ rescue:
+ - fail:
+ msg: "Kill containers failed!"
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
index 4a1180b77f..a593fc7616 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
@@ -1,27 +1,32 @@
---
# file: roles/cleanup/tasks/kill_process.yaml
-- name: Kill process - Get pid of {{ process }}
- shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'"
- when: >
- process is defined and process != ""
- register: running_processes
- tags: kill-process
+- name: Kill process - {{ process }}
+ block:
+ - name: Kill process - Get pid of {{ process }}
+ shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'"
+ when: >
+ process is defined and process != ""
+ register: running_processes
+ tags: kill-process
-- name: Kill process - Safe kill {{ process }}
- shell: "kill {{ item }}"
- with_items: "{{ running_processes.stdout_lines }}"
- tags: kill-process
+ - name: Kill process - Safe kill {{ process }}
+ shell: "kill {{ item }}"
+ with_items: "{{ running_processes.stdout_lines }}"
+ tags: kill-process
-- wait_for:
- path: "/proc/{{ item }}/status"
- state: absent
- with_items: "{{ running_processes.stdout_lines }}"
- ignore_errors: yes
- register: killed_processes
- tags: kill-process
+ - wait_for:
+ path: "/proc/{{ item }}/status"
+ state: absent
+ with_items: "{{ running_processes.stdout_lines }}"
+ ignore_errors: yes
+ register: killed_processes
+ tags: kill-process
-- name: Kill process - Force kill {{ process }}
- shell: "kill -9 {{ item }}"
- with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}"
- tags: kill-process
+ - name: Kill process - Force kill {{ process }}
+ shell: "kill -9 {{ item }}"
+ with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}"
+ tags: kill-process
+ rescue:
+ - fail:
+ msg: "Kill process {{ process }} failed!"
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
index b3f8fffea2..c24b5e6a7f 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
@@ -1,58 +1,60 @@
---
# file: roles/cleanup/tasks/sut.yaml
-- name: Kill processes - qemu
- import_tasks: kill_process.yaml
- vars:
- process: "qemu"
- tags: kill-process
-
-- name: Kill processes - l3fwd
- import_tasks: kill_process.yaml
- vars:
- process: "l3fwd"
- tags: kill-process
-
-- name: Kill processes - testpmd
- import_tasks: kill_process.yaml
- vars:
- process: "testpmd"
- tags: kill-process
-
-- name: Kill processes - iperf3
- import_tasks: kill_process.yaml
- vars:
- process: "iperf3"
- tags: kill-process
-
-- name: Kill processes - vpp_echo
- import_tasks: kill_process.yaml
- vars:
- process: "vpp_echo"
- tags: kill-process
-
-- name: Remove file or dir - Core zip file
- file:
- state: absent
- path: "/tmp/*tar.lzo.lrz.xz*"
- tags: remove-file-dir
-
-- name: Remove file or dir - Core dump file
- file:
- state: absent
- path: "/tmp/*core*"
- tags: remove-file-dir
-
-- name: Kill containers - Remove all containers
- import_tasks: kill_containers.yaml
- tags: kill-containers
-
-- name: Kubernetes - Reset
- raw: 'kubeadm reset --force'
- tags: kill-kubernetes
-
-- name: Remove packages - Remove VPP
- import_tasks: remove_package.yaml
- vars:
- package: "*vpp*"
- tags: remove-package
+- name: Host cleanup
+ block:
+ - name: Kill processes - qemu
+ import_tasks: kill_process.yaml
+ vars:
+ process: "qemu"
+ tags: kill-process
+
+ - name: Kill processes - l3fwd
+ import_tasks: kill_process.yaml
+ vars:
+ process: "l3fwd"
+ tags: kill-process
+
+ - name: Kill processes - testpmd
+ import_tasks: kill_process.yaml
+ vars:
+ process: "testpmd"
+ tags: kill-process
+
+ - name: Kill processes - iperf3
+ import_tasks: kill_process.yaml
+ vars:
+ process: "iperf3"
+ tags: kill-process
+
+ - name: Kill processes - vpp_echo
+ import_tasks: kill_process.yaml
+ vars:
+ process: "vpp_echo"
+ tags: kill-process
+
+ - name: Remove file or dir - Core zip file
+ file:
+ state: absent
+ path: "/tmp/*tar.lzo.lrz.xz*"
+ tags: remove-file-dir
+
+ - name: Remove file or dir - Core dump file
+ file:
+ state: absent
+ path: "/tmp/*core*"
+ tags: remove-file-dir
+
+ - name: Kill containers - Remove all containers
+ import_tasks: kill_containers.yaml
+ tags: kill-containers
+
+ - name: Kubernetes - Reset
+ raw: 'kubeadm reset --force'
+ tags: kill-kubernetes
+
+ - name: Remove packages - Remove VPP
+ import_tasks: remove_package.yaml
+ vars:
+ package: "*vpp*"
+ tags: remove-package