Ansible: Cleanup and speedup

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: Ia2ce482798204ad426f93f334c97e28eb51139fd

8 files changed, 147 insertions, 110 deletions

diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index 2329beba70..2098f789cf 100644
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -44,7 +44,6 @@ activate_virtualenv || die
 generate_tests || die
 archive_tests || die
 reserve_and_cleanup_testbed || die
-ansible_hosts "calibration" || die
 select_tags || die
 compose_pybot_arguments || die
 run_pybot || die
diff --git a/resources/libraries/bash/function/ansible.sh b/resources/libraries/bash/function/ansible.sh
index 431acc7c5d..56665dfa1c 100644
--- a/resources/libraries/bash/function/ansible.sh
+++ b/resources/libraries/bash/function/ansible.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,10 +16,9 @@
 set -exuo pipefail
 
 
-function ansible_hosts () {
+function ansible_adhoc () {
 
-    # Run ansible playbook on hosts in working topology file. Ansible scope is
-    # determined by tags passed as parameters to this function.
+    # Run ansible ad-hoc command module on hosts in working topology file.
     #
     # Variable read:
     # - ${WORKING_TOPOLOGY} - Reserved working topology.
@@ -31,9 +30,33 @@ function ansible_hosts () {
         die "Please install sshpass!"
     fi
 
-    if ! installed ansible-playbook; then
-        # TODO: Consider moving to requirements.txt?
-        pip install ansible==2.7.8 || die "Install ansible via PIP failed!"
+    hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
+        die "Failed to read hosts from working topology!"
+    }
+    pushd "${TOOLS_DIR}"/testbed-setup/ansible || die "Pushd failed!"
+    ANSIBLE_STDOUT_CALLBACK=yaml ansible \
+        --vault-password-file=vault_pass \
+        --extra-vars '@vault.yml' \
+        --inventory inventories/lf_inventory/hosts site.yaml \
+        --limit "$(echo ${hosts[@]//\"})" \
+        --module-name shell \
+        --args \"$(echo $@)\" || die "Failed to run ansible on host!"
+    popd || die "Popd failed!"
+}
+
+function ansible_playbook () {
+
+    # Run ansible playbook on hosts in working topology file. Ansible scope is
+    # determined by tags passed as parameters to this function.
+    #
+    # Variable read:
+    # - ${WORKING_TOPOLOGY} - Reserved working topology.
+    # - ${TOOLS_DIR} - CSIT tools directory, where testbed-setup is located.
+
+    set -exuo pipefail
+
+    if ! installed sshpass; then
+        die "Please install sshpass!"
     fi
 
     hosts=($(fgrep host "${WORKING_TOPOLOGY}" | cut -d ":" -f 2)) || {
@@ -49,6 +72,7 @@ function ansible_hosts () {
     popd || die "Popd failed!"
 }
 
+
 function installed () {
 
     # Check if the given utility is installed. Fail if not installed.
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 38ba60116a..f89f71dc65 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -561,7 +561,7 @@ function reserve_and_cleanup_testbed () {
     # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed.
     # Functions called:
     # - die - Print to stderr and exit.
-    # - ansible_hosts - Perform an action using ansible, see ansible.sh
+    # - ansible_playbook - Perform an action using ansible, see ansible.sh
     # Traps registered:
     # - EXIT - Calls cancel_all for ${WORKING_TOPOLOGY}.
 
@@ -588,9 +588,9 @@ function reserve_and_cleanup_testbed () {
                     }
                     die "Trap attempt failed, unreserve succeeded. Aborting."
                 }
-                # Cleanup check.
+                # Cleanup + calibration checks.
                 set +e
-                ansible_hosts "cleanup"
+                ansible_playbook "cleanup, calibration"
                 result="$?"
                 set -e
                 if [[ "${result}" == "0" ]]; then
@@ -1033,7 +1033,7 @@ function untrap_and_unreserve_testbed () {
     # - EXIT - Failure to untrap is reported, but ignored otherwise.
     # Functions called:
     # - die - Print to stderr and exit.
-    # - ansible_hosts - Perform an action using ansible, see ansible.sh
+    # - ansible_playbook - Perform an action using ansible, see ansible.sh
 
     set -xo pipefail
     set +eu  # We do not want to exit early in a "teardown" function.
@@ -1043,7 +1043,7 @@ function untrap_and_unreserve_testbed () {
         set -eu
         warn "Testbed looks unreserved already. Trap removal failed before?"
     else
-        ansible_hosts "cleanup" || true
+        ansible_playbook "cleanup" || true
         python3 "${PYTHON_SCRIPTS_DIR}/topo_reservation.py" -c -t "${wt}" || {
             die "${1:-FAILED TO UNRESERVE, FIX MANUALLY.}" 2
         }
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml
new file mode 100644
index 0000000000..ca4e75d268
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/aarch64.yaml
@@ -0,0 +1,2 @@
+---
+# file: roles/calibration/tasks/aarch64.yaml
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
index 34c987bdcb..c3c96ac499 100644
--- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
@@ -9,6 +9,6 @@
 - debug: var=spectre_meltdown_output.stdout_lines
   tags: run-spectre-meltdown-checker
 
-- name: x86 specific
-  import_tasks: x86_64.yaml
-  when: ansible_machine == 'x86_64'
+- name: Machine specifics
+  include_tasks: '{{ ansible_machine }}.yaml'
+  tags: run-jitter-tool
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
index 20306100e6..1cd64351a8 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
@@ -1,24 +1,29 @@
 ---
 # file: roles/cleanup/tasks/kill_containers.yaml
 
-- name: Kill container - Get running Docker containers
-  shell: "docker ps -aq"
-  register: running_containers
-  changed_when: no
-  tags: kill-containers
+- name: Kill containers
+  block:
+  - name: Kill container - Get running Docker containers
+    shell: "docker ps -aq"
+    register: running_containers
+    changed_when: no
+    tags: kill-containers
 
-- name: Kill container - Remove all Docker containers
-  shell: "docker rm --force {{ item }}"
-  with_items: "{{ running_containers.stdout_lines }}"
-  tags: kill-containers
+  - name: Kill container - Remove all Docker containers
+    shell: "docker rm --force {{ item }}"
+    with_items: "{{ running_containers.stdout_lines }}"
+    tags: kill-containers
 
-- name: Kill container - Get running LXC containers
-  shell: "lxc-ls"
-  register: running_containers
-  changed_when: no
-  tags: kill-containers
+  - name: Kill container - Get running LXC containers
+    shell: "lxc-ls"
+    register: running_containers
+    changed_when: no
+    tags: kill-containers
 
-- name: Kill container - Remove all LXC containers
-  shell: "lxc-destroy --force -n {{ item }}"
-  with_items: "{{ running_containers.stdout_lines }}"
-  tags: kill-containers
+  - name: Kill container - Remove all LXC containers
+    shell: "lxc-destroy --force -n {{ item }}"
+    with_items: "{{ running_containers.stdout_lines }}"
+    tags: kill-containers
+  rescue:
+  - fail:
+      msg: "Kill containers failed!"
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
index 4a1180b77f..a593fc7616 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
@@ -1,27 +1,32 @@
 ---
 # file: roles/cleanup/tasks/kill_process.yaml
 
-- name: Kill process - Get pid of {{ process }}
-  shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'"
-  when: >
-    process is defined and process != ""
-  register: running_processes
-  tags: kill-process
+- name: Kill process - {{ process }}
+  block:
+  - name: Kill process - Get pid of {{ process }}
+    shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'"
+    when: >
+      process is defined and process != ""
+    register: running_processes
+    tags: kill-process
 
-- name: Kill process - Safe kill {{ process }}
-  shell: "kill {{ item }}"
-  with_items: "{{ running_processes.stdout_lines }}"
-  tags: kill-process
+  - name: Kill process - Safe kill {{ process }}
+    shell: "kill {{ item }}"
+    with_items: "{{ running_processes.stdout_lines }}"
+    tags: kill-process
 
-- wait_for:
-    path: "/proc/{{ item }}/status"
-    state: absent
-  with_items: "{{ running_processes.stdout_lines }}"
-  ignore_errors: yes
-  register: killed_processes
-  tags: kill-process
+  - wait_for:
+      path: "/proc/{{ item }}/status"
+      state: absent
+    with_items: "{{ running_processes.stdout_lines }}"
+    ignore_errors: yes
+    register: killed_processes
+    tags: kill-process
 
-- name: Kill process - Force kill {{ process }}
-  shell: "kill -9 {{ item }}"
-  with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}"
-  tags: kill-process
+  - name: Kill process - Force kill {{ process }}
+    shell: "kill -9 {{ item }}"
+    with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}"
+    tags: kill-process
+  rescue:
+  - fail:
+      msg: "Kill process {{ process }} failed!"
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
index b3f8fffea2..c24b5e6a7f 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
@@ -1,58 +1,60 @@
 ---
 # file: roles/cleanup/tasks/sut.yaml
 
-- name: Kill processes - qemu
-  import_tasks: kill_process.yaml
-  vars:
-    process: "qemu"
-  tags: kill-process
-
-- name: Kill processes - l3fwd
-  import_tasks: kill_process.yaml
-  vars:
-    process: "l3fwd"
-  tags: kill-process
-
-- name: Kill processes - testpmd
-  import_tasks: kill_process.yaml
-  vars:
-    process: "testpmd"
-  tags: kill-process
-
-- name: Kill processes - iperf3
-  import_tasks: kill_process.yaml
-  vars:
-    process: "iperf3"
-  tags: kill-process
-
-- name: Kill processes - vpp_echo
-  import_tasks: kill_process.yaml
-  vars:
-    process: "vpp_echo"
-  tags: kill-process
-
-- name: Remove file or dir - Core zip file
-  file:
-    state: absent
-    path: "/tmp/*tar.lzo.lrz.xz*"
-  tags: remove-file-dir
-
-- name: Remove file or dir - Core dump file
-  file:
-    state: absent
-    path: "/tmp/*core*"
-  tags: remove-file-dir
-
-- name: Kill containers - Remove all containers
-  import_tasks: kill_containers.yaml
-  tags: kill-containers
-
-- name: Kubernetes - Reset
-  raw: 'kubeadm reset --force'
-  tags: kill-kubernetes
-
-- name: Remove packages - Remove VPP
-  import_tasks: remove_package.yaml
-  vars:
-    package: "*vpp*"
-  tags: remove-package
+- name: Host cleanup
+  block:
+    - name: Kill processes - qemu
+      import_tasks: kill_process.yaml
+      vars:
+        process: "qemu"
+      tags: kill-process
+
+    - name: Kill processes - l3fwd
+      import_tasks: kill_process.yaml
+      vars:
+        process: "l3fwd"
+      tags: kill-process
+
+    - name: Kill processes - testpmd
+      import_tasks: kill_process.yaml
+      vars:
+        process: "testpmd"
+      tags: kill-process
+
+    - name: Kill processes - iperf3
+      import_tasks: kill_process.yaml
+      vars:
+        process: "iperf3"
+      tags: kill-process
+
+    - name: Kill processes - vpp_echo
+      import_tasks: kill_process.yaml
+      vars:
+        process: "vpp_echo"
+      tags: kill-process
+
+    - name: Remove file or dir - Core zip file
+      file:
+        state: absent
+        path: "/tmp/*tar.lzo.lrz.xz*"
+      tags: remove-file-dir
+
+    - name: Remove file or dir - Core dump file
+      file:
+        state: absent
+        path: "/tmp/*core*"
+      tags: remove-file-dir
+
+    - name: Kill containers - Remove all containers
+      import_tasks: kill_containers.yaml
+      tags: kill-containers
+
+    - name: Kubernetes - Reset
+      raw: 'kubeadm reset --force'
+      tags: kill-kubernetes
+
+    - name: Remove packages - Remove VPP
+      import_tasks: remove_package.yaml
+      vars:
+        package: "*vpp*"
+      tags: remove-package