aboutsummaryrefslogtreecommitdiffstats
path: root/resources/tools/testbed-setup/ansible/roles/cleanup/files
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2019-10-10 15:31:28 +0000
committerPeter Mikus <pmikus@cisco.com>2019-11-05 07:23:56 +0000
commitd01411c3c4af6c724a3800c621804ea979818d6d (patch)
tree8c2745c25a575c7f637473fe98d3c39c1c8e2b28 /resources/tools/testbed-setup/ansible/roles/cleanup/files
parent50d21f72ff61d06641954c22a8bc13c2468388f9 (diff)
Cleanup via Ansible
+ Remove dependency on topo_ scripts that depends on custom SSH() that depends on framework itself. This way the cleanup is independent of failure in our SSH libs. + Simple ansible command can do cleanup of a machine: ansible-playbook --inventory inventories/lf_inventory/hosts site.yaml \ --limit '10.32.8.18' --tags 'cleanup' + Add vpp_device reset and cleanup. + Remove historical scripts. - Still in testing beta phase. - Need to add SRIOV cleanup. Signed-off-by: Peter Mikus <pmikus@cisco.com> Change-Id: I68e23304c7ad01041f51263c328c6e8d9b555cb7
Diffstat (limited to 'resources/tools/testbed-setup/ansible/roles/cleanup/files')
-rw-r--r--resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh113
1 files changed, 113 insertions, 0 deletions
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh b/resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh
new file mode 100644
index 0000000000..ede2db1273
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/files/reset_vppdevice.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function die () {
+ # Print the message to standard error end exit with error code specified
+ # by the second argument.
+ #
+ # Hardcoded values:
+ # - The default error message.
+ # Arguments:
+ # - ${1} - The whole error message, be sure to quote. Optional
+ # - ${2} - the code to exit with, default: 1.
+
+ set +eu
+ warn "${1:-Unspecified run-time error occurred!}"
+ exit "${2:-1}"
+}
+
+
+function set_eligibility_off {
+ # Set Nomad eligibility to ineligible for scheduling. Fail otherwise.
+
+ set -euo pipefail
+
+ node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+
+ if [[ "${node_status}" != *"ineligible"* ]]; then
+ nomad node eligibility -disable "${node_id}" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+ if [[ "${node_status}" != *"ineligible"* ]]; then
+ die "Set eligibility off failed!"
+ fi
+ fi
+}
+
+
+function set_eligibility_on {
+ # Set Nomad eligibility to eligible for scheduling. Fail otherwise.
+
+ set -euo pipefail
+
+ node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+
+ if [[ "${node_status}" == *"ineligible"* ]]; then
+ nomad node eligibility -enable "${node_id}" || die
+ node_status="$(nomad node status | grep $(hostname))" || die
+ if [[ "${node_status}" == *"ineligible"* ]]; then
+ die "Set eligibility on failed!"
+ fi
+ fi
+}
+
+
+function restart_vfs_service {
+ # Stop and start VF serice. This will reinitialize VFs and driver mappings.
+
+ set -euo pipefail
+
+ warn "Restarting VFs service (this may take few minutes)..."
+ sudo service csit-initialize-vfs stop || die "Failed to stop VFs service!"
+ sudo service csit-initialize-vfs start || die "Failed to start VFs service!"
+}
+
+
+function wait_for_pending_containers {
+ # Wait in loop for defined amount of time for pending containers to
+ # gracefully quit them. If parameter force is specified. Force kill them.
+
+ # Arguments:
+ # - ${@} - Script parameters.
+
+ set -euo pipefail
+
+ retries=60
+ wait_time=60
+ containers=(docker ps --quiet --filter name=csit*)
+
+ for i in $(seq 1 ${retries}); do
+ mapfile -t pending_containers < <( ${containers[@]} ) || die
+ warn "Waiting for pending containers [${pending_containers[@]}] ..."
+ if [ ${#pending_containers[@]} -eq 0 ]; then
+ break
+ fi
+ sleep "${wait_time}" || die
+ done
+ if [ ${#pending_containers[@]} -ne 0 ]; then
+ if [[ "${1-}" == "force" ]]; then
+ warn "Force killing [${pending_containers[@]}] ..."
+ docker rm --force ${pending_containers[@]} || die
+ else
+ die "Still few containers running!"
+ fi
+ fi
+}
+
+
+function warn () {
+ # Print the message to standard error.
+ #
+ # Arguments:
+ # - ${@} - The text of the message.
+
+ echo "$@" >&2
+}
+
+
+set_eligibility_off || die
+wait_for_pending_containers "${@}" || die
+restart_vfs_service || die
+set_eligibility_on || die