blob: ede2db12735e3ada185dffc0ac038a9b43dfde65 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
#!/usr/bin/env bash
set -euo pipefail
function die () {
# Print the message to standard error end exit with error code specified
# by the second argument.
#
# Hardcoded values:
# - The default error message.
# Arguments:
# - ${1} - The whole error message, be sure to quote. Optional
# - ${2} - the code to exit with, default: 1.
set +eu
warn "${1:-Unspecified run-time error occurred!}"
exit "${2:-1}"
}
function set_eligibility_off {
# Set Nomad eligibility to ineligible for scheduling. Fail otherwise.
set -euo pipefail
node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die
node_status="$(nomad node status | grep $(hostname))" || die
if [[ "${node_status}" != *"ineligible"* ]]; then
nomad node eligibility -disable "${node_id}" || die
node_status="$(nomad node status | grep $(hostname))" || die
if [[ "${node_status}" != *"ineligible"* ]]; then
die "Set eligibility off failed!"
fi
fi
}
function set_eligibility_on {
# Set Nomad eligibility to eligible for scheduling. Fail otherwise.
set -euo pipefail
node_id="$(nomad node status | grep $(hostname) | cut -d ' ' -f 1)" || die
node_status="$(nomad node status | grep $(hostname))" || die
if [[ "${node_status}" == *"ineligible"* ]]; then
nomad node eligibility -enable "${node_id}" || die
node_status="$(nomad node status | grep $(hostname))" || die
if [[ "${node_status}" == *"ineligible"* ]]; then
die "Set eligibility on failed!"
fi
fi
}
function restart_vfs_service {
# Stop and start VF serice. This will reinitialize VFs and driver mappings.
set -euo pipefail
warn "Restarting VFs service (this may take few minutes)..."
sudo service csit-initialize-vfs stop || die "Failed to stop VFs service!"
sudo service csit-initialize-vfs start || die "Failed to start VFs service!"
}
function wait_for_pending_containers {
# Wait in loop for defined amount of time for pending containers to
# gracefully quit them. If parameter force is specified. Force kill them.
# Arguments:
# - ${@} - Script parameters.
set -euo pipefail
retries=60
wait_time=60
containers=(docker ps --quiet --filter name=csit*)
for i in $(seq 1 ${retries}); do
mapfile -t pending_containers < <( ${containers[@]} ) || die
warn "Waiting for pending containers [${pending_containers[@]}] ..."
if [ ${#pending_containers[@]} -eq 0 ]; then
break
fi
sleep "${wait_time}" || die
done
if [ ${#pending_containers[@]} -ne 0 ]; then
if [[ "${1-}" == "force" ]]; then
warn "Force killing [${pending_containers[@]}] ..."
docker rm --force ${pending_containers[@]} || die
else
die "Still few containers running!"
fi
fi
}
function warn () {
# Print the message to standard error.
#
# Arguments:
# - ${@} - The text of the message.
echo "$@" >&2
}
set_eligibility_off || die
wait_for_pending_containers "${@}" || die
restart_vfs_service || die
set_eligibility_on || die
|
