Debian Codegraph @

ctdb/config/events/legacy/05.system.script

raw
1 #!/bin/sh
2 # ctdb event script for checking local file system utilization
3
4 [ -n "$CTDB_BASE" ] ||
5 CTDB_BASE=$(d="$(dirname "$0")" && cd -P "$d/../../" && pwd)
6
7 . "${CTDB_BASE}/functions"
8
9 load_script_options
10
11 ctdb_setup_state_dir "service" "system-monitoring"
12
13 validate_percentage()
14 {
15 case "$1" in
16 "") return 1 ;; # A failure that doesn't need a warning
17 [0-9] | [0-9][0-9] | 100) return 0 ;;
18 *)
19 echo "WARNING: ${1} is an invalid percentage in \"${2}\" check"
20 return 1
21 ;;
22 esac
23 }
24
25 check_thresholds()
26 {
27 _thing="$1"
28 _thresholds="$2"
29 _usage="$3"
30 _unhealthy_callout="$4"
31
32 case "$_thresholds" in
33 *:*)
34 _warn_threshold="${_thresholds%:*}"
35 _unhealthy_threshold="${_thresholds#*:}"
36 ;;
37 *)
38 _warn_threshold="$_thresholds"
39 _unhealthy_threshold=""
40 ;;
41 esac
42
43 _t=$(echo "$_thing" | sed -e 's@/@SLASH_@g' -e 's@ @_@g')
44 # script_state_dir set by ctdb_setup_state_dir()
45 # shellcheck disable=SC2154
46 _cache="${script_state_dir}/cache_${_t}"
47 if [ -r "$_cache" ]; then
48 read -r _prev <"$_cache"
49 else
50 _prev=0
51 fi
52 if validate_percentage "$_unhealthy_threshold" "$_thing"; then
53 if [ "$_usage" -ge "$_unhealthy_threshold" ]; then
54 printf 'ERROR: %s utilization %d%% >= threshold %d%%\n' \
55 "$_thing" \
56 "$_usage" \
57 "$_unhealthy_threshold"
58 # Only run unhealthy callout if passing the
59 # unhealthy threshold. That is, if the
60 # previous usage was below the threshold.
61 if [ "$_prev" -lt "$_unhealthy_threshold" ]; then
62 eval "$_unhealthy_callout"
63 fi
64 echo "$_usage" >"$_cache"
65 exit 1
66 fi
67 fi
68
69 if validate_percentage "$_warn_threshold" "$_thing"; then
70 if [ "$_usage" -ge "$_warn_threshold" ]; then
71 if [ "$_usage" = "$_prev" ]; then
72 return
73 fi
74 printf 'WARNING: %s utilization %d%% >= threshold %d%%\n' \
75 "$_thing" \
76 "$_usage" \
77 "$_warn_threshold"
78 echo "$_usage" >"$_cache"
79 else
80 if [ ! -r "$_cache" ]; then
81 return
82 fi
83 printf 'NOTICE: %s utilization %d%% < threshold %d%%\n' \
84 "$_thing" \
85 "$_usage" \
86 "$_warn_threshold"
87 rm -f "$_cache"
88 fi
89 fi
90 }
91
92 set_monitor_filsystem_usage_defaults()
93 {
94 _fs_defaults_cache="${script_state_dir}/cache_filsystem_usage_defaults"
95
96 if [ ! -r "$_fs_defaults_cache" ]; then
97 # Determine filesystem for each database directory, generate
98 # an entry to warn at 90%, de-duplicate entries, put all items
99 # on 1 line (so the read below gets everything)
100 for _t in "${CTDB_DBDIR:-${CTDB_VARDIR}}" \
101 "${CTDB_DBDIR_PERSISTENT:-${CTDB_VARDIR}/persistent}" \
102 "${CTDB_DBDIR_STATE:-${CTDB_VARDIR}/state}"; do
103 df -kP "$_t" | awk 'NR == 2 { printf "%s:90\n", $6 }'
104 done | sort -u | xargs >"$_fs_defaults_cache"
105 fi
106
107 read -r CTDB_MONITOR_FILESYSTEM_USAGE <"$_fs_defaults_cache"
108 }
109
110 monitor_filesystem_usage()
111 {
112 if [ -z "$CTDB_MONITOR_FILESYSTEM_USAGE" ]; then
113 set_monitor_filsystem_usage_defaults
114 fi
115
116 # Check each specified filesystem, specified in format
117 # <fs_mount>:<fs_warn_threshold>[:fs_unhealthy_threshold]
118 for _fs in $CTDB_MONITOR_FILESYSTEM_USAGE; do
119 _fs_mount="${_fs%%:*}"
120 _fs_thresholds="${_fs#*:}"
121
122 if [ ! -d "$_fs_mount" ]; then
123 echo "WARNING: Directory ${_fs_mount} does not exist"
124 continue
125 fi
126
127 # Get current utilization
128 _fs_usage=$(df -kP "$_fs_mount" |
129 sed -n -e 's@.*[[:space:]]\([[:digit:]]*\)%.*@\1@p')
130 if [ -z "$_fs_usage" ]; then
131 printf 'WARNING: Unable to get FS utilization for %s\n' \
132 "$_fs_mount"
133 continue
134 fi
135
136 check_thresholds "Filesystem ${_fs_mount}" \
137 "$_fs_thresholds" \
138 "$_fs_usage"
139 done
140 }
141
142 # shellcheck disable=SC2317
143 # Called indirectly via check_thresholds()
144 dump_memory_info()
145 {
146 get_proc "meminfo"
147 ps auxfww
148 set_proc "sysrq-trigger" "m"
149 }
150
151 monitor_memory_usage()
152 {
153 # Defaults
154 if [ -z "$CTDB_MONITOR_MEMORY_USAGE" ]; then
155 CTDB_MONITOR_MEMORY_USAGE=80
156 fi
157
158 _meminfo=$(get_proc "meminfo")
159 # Intentional word splitting here
160 # shellcheck disable=SC2046
161 set -- $(echo "$_meminfo" | awk '
162 $1 == "MemAvailable:" { memavail += $2 }
163 $1 == "MemFree:" { memfree += $2 }
164 $1 == "Cached:" { memfree += $2 }
165 $1 == "Buffers:" { memfree += $2 }
166 $1 == "MemTotal:" { memtotal = $2 }
167 $1 == "SwapFree:" { swapfree = $2 }
168 $1 == "SwapTotal:" { swaptotal = $2 }
169 END {
170 if (memavail != 0) { memfree = memavail ; }
171 if (memtotal + swaptotal != 0) {
172 usedtotal = memtotal - memfree + swaptotal - swapfree
173 print int(usedtotal / (memtotal + swaptotal) * 100)
174 } else {
175 print 0
176 }
177 }')
178 _mem_usage="$1"
179
180 check_thresholds "System memory" \
181 "$CTDB_MONITOR_MEMORY_USAGE" \
182 "$_mem_usage" \
183 dump_memory_info
184 }
185
186 case "$1" in
187 monitor)
188 # Load/cache database options from configuration file
189 ctdb_get_db_options
190
191 rc=0
192 monitor_filesystem_usage || rc=$?
193 monitor_memory_usage || rc=$?
194 exit $rc
195 ;;
196 esac
197
198 exit 0