Debian Codegraph @

ctdb/config/functions

raw
1 # Hey Emacs, this is a -*- shell-script -*- !!!
2
3 # utility functions for ctdb event scripts
4
5 if [ -z "$CTDB_BASE" ]; then
6 echo 'CTDB_BASE unset in CTDB functions file'
7 exit 1
8 fi
9 export CTDB_BASE
10
11 # CTDB_VARDIR is used elsewhere
12 # shellcheck disable=SC2034
13 CTDB_VARDIR="/usr/local/var/lib/ctdb"
14
15 CTDB="${CTDB:-/usr/local/bin/ctdb}"
16
17 # Only (and always) override these variables in test code
18
19 if [ -z "$CTDB_SCRIPT_VARDIR" ]; then
20 CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/scripts"
21 fi
22
23 if [ -z "$CTDB_SYS_ETCDIR" ]; then
24 CTDB_SYS_ETCDIR="/etc"
25 fi
26
27 if [ -z "$CTDB_HELPER_BINDIR" ]; then
28 CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
29 fi
30
31 #######################################
32 # pull in a system config file, if any
33
34 load_system_config()
35 {
36 for _i; do
37
38 if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/${_i}" ]; then
39 . "${CTDB_SYS_ETCDIR}/sysconfig/${_i}"
40 return
41 elif [ -f "${CTDB_SYS_ETCDIR}/default/${_i}" ]; then
42 . "${CTDB_SYS_ETCDIR}/default/${_i}"
43 return
44 fi
45 done
46 }
47
48 # load_script_options [ component script ]
49 # script is an event script name relative to a component
50 # component is currently ignored
51 load_script_options()
52 {
53 if [ $# -eq 2 ]; then
54 _script="$2"
55 elif [ $# -eq 0 ]; then
56 _script=""
57 else
58 die "usage: load_script_options [ component script ]"
59 fi
60
61 _options="${CTDB_BASE}/script.options"
62
63 if [ -r "$_options" ]; then
64 . "$_options"
65 fi
66
67 if [ -n "$_script" ]; then
68 _s="${CTDB_BASE}/events/legacy/${_script}"
69 else
70 _s="${0%.script}"
71 fi
72 _options="${_s}.options"
73
74 if [ -r "$_options" ]; then
75 . "$_options"
76 fi
77 }
78
79 ##############################################################
80
81 die()
82 {
83 _msg="$1"
84 _rc="${2:-1}"
85
86 echo "$_msg" >&2
87 exit "$_rc"
88 }
89
90 # Log given message or stdin to either syslog or a CTDB log file
91 # $1 is the tag passed to logger if syslog is in use.
92 script_log()
93 {
94 _tag="$1"
95 shift
96
97 case "$CTDB_LOGGING" in
98 file:)
99 if [ -n "$*" ] ; then
100 echo "$*"
101 else
102 cat
103 fi >&2
104 ;;
105 file:* | "")
106 if [ -n "$CTDB_LOGGING" ]; then
107 _file="${CTDB_LOGGING#file:}"
108 else
109 _file="/usr/local/var/log/log.ctdb"
110 fi
111 {
112 if [ -n "$*" ]; then
113 echo "$*"
114 else
115 cat
116 fi
117 } >>"$_file"
118 ;;
119 *)
120 # Handle all syslog:* variants here too. There's no tool to do
121 # the lossy things, so just use logger.
122 logger -t "ctdbd: ${_tag}" "$@"
123 ;;
124 esac
125 }
126
127 # When things are run in the background in an eventscript then logging
128 # output might get lost. This is the "solution". :-)
129 background_with_logging()
130 {
131 (
132 "$@" 2>&1 </dev/null |
133 script_log "${script_name}&"
134 ) &
135
136 return 0
137 }
138
139 ##############################################################
140 # check number of args for different events
141 ctdb_check_args()
142 {
143 case "$1" in
144 takeip | releaseip)
145 if [ $# != 4 ]; then
146 echo "ERROR: must supply interface, IP and maskbits"
147 exit 1
148 fi
149 ;;
150 updateip)
151 if [ $# != 5 ]; then
152 echo "ERROR: must supply old interface, new interface, IP and maskbits"
153 exit 1
154 fi
155 ;;
156 esac
157 }
158
159 ##############################################################
160 # determine on what type of system (init style) we are running
161 detect_init_style()
162 {
163 _init_style_file="${CTDB_SCRIPT_VARDIR}/init-style"
164
165 if [ ! -f "$_init_style_file" ]; then
166 if [ -n "$CTDB_INIT_STYLE" ]; then
167 echo "$CTDB_INIT_STYLE" >"$_init_style_file"
168 return
169 fi
170
171 # Subshell to contain variables in os-release file
172 (
173 _os_release="${CTDB_SYS_ETCDIR}/os-release"
174 if [ -f "$_os_release" ]; then
175 . "$_os_release"
176 case "$ID" in
177 centos | fedora | rhel)
178 echo "redhat"
179 ;;
180 debian | ubuntu)
181 echo "debian"
182 ;;
183 sles | suse)
184 echo "suse"
185 ;;
186 *)
187 case "$ID_LIKE" in
188 *centos* | *rhel*)
189 echo "redhat"
190 ;;
191 *)
192 echo "$ID"
193 ;;
194 esac
195 ;;
196 esac
197 else
198 echo "WARNING: unknown distribution ${ID}" >&2
199 echo "unknown"
200 fi
201 ) >"$_init_style_file"
202 fi
203
204 read -r CTDB_INIT_STYLE <"$_init_style_file"
205 }
206
207 ######################################################
208 # simulate /sbin/service on platforms that don't have it
209 # _service() makes it easier to hook the service() function for
210 # testing.
211 _service()
212 {
213 _service_name="$1"
214 _op="$2"
215
216 # do nothing, when no service was specified
217 [ -z "$_service_name" ] && return
218
219 if [ -x /sbin/service ]; then
220 $_nice /sbin/service "$_service_name" "$_op"
221 elif [ -x /usr/sbin/service ]; then
222 $_nice /usr/sbin/service "$_service_name" "$_op"
223 elif [ -x /bin/systemctl ]; then
224 $_nice /bin/systemctl "$_op" "$_service_name"
225 elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
226 $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
227 elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
228 $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
229 fi
230 }
231
232 service()
233 {
234 _nice=""
235 _service "$@"
236 }
237
238 ######################################################
239 # simulate /sbin/service (niced) on platforms that don't have it
240 nice_service()
241 {
242 _nice="nice"
243 _service "$@"
244 }
245
246 ######################################################
247 # Cached retrieval of PNN from local node. This never changes so why
248 # open a client connection to the server each time this is needed?
249 ctdb_get_pnn()
250 {
251 _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
252 if [ ! -f "$_pnn_file" ]; then
253 $CTDB pnn >"$_pnn_file"
254 fi
255
256 cat "$_pnn_file"
257 }
258
259 # Cached retrieval of private IP address from local node. This never
260 # changes.
261 ctdb_get_ip_address()
262 {
263 _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
264 if [ ! -f "$_ip_addr_file" ]; then
265 $CTDB -X nodestatus |
266 awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
267 fi
268
269 cat "$_ip_addr_file"
270 }
271
272 # Cache of public IP addresses assigned to this node. This function
273 # exists mainly so statd_callout does not need to talk to ctdbd, so
274 # can be run as non-root, but it may be used in other places. This
275 # must be updated/refreshed on failover. This is done in
276 # 10.interface, but doing it in "ipreallocated" isn't enough because
277 # clients may connect as soon as "takeip" completes. Also, the VNN in
278 # the daemon is only updated after the "releaseip" event completes, so
279 # "ctdb -X ip" can't be relied on there. Hence, complex updates
280 # involving locking for "takeip" & "releaseip". A future
281 # restructuring of the failover model will obsolete all of these
282 # moving parts.
283 CTDB_MY_PUBLIC_IPS_CACHE="${CTDB_SCRIPT_VARDIR}/my-public-ip-addresses"
284 update_my_public_ip_addresses()
285 {
286 _event="$1"
287
288 _f="$CTDB_MY_PUBLIC_IPS_CACHE"
289 _lock="${_f}.lock"
290
291 # In private CTDB state directory - no $$ security issue
292 _new="${_f}.new.$$"
293 {
294 flock --timeout 10 9 ||
295 die "ctdb_get_my_public_ip_addresses: timeout"
296
297 case "$_event" in
298 takeip)
299 _ip="$2"
300 # Redirect of stderr guards against initial
301 # missing file
302 cat "$_f" 2>/dev/null >"$_new"
303 echo "$_ip" >>"$_new"
304 ;;
305 releaseip)
306 _ip="$2"
307 # Redirect of stderr guards against initial
308 # missing file, which shouldn't happen in
309 # releaseip...
310 grep -Fvx "$_ip" "$_f" 2>/dev/null >"$_new"
311 ;;
312 ipreallocated)
313 _pnn=$(ctdb_get_pnn)
314 $CTDB -X ip |
315 awk -F'|' -v pnn="$_pnn" \
316 '$3 == pnn {print $2}' >"$_new"
317 ;;
318 esac
319
320 mv "$_new" "$_f"
321
322 } 9>"$_lock"
323 }
324
325 # Cached retrieval of database options for use by event scripts.
326 #
327 # If the variables are already set then they should not be overwritten
328 # - this should only happen during event script testing.
329 ctdb_get_db_options()
330 {
331 _db_opts_file="${CTDB_SCRIPT_VARDIR}/db_options.cache"
332
333 if [ ! -f "$_db_opts_file" ]; then
334 {
335 ctdb_translate_option "database" \
336 "volatile database directory" \
337 "CTDB_DBDIR"
338 ctdb_translate_option "database" \
339 "persistent database directory" \
340 "CTDB_DBDIR_PERSISTENT"
341 ctdb_translate_option "database" \
342 "state database directory" \
343 "CTDB_DBDIR_STATE"
344 } >"$_db_opts_file"
345 fi
346
347 . "$_db_opts_file"
348 }
349
350 ctdb_translate_option()
351 {
352 _section="$1"
353 _opt="$2"
354 _variable="$3"
355
356 # ctdb-config already prints an error if something goes wrong
357 _t=$("${CTDB_HELPER_BINDIR}/ctdb-config" get "$_section" "$_opt") ||
358 exit $?
359 echo "${_variable}=\"${_t}\""
360 }
361
362 ######################################################
363 # wrapper around /proc/ settings to allow them to be hooked
364 # for testing
365 # 1st arg is relative path under /proc/, 2nd arg is value to set
366 set_proc()
367 {
368 echo "$2" >"/proc/$1"
369 }
370
371 set_proc_maybe()
372 {
373 if [ -w "/proc/$1" ]; then
374 set_proc "$1" "$2"
375 fi
376 }
377
378 ######################################################
379 # wrapper around getting file contents from /proc/ to allow
380 # this to be hooked for testing
381 # 1st arg is relative path under /proc/
382 get_proc()
383 {
384 cat "/proc/$1"
385 }
386
387 ######################################################
388 # Print up to $_max kernel stack traces for processes named $_program
389 program_stack_traces()
390 {
391 _prog="$1"
392 _max="${2:-1}"
393
394 _count=1
395 for _pid in $(pidof "$_prog"); do
396 [ "$_count" -le "$_max" ] || break
397
398 # Do this first to avoid racing with process exit
399 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
400 if [ -n "$_stack" ]; then
401 echo "Stack trace for ${_prog}[${_pid}]:"
402 echo "$_stack"
403 _count=$((_count + 1))
404 fi
405 done
406 }
407
408 ######################################################
409 # Ensure $service_name is set
410 assert_service_name()
411 {
412 # service_name is set by the event script
413 # shellcheck disable=SC2154
414 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
415 }
416
417 ######################################################
418 # check a set of directories is available
419 # return 1 on a missing directory
420 # directories are read from stdin
421 ######################################################
422 ctdb_check_directories_probe()
423 {
424 while IFS="" read -r d; do
425 case "$d" in
426 *%*)
427 continue
428 ;;
429 *)
430 [ -d "${d}/." ] || return 1
431 ;;
432 esac
433 done
434 }
435
436 ######################################################
437 # check a set of directories is available
438 # directories are read from stdin
439 ######################################################
440 ctdb_check_directories()
441 {
442 ctdb_check_directories_probe || {
443 echo "ERROR: $service_name directory \"$d\" not available"
444 exit 1
445 }
446 }
447
448 ######################################################
449 # check a set of tcp ports
450 # usage: ctdb_check_tcp_ports <ports...>
451 ######################################################
452
453 # Check whether something is listening on all of the given TCP ports
454 # using the "ctdb checktcpport" command.
455 ctdb_check_tcp_ports()
456 {
457 if [ -z "$1" ]; then
458 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
459 exit 1
460 fi
461
462 for _p; do # process each function argument (port)
463 _cmd="$CTDB checktcpport $_p"
464 _out=$($_cmd 2>&1)
465 _ret=$?
466 case "$_ret" in
467 0)
468 echo "$service_name not listening on TCP port $_p"
469 return 1
470 ;;
471 98)
472 # Couldn't bind, something already listening, next port
473 continue
474 ;;
475 *)
476 echo "unexpected error (${_ret}) running \"${_cmd}\""
477 if [ -n "$_out" ]; then
478 echo "$_out"
479 fi
480 return $_ret
481 ;;
482 esac
483 done
484
485 # All ports listening
486 return 0
487 }
488
489 ######################################################
490 # check a unix socket
491 # usage: ctdb_check_unix_socket SOCKPATH
492 ######################################################
493 ctdb_check_unix_socket()
494 {
495 _sockpath="$1"
496
497 if [ -z "$_sockpath" ]; then
498 echo "ERROR: ctdb_check_unix_socket() requires socket path"
499 return 1
500 fi
501
502 _out=$(ss -l -xH "src ${_sockpath}")
503 if [ -z "$_out" ]; then
504 echo "ERROR: ${service_name} not listening on ${_sockpath}"
505 return 1
506 fi
507 }
508
509 ################################################
510 # kill off any TCP connections with the given IP
511 ################################################
512
513 kill_tcp_summarise()
514 {
515 _mode="$1"
516 _count="$2"
517 _method="$3"
518
519 _connections=$(get_tcp_connections_for_ip "$_ip")
520 if [ -z "$_connections" ]; then
521 _remaining=0
522 else
523 _remaining=$(echo "$_connections" | wc -l)
524 fi
525
526 case "$_mode" in
527 total)
528 _total="$_count"
529 _killed=$((_total - _remaining))
530 ;;
531 killed)
532 _killed="$_count"
533 _total=$((_killed + _remaining))
534 ;;
535 esac
536
537 _t="${_killed}/${_total}"
538 _m=""
539 if [ -n "$_method" ]; then
540 _m=", using ${_method}"
541 fi
542 echo "Killed ${_t} TCP connections to released IP ${_ip}${_m}"
543 if [ -n "$_connections" ]; then
544 echo "Remaining connections:"
545 echo "$_connections" | sed -e 's|^| |'
546 fi
547 }
548
549 kill_tcp_connections()
550 {
551 _iface="$1"
552 _ip="$2"
553
554 _oneway=false
555 if [ "$3" = "oneway" ]; then
556 _oneway=true
557 fi
558
559 case "$CTDB_KILLTCP_USE_SS_KILL" in
560 yes | try)
561 _killcount=$(ss -K -tnH state established src "$_ip" | wc -l)
562 kill_tcp_summarise "killed" "$_killcount" "ss -K"
563 if [ "$CTDB_KILLTCP_USE_SS_KILL" = "yes" ]; then
564 return
565 fi
566 ;;
567 esac
568
569 get_tcp_connections_for_ip "$_ip" | {
570 _killcount=0
571 _connections=""
572 _nl="
573 "
574 while read -r _dst _src; do
575 _destport="${_dst##*:}"
576 __oneway=$_oneway
577 case $_destport in
578 # we only do one-way killtcp for CIFS
579 139 | 445) __oneway=true ;;
580 esac
581
582 _connections="${_connections}${_nl}${_src} ${_dst}"
583 if ! $__oneway; then
584 _connections="${_connections}${_nl}${_dst} ${_src}"
585 fi
586
587 _killcount=$((_killcount + 1))
588 done
589
590 if [ $_killcount -eq 0 ]; then
591 return
592 fi
593
594 if [ -n "$CTDB_KILLTCP_DEBUGLEVEL" ]; then
595 _debuglevel="$CTDB_KILLTCP_DEBUGLEVEL"
596 else
597 _debuglevel="$CTDB_DEBUGLEVEL"
598 fi
599 echo "$_connections" |
600 CTDB_DEBUGLEVEL="$_debuglevel" \
601 "${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
602 echo "Failed to kill TCP connections"
603 return
604 }
605
606 _method=""
607 if [ "$CTDB_KILLTCP_USE_SS_KILL" = "try" ]; then
608 _method="ctdb_killtcp"
609 fi
610 kill_tcp_summarise "total" "$_killcount" "$_method"
611 }
612 }
613
614 ##################################################################
615 # kill off the local end for any TCP connections with the given IP
616 ##################################################################
617 kill_tcp_connections_local_only()
618 {
619 kill_tcp_connections "$@" "oneway"
620 }
621
622 ##################################################################
623 # tickle any TCP connections with the given IP
624 ##################################################################
625 tickle_tcp_connections()
626 {
627 _ip="$1"
628
629 # Get connections, both directions
630 _conns=$(get_tcp_connections_for_ip "$_ip" |
631 awk '{ print $1, $2 ; print $2, $1 }')
632
633 if [ -z "$_conns" ]; then
634 return
635 fi
636
637 echo "$_conns" | awk '{ print "Tickle TCP connection", $1, $2 }'
638 echo "$_conns" | ctdb tickle
639 }
640
641 get_tcp_connections_for_ip()
642 {
643 _ip="$1"
644
645 ss -tnH state established "src [$_ip]" | awk '{print $3, $4}'
646 }
647
648 ########################################################
649
650 add_ip_to_iface()
651 {
652 _iface=$1
653 _ip=$2
654 _maskbits=$3
655
656 # Ensure interface is up
657 ip link set "$_iface" up ||
658 die "Failed to bringup interface $_iface"
659
660 # Only need to define broadcast for IPv4
661 case "$_ip" in
662 *:*) _bcast="" ;;
663 *) _bcast="brd +" ;;
664 esac
665
666 # Intentionally unquoted multi-word value here
667 # shellcheck disable=SC2086
668 ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
669 echo "Failed to add $_ip/$_maskbits on dev $_iface"
670 return 1
671 }
672
673 # Wait 5 seconds for IPv6 addresses to stop being tentative...
674 if [ -z "$_bcast" ]; then
675 for _x in $(seq 1 10); do
676 ip addr show to "${_ip}/128" | grep -q "tentative" || break
677 sleep 0.5
678 done
679
680 # If the address was a duplicate then it won't be on the
681 # interface so flag an error.
682 _t=$(ip addr show to "${_ip}/128")
683 case "$_t" in
684 "")
685 echo "Failed to add $_ip/$_maskbits on dev $_iface"
686 return 1
687 ;;
688 *tentative* | *dadfailed*)
689 echo "Failed to add $_ip/$_maskbits on dev $_iface"
690 ip addr del "$_ip/$_maskbits" dev "$_iface"
691 return 1
692 ;;
693 esac
694 fi
695 }
696
697 delete_ip_from_iface()
698 {
699 _iface=$1
700 _ip=$2
701 _maskbits=$3
702
703 # This could be set globally for all interfaces but it is probably
704 # better to avoid surprises, so limit it the interfaces where CTDB
705 # has public IP addresses. There isn't anywhere else convenient
706 # to do this so just set it each time. This is much cheaper than
707 # remembering and re-adding secondaries.
708 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
709
710 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
711 echo "Failed to del $_ip on dev $_iface"
712 return 1
713 }
714 }
715
716 # If the given IP is hosted then print 2 items: maskbits and iface
717 ip_maskbits_iface()
718 {
719 _addr="$1"
720
721 case "$_addr" in
722 *:*) _bits=128 ;;
723 *) _bits=32 ;;
724 esac
725 ip addr show to "${_addr}/${_bits}" 2>/dev/null |
726 awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
727 sub("@.*", "", iface) }
728 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
729 print mask, iface }'
730 }
731
732 drop_ip()
733 {
734 _addr="${1%/*}" # Remove optional maskbits
735
736 # Intentional word splitting here
737 # shellcheck disable=SC2046
738 set -- $(ip_maskbits_iface "$_addr")
739 if [ -n "$1" ]; then
740 _maskbits="$1"
741 _iface="$2"
742 echo "Removing public address $_addr/$_maskbits from device $_iface"
743 delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
744 fi
745 }
746
747 have_public_addresses()
748 {
749 [ -f "${CTDB_BASE}/public_addresses" ]
750 }
751
752 # This sets $public_ifaces as a side-effect.
753 get_public_ifaces()
754 {
755 # Get all the interfaces listed in the public_addresses file
756 public_ifaces=$(sed -e '/^#.*/d' \
757 -e 's/^[^\t ]*[\t ]*//' \
758 -e 's/,/ /g' \
759 -e 's/[\t ]*$//' "${CTDB_BASE}/public_addresses")
760
761 # Get the interfaces for which CTDB has public IPs configured.
762 # That is, for all but the 1st line, get the 1st field.
763 ctdb_ifaces=$($CTDB -X ifaces | sed -e '1d' -e 's@^|@@' -e 's@|.*@@')
764
765 # Add $ctdb_ifaces and make $public_ifaces unique
766 # Use word splitting to squash whitespace
767 # shellcheck disable=SC2086
768 public_ifaces=$(echo $public_ifaces $ctdb_ifaces | tr ' ' '\n' | sort -u)
769 }
770
771 drop_all_public_ips()
772 {
773 # _x is intentionally ignored
774 # shellcheck disable=SC2034
775 while read -r _ip _x; do
776 case "$_ip" in
777 \#*) continue ;;
778 esac
779 drop_ip "$_ip"
780 done <"${CTDB_BASE}/public_addresses"
781 }
782
783 flush_route_cache()
784 {
785 set_proc_maybe sys/net/ipv4/route/flush 1
786 set_proc_maybe sys/net/ipv6/route/flush 1
787 }
788
789 ########################################################
790 # Interface monitoring
791
792 # If the interface is a virtual one (e.g. VLAN) then get the
793 # underlying interface
794 interface_get_real()
795 {
796 _iface="$1"
797
798 # If $_iface is a VLAN (i.e. contains an '@') then strip every
799 # before the '@', otherwise print the whole interface
800 echo "${_iface##*@}"
801 }
802
803 # Check whether an interface is operational
804 interface_monitor()
805 {
806 _iface="$1"
807
808 _iface_info=$(ip -br link show "$_iface" 2>&1) || {
809 echo "ERROR: Monitored interface ${_iface} does not exist"
810 return 1
811 }
812
813 # If the interface is a virtual one (e.g. VLAN) then get the
814 # underlying interface.
815 _realiface=$(interface_get_real "${_iface_info%% *}")
816
817 if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null); then
818 # This is a bond: various monitoring strategies
819 echo "$_bi" | grep -q 'Currently Active Slave: None' && {
820 echo "ERROR: No active slaves for bond device ${_realiface}"
821 return 1
822 }
823 echo "$_bi" | grep -q '^MII Status: up' || {
824 echo "ERROR: public network interface ${_realiface} is down"
825 return 1
826 }
827 echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
828 # This works around a bug in the driver where the
829 # overall bond status can be up but none of the actual
830 # physical interfaces have a link.
831 echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
832 echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
833 return 1
834 }
835 }
836
837 return 0
838 else
839 # Not a bond
840 case "$_iface" in
841 lo*)
842 # loopback is always working
843 return 0
844 ;;
845 ib*)
846 # we don't know how to test ib links
847 return 0
848 ;;
849 *)
850 ethtool "$_iface" | grep -q 'Link detected: yes' || {
851 # On some systems, this is not successful when a
852 # cable is plugged but the interface has not been
853 # brought up previously. Bring the interface up
854 # and try again...
855 ip link set "$_iface" up
856 ethtool "$_iface" | grep -q 'Link detected: yes' || {
857 echo "ERROR: No link on the public network interface ${_iface}"
858 return 1
859 }
860 }
861 return 0
862 ;;
863 esac
864 fi
865 }
866
867 ########################################################
868 # Simple counters
869 _ctdb_counter_common()
870 {
871 [ $# -le 1 ] || die "usage: _ctdb_counter_common [name]"
872
873 if [ $# -eq 1 ]; then
874 _counter_name="${1}.failcount"
875 else
876 _counter_name="failcount"
877 fi
878
879 if [ -z "$script_state_dir" ]; then
880 die "ctdb_counter_* functions need ctdb_setup_state_dir()"
881 fi
882
883 _counter_file="${script_state_dir}/${_counter_name}"
884 }
885 # Some code passes an argument
886 # shellcheck disable=SC2120
887 ctdb_counter_init()
888 {
889 _ctdb_counter_common "$1"
890
891 : >"$_counter_file"
892 }
893 ctdb_counter_incr()
894 {
895 _ctdb_counter_common "$1"
896
897 # unary counting using newlines!
898 echo >>"$_counter_file"
899 }
900 ctdb_counter_get()
901 {
902 _ctdb_counter_common "$1"
903 # unary counting!
904 _val=$(wc -c 2>/dev/null <"$_counter_file" || echo 0)
905 # Strip leading spaces from output of wc (on freebsd)
906 # shellcheck disable=SC2086
907 echo $_val
908 }
909 ctdb_counter_exists()
910 {
911 _ctdb_counter_common "$1"
912 [ -e "$_counter_file" ]
913 }
914
915 #
916 # Fail counter/threshold combination to control warnings and node unhealthy
917 #
918
919 _failcount_validate_threshold()
920 {
921 case "$1" in
922 "") return 1 ;; # A failure that doesn't need a warning
923 *)
924 if echo "$1" | grep -qx '[0-9]*'; then
925 return 0
926 fi
927
928 echo "WARNING: ${1} is an invalid threshold in \"${2}\" check"
929 return 1
930 ;;
931 esac
932 }
933
934 _failcount_common()
935 {
936 _thing="$1"
937
938 _counter=$(echo "$_thing" | sed -e 's@/@_SLASH_@g' -e 's@ @_@g')
939 }
940
941 failcount_init()
942 {
943 _thing="$1"
944
945 _failcount_common "$_thing"
946
947 ctdb_counter_init "$_counter"
948 }
949
950 failcount_reset()
951 {
952 _thing="$1"
953
954 _failcount_common "$_thing"
955
956 _failcount=$(ctdb_counter_get "$_counter")
957 if [ "$_failcount" -eq 0 ]; then
958 return
959 fi
960
961 printf 'NOTICE: %s: no longer failing\n' "$_thing"
962 ctdb_counter_init "$_counter"
963 }
964
965 failcount_incr()
966 {
967 _thing="$1"
968 _thresholds="$2"
969 _output="$3"
970
971 _failcount_common "$_thing"
972
973 ctdb_counter_incr "$_counter"
974 _failcount=$(ctdb_counter_get "$_counter")
975
976 case "$_thresholds" in
977 *:*)
978 _warn_threshold="${_thresholds%:*}"
979 _unhealthy_threshold="${_thresholds#*:}"
980 ;;
981 "")
982 _warn_threshold=1
983 _unhealthy_threshold=""
984 ;;
985 *)
986 _warn_threshold="$_thresholds"
987 _unhealthy_threshold=""
988 ;;
989 esac
990
991 if _failcount_validate_threshold "$_unhealthy_threshold" "$_thing"; then
992 if [ "$_failcount" -ge "$_unhealthy_threshold" ]; then
993 printf 'ERROR: %s: fail count %d >= threshold %d\n' \
994 "$_thing" \
995 "$_failcount" \
996 "$_unhealthy_threshold"
997 # Only print output when exceeding the
998 # unhealthy threshold
999 if [ "$_failcount" -eq "$_unhealthy_threshold" ] && \
1000 [ -n "$_output" ]; then
1001 echo "$_output"
1002 fi
1003 exit 1
1004 fi
1005 fi
1006
1007 if _failcount_validate_threshold "$_warn_threshold" "$_thing"; then
1008 if [ "$_failcount" -lt "$_warn_threshold" ]; then
1009 return 0
1010 fi
1011 fi
1012
1013 printf 'WARNING: %s: fail count %d >= threshold %d\n' \
1014 "$_thing" \
1015 "$_failcount" \
1016 "$_warn_threshold"
1017 if [ "$_failcount" -eq "$_warn_threshold" ] && [ -n "$_output" ]; then
1018 # Only print output when exceeding the warning threshold
1019 echo "$_output"
1020 fi
1021 }
1022
1023 ########################################################
1024
1025 # ctdb_setup_state_dir <type> <name>
1026 # Sets/creates script_state_dir)
1027 ctdb_setup_state_dir()
1028 {
1029 [ $# -eq 2 ] || die "usage: ctdb_setup_state_dir <type> <name>"
1030
1031 _type="$1"
1032 _name="$2"
1033
1034 script_state_dir="${CTDB_SCRIPT_VARDIR}/${_type}/${_name}"
1035
1036 mkdir -p "$script_state_dir" ||
1037 die "Error creating script state dir \"${script_state_dir}\""
1038 }
1039
1040 ##################################################################
1041 # Reconfigure a service on demand
1042
1043 _ctdb_service_reconfigure_common()
1044 {
1045 if [ -z "$script_state_dir" ]; then
1046 die "ctdb_service_*_reconfigure() needs ctdb_setup_state_dir()"
1047 fi
1048
1049 _ctdb_service_reconfigure_flag="${script_state_dir}/need_reconfigure"
1050 }
1051
1052 ctdb_service_needs_reconfigure()
1053 {
1054 _ctdb_service_reconfigure_common
1055 [ -e "$_ctdb_service_reconfigure_flag" ]
1056 }
1057
1058 ctdb_service_set_reconfigure()
1059 {
1060 _ctdb_service_reconfigure_common
1061 : >"$_ctdb_service_reconfigure_flag"
1062 }
1063
1064 ctdb_service_unset_reconfigure()
1065 {
1066 _ctdb_service_reconfigure_common
1067 rm -f "$_ctdb_service_reconfigure_flag"
1068 }
1069
1070 ctdb_service_reconfigure()
1071 {
1072 echo "Reconfiguring service \"${service_name}\"..."
1073 ctdb_service_unset_reconfigure
1074 service_reconfigure || return $?
1075 # Intentionally have this use $service_name as default
1076 # shellcheck disable=SC2119
1077 ctdb_counter_init
1078 }
1079
1080 # Default service_reconfigure() function does nothing.
1081 service_reconfigure()
1082 {
1083 :
1084 }
1085
1086 # Default service_start() and service_stop() functions.
1087
1088 # These may be overridden in an eventscript.
1089 service_start()
1090 {
1091 service "$service_name" start
1092 }
1093
1094 service_stop()
1095 {
1096 service "$service_name" stop
1097 }
1098
1099 ##################################################################
1100
1101 # This exists only for backward compatibility with 3rd party scripts
1102 # that call it
1103 ctdb_standard_event_handler()
1104 {
1105 :
1106 }
1107
1108 iptables_wrapper()
1109 {
1110 _family="$1"
1111 shift
1112 if [ "$_family" = "inet6" ]; then
1113 _iptables_cmd="ip6tables"
1114 else
1115 _iptables_cmd="iptables"
1116 fi
1117
1118 # iptables doesn't like being re-entered, so flock-wrap it.
1119 flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
1120 }
1121
1122 # AIX (and perhaps others?) doesn't have mktemp
1123 # type is commonly supported and more portable than which(1)
1124 # shellcheck disable=SC2039
1125 if ! type mktemp >/dev/null 2>&1; then
1126 mktemp()
1127 {
1128 _dir=false
1129 if [ "$1" = "-d" ]; then
1130 _dir=true
1131 shift
1132 fi
1133 _d="${TMPDIR:-/tmp}"
1134 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null |
1135 cksum |
1136 awk '{print $1}')
1137 _t="${_d}/tmp.${_hex10}"
1138 (
1139 umask 077
1140 if $_dir; then
1141 mkdir "$_t"
1142 else
1143 : >"$_t"
1144 fi
1145 )
1146 echo "$_t"
1147 }
1148 fi
1149
1150 ######################################################################
1151 # NFS callout handling
1152
1153 nfs_callout_init()
1154 {
1155 _state_dir="$1"
1156
1157 if [ -z "$CTDB_NFS_CALLOUT" ]; then
1158 CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
1159 fi
1160 # Always export, for statd callout
1161 export CTDB_NFS_CALLOUT
1162
1163 # If the callout wants to use this then it must create it
1164 export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"
1165
1166 # Export, if set, for use by clustered NFS callouts
1167 if [ -n "$CTDB_NFS_SHARED_STATE_DIR" ]; then
1168 export CTDB_NFS_SHARED_STATE_DIR
1169 fi
1170 if [ -n "$CTDB_NFS_EXPORTS_FILE" ]; then
1171 export CTDB_NFS_EXPORTS_FILE
1172 fi
1173
1174 nfs_callout_cache="${_state_dir}/nfs_callout_cache"
1175 nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
1176 nfs_callout_cache_ops="${nfs_callout_cache}/ops"
1177 }
1178
1179 nfs_callout_register()
1180 {
1181 mkdir -p "$nfs_callout_cache_ops"
1182 rm -f "$nfs_callout_cache_ops"/*
1183
1184 echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
1185
1186 _t=$("$CTDB_NFS_CALLOUT" "register")
1187 if [ -n "$_t" ]; then
1188 echo "$_t" |
1189 while IFS="" read -r _op; do
1190 touch "${nfs_callout_cache_ops}/${_op}"
1191 done
1192 else
1193 touch "${nfs_callout_cache_ops}/ALL"
1194 fi
1195 }
1196
1197 nfs_callout()
1198 {
1199 # Re-run registration if $CTDB_NFS_CALLOUT has changed
1200 _prev=""
1201 if [ -r "$nfs_callout_cache_callout" ]; then
1202 read -r _prev <"$nfs_callout_cache_callout"
1203 fi
1204 if [ "$CTDB_NFS_CALLOUT" != "$_prev" ]; then
1205 nfs_callout_register
1206 fi
1207
1208 # Run the operation if it is registered...
1209 if [ -e "${nfs_callout_cache_ops}/${1}" ] ||
1210 [ -e "${nfs_callout_cache_ops}/ALL" ]; then
1211 "$CTDB_NFS_CALLOUT" "$@"
1212 fi
1213 }
1214
1215 ########################################################
1216 # tickle handling
1217 ########################################################
1218
1219 update_tickles()
1220 {
1221 tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
1222 mkdir -p "$tickledir"
1223
1224 # If not hosting any public IPs then can't have any connections...
1225 if [ ! -s "$CTDB_MY_PUBLIC_IPS_CACHE" ]; then
1226 return
1227 fi
1228
1229 # IPs ss filter
1230 _ip_filter=""
1231 while read -r _ip; do
1232 _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
1233 done <"$CTDB_MY_PUBLIC_IPS_CACHE"
1234
1235 # Record our current tickles in a temporary file
1236 _my_tickles="${tickledir}/all.tickles.$$"
1237 while read -r _i; do
1238 $CTDB -X gettickles "$_i" |
1239 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1240 done <"$CTDB_MY_PUBLIC_IPS_CACHE" |
1241 sort >"$_my_tickles"
1242
1243 # Record connections to our public IPs in a temporary file.
1244 # This temporary file is in CTDB's private state directory and
1245 # $$ is used to avoid a very rare race involving CTDB's script
1246 # debugging. No security issue, nothing to see here...
1247 _my_connections="${tickledir}/all.connections.$$"
1248 # Parentheses are needed around the IP filter for precedence but
1249 # the parentheses can't be empty!
1250 ss -tnH state established "${_ip_filter:+( ${_ip_filter} )}" |
1251 awk '{print $4, $3}' |
1252 sort >"$_my_connections"
1253
1254 # Add tickles for connections that we haven't already got tickles for
1255 comm -23 "$_my_connections" "$_my_tickles" |
1256 $CTDB addtickle
1257
1258 # Remove tickles for connections that are no longer there
1259 comm -13 "$_my_connections" "$_my_tickles" |
1260 $CTDB deltickle
1261
1262 rm -f "$_my_connections" "$_my_tickles"
1263
1264 # Remove stale files from killed scripts
1265 # Files can't have spaces in name, more portable than -print0/-0
1266 # shellcheck disable=SC2038
1267 (cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
1268 }
1269
1270 ########################################################
1271 # load a site local config file
1272 ########################################################
1273
1274 [ -x "${CTDB_BASE}/rc.local" ] && {
1275 . "${CTDB_BASE}/rc.local"
1276 }
1277
1278 [ -d "${CTDB_BASE}/rc.local.d" ] && {
1279 for i in "${CTDB_BASE}/rc.local.d"/*; do
1280 [ -x "$i" ] && . "$i"
1281 done
1282 }
1283
1284 script_name="${0##*/}" # basename