Debian Codegraph @

heartbeat/podman-etcd

raw
1 #!/bin/sh
2 #
3 # The podman etcd HA resource agent creates and launches a etcd podman
4 # container based off a supplied podman image. Containers managed by
5 # this agent are both created and removed upon the agent's start and
6 # stop actions.
7 #
8 # Based on the podman resource agent.
9 #
10 # Copyright (c) 2014 David Vossel <[email protected]>
11 # Michele Baldessari <[email protected]>
12 # All Rights Reserved.
13 #
14 # This program is free software; you can redistribute it and/or modify
15 # it under the terms of version 2 of the GNU General Public License as
16 # published by the Free Software Foundation.
17 #
18 # This program is distributed in the hope that it would be useful, but
19 # WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21 #
22 # Further, this software is distributed without any warranty that it is
23 # free of the rightful claim of any third person regarding infringement
24 # or the like. Any license provided herein, whether implied or
25 # otherwise, applies only to this software file. Patent licenses, if
26 # any, provided herein do not apply to combinations of this program with
27 # other software, or any other product whatsoever.
28 #
29 # You should have received a copy of the GNU General Public License
30 # along with this program; if not, write the Free Software Foundation,
31 # Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
32 #
33
34 #######################################################################
35 # Initialization:
36
37 : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
38 . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
39
40 # Parameter defaults
41 OCF_RESKEY_image_default="default"
42 OCF_RESKEY_pod_manifest_default="/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/external-etcd-pod/pod.yaml"
43 OCF_RESKEY_etcd_certs_dir_default="/etc/kubernetes/static-pod-resources/etcd-certs"
44 OCF_RESKEY_name_default="etcd"
45 OCF_RESKEY_nic_default="br-ex"
46 OCF_RESKEY_authfile_default="/var/lib/kubelet/config.json"
47 OCF_RESKEY_allow_pull_default="1"
48 OCF_RESKEY_reuse_default="0"
49 OCF_RESKEY_oom_default="-997"
50 OCF_RESKEY_config_location_default="/var/lib/etcd"
51 OCF_RESKEY_backup_location_default="/var/lib/etcd"
52 OCF_RESKEY_max_backup_snapshots_default="3"
53
54 : ${OCF_RESKEY_image=${OCF_RESKEY_image_default}}
55 : ${OCF_RESKEY_pod_manifest=${OCF_RESKEY_pod_manifest_default}}
56 : ${OCF_RESKEY_etcd_certs_dir=${OCF_RESKEY_etcd_certs_dir_default}}
57 : ${OCF_RESKEY_name=${OCF_RESKEY_name_default}}
58 : ${OCF_RESKEY_nic=${OCF_RESKEY_nic_default}}
59 : ${OCF_RESKEY_authfile=${OCF_RESKEY_authfile_default}}
60 : ${OCF_RESKEY_allow_pull=${OCF_RESKEY_allow_pull_default}}
61 : ${OCF_RESKEY_reuse=${OCF_RESKEY_reuse_default}}
62 : ${OCF_RESKEY_oom=${OCF_RESKEY_oom_default}}
63 : ${OCF_RESKEY_config_location=${OCF_RESKEY_config_location_default}}
64 : ${OCF_RESKEY_backup_location=${OCF_RESKEY_backup_location_default}}
65 : ${OCF_RESKEY_max_backup_snapshots=${OCF_RESKEY_max_backup_snapshots_default}}
66
67
68 #######################################################################
69
70 meta_data()
71 {
72 cat <<END
73 <?xml version="1.0"?>
74 <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
75 <resource-agent name="podman-etcd" version="1.0">
76 <version>1.0</version>
77
78 <longdesc lang="en">
79 The podman-etcd HA resource agent creates and launches a etcd podman
80 container based off a supplied podman image. Containers managed by
81 this agent are both created and removed upon the agent's start and
82 stop actions.
83 </longdesc>
84 <shortdesc lang="en">Podman etcd container resource agent.</shortdesc>
85
86 <parameters>
87 <parameter name="pod_manifest" required="0" unique="0">
88 <longdesc lang="en">
89 The Pod manifest with the configuration for Etcd.
90 </longdesc>
91 <shortdesc lang="en">Etcd pod manifest</shortdesc>
92 <content type="string" default="${OCF_RESKEY_pod_manifest_default}"/>
93 </parameter>
94
95 <parameter name="etcd_certs_dir" required="0" unique="0">
96 <longdesc lang="en">
97 The Etcd certificates directory mounted into the etcd container.
98 The agent will monitor this directory for changes and restart the etcd container if the certificates have changed.
99 </longdesc>
100 <shortdesc lang="en">Etcd certificates directory</shortdesc>
101 <content type="string" default="${OCF_RESKEY_etcd_certs_dir_default}"/>
102 </parameter>
103
104 <parameter name="image" required="0" unique="0">
105 <longdesc lang="en">
106 The podman image to base this container off of.
107 </longdesc>
108 <shortdesc lang="en">podman image</shortdesc>
109 <content type="string" default="${OCF_RESKEY_image_default}"/>
110 </parameter>
111
112 <parameter name="name" required="0" unique="0">
113 <longdesc lang="en">
114 The name to give the created container. By default this will
115 be that resource's instance name.
116 </longdesc>
117 <shortdesc lang="en">podman container name</shortdesc>
118 <content type="string" default="${OCF_RESKEY_name_default}"/>
119 </parameter>
120
121 <parameter name="node_ip_map" unique="0" required="1">
122 <longdesc lang="en">
123 A mapping of node names to IPs.
124
125 This takes the form of:
126 n1:ip1;n2:ip2
127
128 where the etcd container on n1 would have IP ip1
129 </longdesc>
130 <shortdesc lang="en">Container node name to IP mapping</shortdesc>
131 <content type="string"/>
132 </parameter>
133
134 <parameter name="nic" unique="0">
135 <longdesc lang="en">
136 Network interface to lookup interface for host.
137 </longdesc>
138 <shortdesc lang="en">Network interface</shortdesc>
139 <content type="string" default="${OCF_RESKEY_nic_default}"/>
140 </parameter>
141
142 <parameter name="authfile" required="0" unique="0">
143 <longdesc lang="en">
144 Path of the authentication file.
145
146 The file is created by podman login.
147 </longdesc>
148 <shortdesc lang="en">Path of the authentication file </shortdesc>
149 <content type="string" default="${OCF_RESKEY_authfile_default}"/>
150 </parameter>
151
152 <parameter name="allow_pull" unique="0">
153 <longdesc lang="en">
154 Allow the image to be pulled from the configured podman registry when
155 the image does not exist locally. NOTE, this can drastically increase
156 the time required to start the container if the image repository is
157 pulled over the network.
158 </longdesc>
159 <shortdesc lang="en">Allow pulling non-local images</shortdesc>
160 <content type="boolean" default="${OCF_RESKEY_allow_pull_default}"/>
161 </parameter>
162
163 <parameter name="run_opts" required="0" unique="0">
164 <longdesc lang="en">
165 Add options to be appended to the 'podman run' command which is used
166 when creating the container during the start action. This option allows
167 users to do things such as setting a custom entry point and injecting
168 environment variables into the newly created container. Note the '-d'
169 option is supplied regardless of this value to force containers to run
170 in the background.
171
172 NOTE: Do not explicitly specify the --name argument in the run_opts. This
173 agent will set --name using either the resource's instance or the name
174 provided in the 'name' argument of this agent.
175
176 </longdesc>
177 <shortdesc lang="en">run options</shortdesc>
178 <content type="string"/>
179 </parameter>
180
181 <parameter name="run_cmd" required="0" unique="0">
182 <longdesc lang="en">
183 Specify a command to launch within the container once
184 it has initialized.
185 </longdesc>
186 <shortdesc lang="en">run command</shortdesc>
187 <content type="string"/>
188 </parameter>
189
190 <parameter name="run_cmd_opts" required="0" unique="0">
191 <longdesc lang="en">
192 Options to be added to the 'run_cmd'.
193 </longdesc>
194 <shortdesc lang="en">run command options</shortdesc>
195 <content type="string"/>
196 </parameter>
197
198 <parameter name="mount_points" required="0" unique="0">
199 <longdesc lang="en">
200 A comma separated list of directories that the container is expecting to use.
201 The agent will ensure they exist by running 'mkdir -p'
202 </longdesc>
203 <shortdesc lang="en">Required mount points</shortdesc>
204 <content type="string"/>
205 </parameter>
206
207 <parameter name="monitor_cmd" required="0" unique="0">
208 <longdesc lang="en">
209 Specify the full path of a command to launch within the container to check
210 the health of the container. This command must return 0 to indicate that
211 the container is healthy. A non-zero return code will indicate that the
212 container has failed and should be recovered.
213
214 Note: Using this method for monitoring processes inside a container
215 is not recommended, as containerd tries to track processes running
216 inside the container and does not deal well with many short-lived
217 processes being spawned. Ensure that your container monitors its
218 own processes and terminates on fatal error rather than invoking
219 a command from the outside.
220 </longdesc>
221 <shortdesc lang="en">monitor command</shortdesc>
222 <content type="string"/>
223 </parameter>
224
225 <parameter name="force_kill" required="0" unique="0">
226 <longdesc lang="en">
227 Kill a container immediately rather than waiting for it to gracefully
228 shutdown
229 </longdesc>
230 <shortdesc lang="en">force kill</shortdesc>
231 <content type="boolean"/>
232 </parameter>
233
234 <parameter name="reuse" required="0" unique="0">
235 <longdesc lang="en">
236 Allow the container to be reused once it is stopped. By default,
237 containers get removed once they are stopped. Enable this option
238 to have the particular one persist when this happens.
239 </longdesc>
240 <shortdesc lang="en">reuse container</shortdesc>
241 <content type="boolean" default="${OCF_RESKEY_reuse_default}"/>
242 </parameter>
243
244 <parameter name="drop_in_dependency" required="0" unique="0">
245 <longdesc lang="en">
246 Use transient drop-in files to add extra dependencies to the systemd
247 scopes associated to the container. During reboot, this prevents systemd
248 to stop the container before pacemaker.
249 </longdesc>
250 <shortdesc lang="en">drop-in dependency</shortdesc>
251 <content type="boolean"/>
252 </parameter>
253
254 <parameter name="oom" required="0" unique="0">
255 <longdesc lang="en">
256 Tune the host's Out-Of-Memory (OOM) preferences for containers (accepts values from -1000 to 1000).
257 Default to same OOM score as system-node-critical
258 https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#node-out-of-memory-behavior
259 </longdesc>
260 <shortdesc lang="en">OOM for container</shortdesc>
261 <content type="integer" default="${OCF_RESKEY_oom_default}"/>
262 </parameter>
263
264 <parameter name="config_location" required="0" unique="0">
265 <longdesc lang="en">
266 The directory where the resource agent stores its state files, such as the generated etcd configuration and a copy of the pod manifest.
267 </longdesc>
268 <shortdesc lang="en">Resource agent state directory</shortdesc>
269 <content type="string" default="${OCF_RESKEY_config_location_default}"/>
270 </parameter>
271
272 <parameter name="backup_location" required="0" unique="0">
273 <longdesc lang="en">
274 The directory where the resource agent stores its backups.
275 </longdesc>
276 <shortdesc lang="en">Resource agent backup directory</shortdesc>
277 <content type="string" default="${OCF_RESKEY_backup_location_default}"/>
278 </parameter>
279
280 <parameter name="max_backup_snapshots" required="0" unique="0">
281 <longdesc lang="en">
282 Maximum number of etcd database snapshots to retain. When a new snapshot is created,
283 older snapshots will be automatically removed to maintain this limit. This helps
284 control storage usage while ensuring recent backups are available for recovery.
285 Set max_backup_snapshots=0 to disable backups.
286 </longdesc>
287 <shortdesc lang="en">Maximum number of backup snapshots to retain</shortdesc>
288 <content type="integer" default="${OCF_RESKEY_max_backup_snapshots_default}"/>
289 </parameter>
290
291 </parameters>
292
293 <actions>
294 <action name="start" timeout="600s" />
295 <action name="stop" timeout="90s" />
296 <action name="monitor" timeout="25s" interval="30s" depth="0" />
297 <action name="meta-data" timeout="5s" />
298 <action name="validate-all" timeout="30s" />
299 </actions>
300 </resource-agent>
301 END
302 }
303
304 #######################################################################
305 REQUIRE_IMAGE_PULL=0
306
307 podman_usage()
308 {
309 cat <<END
310 usage: $0 {start|stop|monitor|validate-all|meta-data}
311
312 Expects to have a fully populated OCF RA-compliant environment set.
313 END
314 }
315
316 etcd_certificates_hash_manager()
317 {
318 local action="$1"
319 local current_hash
320 local stored_hash
321
322 # If the certs directory doesn't exist, consider it unchanged
323 if [ ! -d "$OCF_RESKEY_etcd_certs_dir" ]; then
324 ocf_log warn "certificates directory $OCF_RESKEY_etcd_certs_dir does not exist, skipping certificate monitoring"
325 return $OCF_SUCCESS
326 fi
327
328 # Calculate hash of all certificate files, ignore key files to avoid accidental disclosure of sensitive information
329 # we only need to monitor the certificate files to detect changes.
330 if ! current_hash=$(find "$OCF_RESKEY_etcd_certs_dir" -type f \( -name "*.crt" \) -exec sha256sum {} \; | sort | sha256sum | cut -d' ' -f1); then
331 ocf_log err "failed to calculate certificate files hash"
332 return $OCF_ERR_GENERIC
333 fi
334
335 # If no stored hash exists, create one and return success
336 if [ ! -f "$ETCD_CERTS_HASH_FILE" ]; then
337 echo "$current_hash" > "$ETCD_CERTS_HASH_FILE"
338 ocf_log info "created initial certificate hash: $current_hash"
339 return $OCF_SUCCESS
340 fi
341
342 case "$action" in
343 "update")
344 if ! echo "$current_hash" > "$ETCD_CERTS_HASH_FILE"; then
345 ocf_log err "failed to update certificate hash file $ETCD_CERTS_HASH_FILE"
346 fi
347 ocf_log info "updated certificate hash: $current_hash"
348 ;;
349 "check")
350 if ! stored_hash=$(cat "$ETCD_CERTS_HASH_FILE"); then
351 ocf_log err "failed to read stored certificate hash from $ETCD_CERTS_HASH_FILE"
352 # This should not happen but if for some reason we can not read the stored hash,
353 # use the current hash and log the error but allow etcd to run as long as possible.
354 stored_hash="$current_hash"
355 fi
356 if [ "$current_hash" != "$stored_hash" ]; then
357 ocf_exit_reason "$NODENAME etcd certificate files have changed (stored: $stored_hash, current: $current_hash)"
358 return $OCF_ERR_GENERIC
359 fi
360 ;;
361 *)
362 ocf_log err "unsupported action: $action"
363 return $OCF_ERR_GENERIC
364 ;;
365 esac
366
367 return $OCF_SUCCESS
368 }
369
370 monitor_cmd_exec()
371 {
372 local rc=$OCF_SUCCESS
373 local out
374 local attempt
375 # 3 attempts × 5s = 15s worst case, fits within the 25s monitor timeout.
376 # The health check normally completes in <1s; the 5s per-attempt timeout
377 # is a safety net for when the container's process namespace is slow.
378 local max_attempts=3
379 local attempt_timeout=5
380
381 for attempt in $(seq 1 $max_attempts); do
382 out=$(timeout $attempt_timeout podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1)
383 rc=$?
384 # 125: no container with name or ID ${CONTAINER} found
385 # 126: container state improper (not running)
386 # 127: any other error
387 # 255: podman 2+: container not running
388 case "$rc" in
389 125|126|255)
390 return $OCF_NOT_RUNNING
391 ;;
392 0)
393 ocf_log debug "monitor cmd passed: exit code = $rc"
394 return $OCF_SUCCESS
395 ;;
396 *)
397 ocf_log warn "monitor cmd failed (rc=$rc), output: $out"
398 rc=$OCF_ERR_GENERIC
399 ;;
400 esac
401 done
402
403 return $rc
404 }
405
406 container_exists()
407 {
408 local rc
409 local out
410
411 out=$(podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1)
412 rc=$?
413 # 125: no container with name or ID ${CONTAINER} found
414 if [ $rc -ne 125 ]; then
415 return 0
416 fi
417 return 1
418 }
419
420 # archive_current_container archives the current
421 # podman etcd container and its configuration files.
422 archive_current_container()
423 {
424 # don't attempt to archive a container that doesn't exist
425 if ! container_exists; then
426 return
427 fi
428
429 # delete any container named "*-previous", or we won't be able to archive the current container.
430 if podman inspect "${CONTAINER}-previous" >/dev/null 2>&1; then
431 ocf_log info "removing old archived container '$CONTAINER-previous'"
432 if ! ocf_run podman rm --volumes --force "$CONTAINER-previous"; then
433 ocf_log warn "could not remove old archived container (podman rm failed, error code: $?). Won't be able to archive current container"
434 return
435 fi
436 fi
437
438 ocf_log info "archiving '$CONTAINER' container as '$CONTAINER-previous' for debugging purposes"
439 if ! ocf_run podman rename "$CONTAINER" "$CONTAINER-previous"; then
440 ocf_log err "could not archive container '$CONTAINER', error code: $?"
441 return
442 fi
443
444 # archive corresponding etcd configuration files
445 local files_to_archive=""
446 for file in "$OCF_RESKEY_authfile" "$POD_MANIFEST_COPY" "$ETCD_CONFIGURATION_FILE" "$ETCD_CERTS_HASH_FILE"; do
447 if [ -f "$file" ]; then
448 files_to_archive="$files_to_archive $file"
449 else
450 ocf_log warn "file '$file' is missing and won't be archived"
451 fi
452 done
453
454 if [ -z "$files_to_archive" ]; then
455 ocf_log warn "could not find any file to archive."
456 return
457 fi
458
459 # NOTE: tar will override any existing archive as wanted
460 # shellcheck disable=SC2086
461 if ! ocf_run tar --create --verbose --gzip --file "$ETCD_BACKUP_FILE" $files_to_archive; then
462 ocf_log warn "container archived successfully, but configuration backup failed (error: $?). Container debugging available, but without matching configuration files"
463 else
464 ocf_log info "container configuration also archived in '$ETCD_BACKUP_FILE'"
465 fi
466 }
467
468 # Correctly wraps an ipv6 in [] for url otherwise use return normal ipv4 address.
469 ip_url() {
470 local ip_addr=$1
471 local value
472 if echo "$ip_addr" | grep -q ":" ; then
473 value="[$ip_addr]"
474 else
475 value="$ip_addr"
476 fi
477 echo "https://$value"
478 }
479
480 attribute_node_ip()
481 {
482 local action="$1"
483 local attribute="node_ip"
484 local ip_addr name
485
486 # TODO: We can retrieve both the local and peer IP addresses from this map, which eliminates the need to use CIB to share them between nodes
487 for node in $(echo "$OCF_RESKEY_node_ip_map" | sed "s/\s//g;s/;/ /g"); do
488 name=$(echo "$node" | cut -d: -f1)
489 # ignore other nodes
490 if [ "$name" != "$NODENAME" ]; then
491 continue
492 fi
493 ip_addr=$(echo "$node" | cut -d: -f2-) # Grab everything after the first : this covers ipv4/ipv6
494 done
495
496 if [ -z "$ip_addr" ]; then
497 ocf_log err "could not get local ip address from node_ip_map: '$OCF_RESKEY_node_ip_map'"
498 return 1
499 fi
500
501 case "$action" in
502 get)
503 echo "$ip_addr"
504 ;;
505 update)
506 if ! crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$ip_addr"; then
507 rc="$?"
508 ocf_log err "could not set $attribute to $ip_addr, error code: $rc"
509 return "$rc"
510 fi
511 ;;
512 clear)
513 crm_attribute --name "$attribute" --delete
514 ;;
515 *)
516 ocf_log err "unsupported $action for $attribute"
517 return $OCF_ERR_GENERIC
518 ;;
519 esac
520 }
521
522 attribute_node_ip_peer() {
523 local peer_name
524 peer_name=$(get_peer_node_name)
525 crm_attribute --query --name "node_ip" --node "$peer_name" | awk -F"value=" '{print $2}'
526 }
527
528 get_env_from_manifest() {
529 local env_var_name="$1"
530 local env_var_value
531
532 # The agent waits for the manifest to exist before starting, so the
533 # file should exist already, but this check is included for robustness.
534 if [ ! -f "$OCF_RESKEY_pod_manifest" ]; then
535 ocf_log err "external etcd pod manifest ($OCF_RESKEY_pod_manifest) not found"
536 exit "$OCF_ERR_INSTALLED"
537 fi
538
539 env_var_value=$(jq -r ".spec.containers[].env[] | select( .name == \"$env_var_name\" ).value" "$OCF_RESKEY_pod_manifest")
540 rc=$?
541 if [ $rc -ne 0 ]; then
542 ocf_log err "could not find environment variable $env_var_name in etcd pod manifest, error code: $rc"
543 exit "$OCF_ERR_INSTALLED"
544 fi
545
546 ocf_log debug "ETCD pod environment variable $env_var_name: $env_var_value"
547
548 echo "$env_var_value"
549 }
550
551 # etcd configuration file expects duration to be expressed in nanoseconds
552 convert_duration_in_nanoseconds() {
553 local duration=$1
554 local value unit nanoseconds
555
556 if [ -z "$duration" ]; then
557 ocf_log err "convert_duration_in_nanoseconds: no duration provided"
558 return 1
559 fi
560
561 if ! echo "$duration" | grep -qE '^[0-9]+[numµ]?s$'; then
562 ocf_log err "convert_duration_in_nanoseconds: invalid duration format \"$duration\". Expected format: <number><unit> where unit is one of s, ms, us, µs, ns"
563 return 1
564 fi
565
566 # Extract numeric value and unit from duration string
567 value=$(echo "$duration" | sed 's/[^0-9]*$//')
568 unit=$(echo "$duration" | sed 's/^[0-9]*//')
569
570 case "$unit" in
571 ns)
572 nanoseconds=$value
573 ;;
574 us|µs)
575 nanoseconds=$((value * 1000))
576 ;;
577 ms)
578 nanoseconds=$((value * 1000000))
579 ;;
580 s)
581 nanoseconds=$((value * 1000000000))
582 ;;
583 *)
584 # this should not happen as the input is already validated
585 ocf_log err "convert_duration_in_nanoseconds: unknown duration unit \"$unit\""
586 return 1
587 ;;
588 esac
589
590 echo "$nanoseconds"
591 }
592
593 prepare_env() {
594 local name ip ipurl standalone_node
595
596 NODEIP="$(attribute_node_ip get)"
597 NODEIPURL=$(ip_url $NODEIP)
598
599 if is_force_new_cluster; then
600 ALL_ETCD_ENDPOINTS="$NODEIPURL:2379"
601 ETCD_INITIAL_CLUSTER_STATE="new"
602 ETCD_INITIAL_CLUSTER="$NODENAME=$NODEIPURL:2380"
603 else
604 ETCD_INITIAL_CLUSTER_STATE="existing"
605 for node in $(echo "$OCF_RESKEY_node_ip_map" | sed "s/\s//g;s/;/ /g"); do
606 name=$(echo "$node" | cut -d: -f1)
607 ip=$(echo "$node" | cut -d: -f2-) # Grab everything after the first : this covers ipv4/ipv6
608 ipurl="$(ip_url $ip)"
609 if [ -z "$name" ] || [ -z "$ip" ]; then
610 ocf_exit_reason "name or ip missing for 1 or more nodes"
611 exit $OCF_ERR_CONFIGURED
612 fi
613
614 [ -z "$ALL_ETCD_ENDPOINTS" ] && ALL_ETCD_ENDPOINTS="$ipurl:2379" || ALL_ETCD_ENDPOINTS="$ALL_ETCD_ENDPOINTS,$ipurl:2379"
615 [ -z "$ETCD_INITIAL_CLUSTER" ] && ETCD_INITIAL_CLUSTER="$name=$ipurl:2380" || ETCD_INITIAL_CLUSTER="$ETCD_INITIAL_CLUSTER,$name=$ipurl:2380"
616 done
617 fi
618
619 ETCD_CIPHER_SUITES=$(get_env_from_manifest "ETCD_CIPHER_SUITES")
620 ETCD_DATA_DIR=$(get_env_from_manifest "ETCD_DATA_DIR")
621 if [ ! -d "$ETCD_DATA_DIR" ]; then
622 ocf_log err "could not find data-dir at path \"$ETCD_DATA_DIR\""
623 return "$OCF_ERR_ARGS"
624 else
625 ocf_log info "using data-dir: $ETCD_DATA_DIR"
626 fi
627 ETCD_ELECTION_TIMEOUT=$(get_env_from_manifest "ETCD_ELECTION_TIMEOUT")
628 ETCD_ENABLE_PPROF=$(get_env_from_manifest "ETCD_ENABLE_PPROF")
629 ETCD_WARNING_APPLY_DURATION=$(get_env_from_manifest "ETCD_WARNING_APPLY_DURATION")
630 ETCD_WATCH_PROGRESS_NOTIFY_INTERVAL=$(get_env_from_manifest "ETCD_WATCH_PROGRESS_NOTIFY_INTERVAL")
631 ETCD_HEARTBEAT_INTERVAL=$(get_env_from_manifest "ETCD_HEARTBEAT_INTERVAL")
632 ETCD_QUOTA_BACKEND_BYTES=$(get_env_from_manifest "ETCD_QUOTA_BACKEND_BYTES")
633 ETCD_SOCKET_REUSE_ADDRESS=$(get_env_from_manifest "ETCD_SOCKET_REUSE_ADDRESS")
634
635 SERVER_CACERT=$(get_env_from_manifest "ETCDCTL_CACERT")
636 ETCD_PEER_CERT=$(get_env_from_manifest "ETCDCTL_CERT")
637 ETCD_PEER_KEY=$(get_env_from_manifest "ETCDCTL_KEY")
638
639 LISTEN_CLIENT_URLS="0.0.0.0"
640 LISTEN_PEER_URLS="0.0.0.0"
641 LISTEN_METRICS_URLS="0.0.0.0"
642
643 return $OCF_SUCCESS
644 }
645
646 compute_bump_revision() {
647 local rc
648
649 # Same logic used by cluster-etcd-operator quorum-restore-pod utility.
650 # see https://github.com/openshift/cluster-etcd-operator/blob/215998939f5223da916622c71fd07d17656faf6b/bindata/etcd/quorum-restore-pod.yaml#L26-L34
651 # set a default value: 1bn would be an etcd running at 1000 writes/s for about eleven days.
652 BUMP_REV=$ETCD_BUMP_REV_DEFAULT
653 if [ ! -f "${ETCD_REVISION_JSON}" ]; then
654 ocf_log err "could not compute bump revision: ${ETCD_REVISION_JSON} not found. Defaulting to ${ETCD_BUMP_REV_DEFAULT} revision bump"
655 return
656 fi
657
658 # this will bump by the amount of 20% of the last known live revision.
659 if ! COMPUTED_BUMP=$(jq -r "(.maxRaftIndex*${ETCD_REVISION_BUMP_PERCENTAGE}|floor)" "${ETCD_REVISION_JSON}"); then
660 ocf_log err "could not compute maxRaftIndex for bump revision, jq error code: $?. Defaulting to ${ETCD_BUMP_REV_DEFAULT} revision bump"
661 return
662 fi
663
664 if [ -z "${COMPUTED_BUMP}" ] || [ "${COMPUTED_BUMP}" -le 0 ] || [ "${COMPUTED_BUMP}" -gt "${ETCD_BUMP_REV_DEFAULT}" ]; then
665 ocf_log err "computed bump revision (${COMPUTED_BUMP}) is invalid. Defaulting to ${ETCD_BUMP_REV_DEFAULT} revision bump"
666 return
667 fi
668
669 BUMP_REV="${COMPUTED_BUMP}"
670 ocf_log info "bumping etcd revisions by ${BUMP_REV}"
671 }
672
673 generate_etcd_configuration() {
674 if is_force_new_cluster; then
675 compute_bump_revision
676 # The embedded newline is required for correct YAML formatting.
677 FORCE_NEW_CLUSTER_CONFIG="force-new-cluster: true
678 force-new-cluster-bump-amount: $BUMP_REV"
679 else
680 FORCE_NEW_CLUSTER_CONFIG="force-new-cluster: false"
681 fi
682
683 # the space indentation for client-transport-security and peer-transport-security
684 # is required for correct YAML formatting.
685 # TODO: replace flags deprecated in Etcd v3.6
686 cat > "$ETCD_CONFIGURATION_FILE" << EOF
687 logger: zap
688 log-level: info
689 snapshot-count: 10000
690 name: $NODENAME
691 data-dir: $ETCD_DATA_DIR
692 $FORCE_NEW_CLUSTER_CONFIG
693 socket-reuse-address: $ETCD_SOCKET_REUSE_ADDRESS
694 election-timeout: $ETCD_ELECTION_TIMEOUT
695 enable-pprof: $ETCD_ENABLE_PPROF
696 heartbeat-interval: $ETCD_HEARTBEAT_INTERVAL
697 quota-backend-bytes: $ETCD_QUOTA_BACKEND_BYTES
698 initial-advertise-peer-urls: "$NODEIPURL:2380"
699 listen-peer-urls: "$(ip_url ${LISTEN_PEER_URLS}):2380"
700 listen-client-urls: "$(ip_url ${LISTEN_CLIENT_URLS}):2379,unixs://${NODEIP}:0"
701 initial-cluster: $ETCD_INITIAL_CLUSTER
702 initial-cluster-state: $ETCD_INITIAL_CLUSTER_STATE
703 client-transport-security:
704 cert-file: /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-${NODENAME}.crt
705 key-file: /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-${NODENAME}.key
706 client-cert-auth: true
707 trusted-ca-file: $SERVER_CACERT
708 peer-transport-security:
709 cert-file: $ETCD_PEER_CERT
710 key-file: $ETCD_PEER_KEY
711 client-cert-auth: true
712 trusted-ca-file: $SERVER_CACERT
713 advertise-client-urls: "$NODEIPURL:2379"
714 listen-metrics-urls: "$(ip_url ${LISTEN_METRICS_URLS}):9978"
715 metrics: extensive
716 experimental-initial-corrupt-check: true
717 experimental-max-learners: 1
718 experimental-warning-apply-duration: $(convert_duration_in_nanoseconds "$ETCD_WARNING_APPLY_DURATION")
719 experimental-watch-progress-notify-interval: $(convert_duration_in_nanoseconds "$ETCD_WATCH_PROGRESS_NOTIFY_INTERVAL")
720 EOF
721 rc=$?
722 if [ $rc -ne 0 ]; then
723 ocf_log err "could not create etcd configuration, 'cat' error code: $rc"
724 return $OCF_ERR_CONFIGURED
725 fi
726
727 # Append cipher suites from the env variable where the entries are comma separated.
728 {
729 if [ -n "$ETCD_CIPHER_SUITES" ]; then
730 echo "cipher-suites:"
731 echo "$ETCD_CIPHER_SUITES" | tr ',' '\n' | while read -r cipher; do
732 echo " - \"$cipher\""
733 done
734 fi
735 } >> "$ETCD_CONFIGURATION_FILE"
736 rc=$?
737 if [ $rc -ne 0 ]; then
738 ocf_log err "could not append cipher suites to etcd configuration, error code: $rc"
739 return $OCF_ERR_CONFIGURED
740 fi
741
742 return $OCF_SUCCESS
743 }
744
745 # Remove etcd member directory to allow the node to rejoin the cluster as a learner.
746 #
747 # When a node rejoins an etcd cluster, it must start fresh as a learner to prevent
748 # data inconsistencies. This function removes the member directory and syncs to disk.
749 #
750 # Returns:
751 # OCF_SUCCESS - Member directory successfully removed
752 # OCF_ERR_GENERIC - Failed to remove member directory (critical error)
753 wipe_data_folder_for_learner()
754 {
755 ocf_log info "deleting etcd member directory ($ETCD_MEMBER_DIR) to enable learner rejoin"
756 if ! rm -rf "$ETCD_MEMBER_DIR"; then
757 ocf_log err "could not delete etcd member directory ($ETCD_MEMBER_DIR), error code: $?"
758 return $OCF_ERR_GENERIC
759 fi
760 sync
761 return $OCF_SUCCESS
762 }
763
764
765 # Calculate available disk space in bytes for a given directory.
766 #
767 # This function queries the filesystem and returns available space in bytes.
768 # It converts df output (KB) to bytes for consistent size comparisons.
769 #
770 # Arguments:
771 # $1 - Target directory path to check
772 #
773 # Returns:
774 # OCF_SUCCESS - Available space in bytes (via stdout)
775 # OCF_ERR_GENERIC - Failed to determine available space (error message via stdout)
776 get_available_space_in_directory()
777 {
778 local target_dir=$1
779 local available_space_kb
780 local available_space_bytes
781
782 available_space_kb=$(df -P "$target_dir" | awk 'NR==2 {print $4}' 2>&1)
783
784 # Validate output is numeric
785 if ! echo "$available_space_kb" | grep -q '^[0-9]\+$'; then
786 echo "df command failed or returned invalid value: $available_space_kb"
787 return $OCF_ERR_GENERIC
788 fi
789
790 available_space_bytes=$((available_space_kb*1024))
791 echo "$available_space_bytes"
792 return $OCF_SUCCESS
793 }
794
795 # Archive etcd database with backup and cleanup
796 #
797 # This function creates a backup copy of the etcd database, validates it, and
798 # removes old backups according to the retention policy. Backups are optional
799 # and can be disabled by setting max_backup_snapshots=0.
800 #
801 # Error handling strategy:
802 # All backup failures return OCF_SUCCESS to prevent blocking cluster recovery.
803 # Backups are beneficial but not critical for recovery operations.
804 #
805 # NOTE: This function cannot use etcdctl/etcdutl utilities because the etcd
806 # server is not running when this backup is performed.
807 archive_data_folder()
808 {
809 local backup_dir="$OCF_RESKEY_backup_location"
810 local etcd_db_path="$ETCD_MEMBER_DIR/snap/db"
811
812 if [ "$OCF_RESKEY_max_backup_snapshots" -eq 0 ]; then
813 ocf_log debug "etcd backup disabled (max_backup_snapshots=0)"
814 return $OCF_SUCCESS
815 fi
816
817 # Check if the etcd database file exists
818 if [ ! -f "$etcd_db_path" ]; then
819 ocf_log warn "backup skipped: etcd database file not found at '$etcd_db_path'"
820 return $OCF_SUCCESS
821 fi
822
823 # Ensure backup directory exists
824 if [ ! -d "$backup_dir" ]; then
825 ocf_log debug "creating backup directory: '$backup_dir'"
826 if ! mkdir -p "$backup_dir"; then
827 ocf_log warn "backup skipped: failed to create backup directory '$backup_dir'"
828 return $OCF_SUCCESS
829 fi
830 fi
831
832 ocf_log debug "checking disk space: backup_dir=$backup_dir"
833 local available_space_bytes
834 if ! available_space_bytes=$(get_available_space_in_directory "$backup_dir"); then
835 ocf_log warn "backup skipped: could not compute available disk space in '$backup_dir', error msg: $available_space_bytes"
836 return $OCF_SUCCESS
837 fi
838
839 local required_space_bytes
840 required_space_bytes=$(stat -c %s "$etcd_db_path" 2>&1)
841 if ! echo "$required_space_bytes" | grep -q '^[0-9]\+$'; then
842 ocf_log warn "backup skipped: could not compute etcd database size at '$etcd_db_path', error msg: $required_space_bytes"
843 return $OCF_SUCCESS
844 fi
845
846 if [ "$required_space_bytes" -gt "$available_space_bytes" ]; then
847 ocf_log warn "backup skipped: insufficient disk space (required: ${required_space_bytes}B, available: ${available_space_bytes}B)"
848 return $OCF_SUCCESS
849 fi
850
851 # Generate timestamp and backup filename
852 local timestamp
853 timestamp=$(date +%Y%m%d-%H%M%S)
854
855 local backup_file
856 backup_file="$backup_dir/snapshot-$timestamp.db"
857
858 ocf_log info "creating etcd database backup: '$backup_file'"
859
860 # Create the backup by copying the database file (enable Copy-on-Write copy)
861 if ! cp --reflink=auto "$etcd_db_path" "$backup_file"; then
862 ocf_log warn "backup creation failed: could not copy '$etcd_db_path' to '$backup_file', error code: $?"
863 return $OCF_SUCCESS
864 fi
865
866 # Validate the backup file exists and has the expected size
867 if [ ! -f "$backup_file" ]; then
868 ocf_log warn "backup validation failed: snapshot file '$backup_file' does not exist"
869 return $OCF_SUCCESS
870 fi
871
872 local backup_size_bytes
873 backup_size_bytes=$(stat -c %s "$backup_file" 2>/dev/null || echo "0")
874 if [ "$backup_size_bytes" -ne "$required_space_bytes" ]; then
875 ocf_log warn "backup validation failed: size mismatch (expected: ${required_space_bytes}B, got: ${backup_size_bytes}B)"
876 rm -f "$backup_file"
877 return $OCF_SUCCESS
878 fi
879
880 ocf_log info "backup created successfully: $backup_file (${backup_size_bytes}B)"
881
882 # Cleanup old backups based on retention policy
883 cleanup_old_backups "$backup_dir"
884
885 return $OCF_SUCCESS
886 }
887
888 cleanup_old_backups()
889 {
890 local backup_dir="$1"
891 local max_snapshots="$OCF_RESKEY_max_backup_snapshots"
892 local backup_count
893 local backups_to_remove
894 local old_backups
895
896 # Validate max_snapshots is a positive integer
897 if ! echo "$max_snapshots" | grep -q '^[1-9][0-9]*$'; then
898 ocf_log warn "invalid max_backup_snapshots value. Positive integer expected, got '$max_snapshots' instead, skipping cleanup"
899 return $OCF_SUCCESS
900 fi
901
902 # Count existing backup files
903 backup_count=$(find "$backup_dir" -maxdepth 1 -name "snapshot-*.db" -type f 2>/dev/null | wc -l)
904
905 if [ "$backup_count" -le "$max_snapshots" ]; then
906 ocf_log info "backup count ($backup_count) is within retention limit ($max_snapshots), no cleanup needed"
907 return $OCF_SUCCESS
908 fi
909
910 # Calculate how many backups to remove
911 backups_to_remove=$((backup_count - max_snapshots))
912 ocf_log info "removing $backups_to_remove old backup(s) to maintain retention limit of $max_snapshots"
913
914 # Find oldest backups sorted by modification time
915 # -t sorts by modification time, -r reverses (oldest first)
916 # -print0 and -0 handle filenames with spaces/special characters
917 old_backups=$(find "$backup_dir" -maxdepth 1 -name "snapshot-*.db" -type f -print0 2>/dev/null | \
918 xargs -0 -r ls -tr | \
919 head -n "$backups_to_remove")
920
921 if [ -n "$old_backups" ]; then
922 ocf_log info "removing old backups: $old_backups"
923 if ! echo "$old_backups" | xargs -r rm -f; then
924 ocf_log warn "failed to remove some old backups, error code: $?"
925 fi
926 fi
927
928 return $OCF_SUCCESS
929 }
930
931 etcd_pod_container_exists() {
932 local count_matches
933 # Check whether the etcd pod exists on the same node (including stopped/exited containers)
934 count_matches=$(crictl pods --label app=etcd -q | xargs -I {} crictl ps -a --pod {} -o json | jq -r '.containers[].metadata | select ( .name == "etcd" ).name' | wc -l)
935 if [ "$count_matches" -eq 1 ]; then
936 # etcd pod found
937 return 0
938 fi
939 # etcd pod not found
940 return 1
941 }
942
943 attribute_node_cluster_id()
944 {
945 local action="$1"
946 local value
947 local rc
948 value=$(jq -r ".clusterId" "$ETCD_REVISION_JSON")
949 rc=$?
950 if [ $rc -ne 0 ]; then
951 # Log the error but return success to avoid monitor failure if the file is not available yet.
952 #This should not block cluster recovery.
953 ocf_log err "could not get cluster_id, error code: $rc"
954 return $OCF_SUCCESS
955 fi
956
957 case "$action" in
958 get)
959 echo "$value"
960 ;;
961 update)
962 crm_attribute --type nodes --node "$NODENAME" --name "cluster_id" --update "$value"
963 rc=$?
964 if [ $rc -ne 0 ]; then
965 # Log the error but return success to avoid monitor failure if we can not update the attribute.
966 ocf_log err "could not update cluster_id, error code: $rc"
967 return $OCF_SUCCESS
968 fi
969 ;;
970 *)
971 ocf_log err "unsupported $action for attribute_node_cluster_id"
972 return $OCF_ERR_GENERIC
973 ;;
974 esac
975 }
976
977 attribute_node_cluster_id_peer()
978 {
979 local nodename
980
981 nodename=$(get_peer_node_name)
982 crm_attribute --query --type nodes --node "$nodename" --name "cluster_id" | awk -F"value=" '{print $2}'
983 }
984
985 attribute_node_revision()
986 {
987 local action="$1"
988 local value
989 local attribute="revision"
990
991 if ! value=$(jq -r ".maxRaftIndex" "$ETCD_REVISION_JSON"); then
992 rc=$?
993 ocf_log err "could not get $attribute, error code: $rc"
994 return "$rc"
995 fi
996
997 case "$action" in
998 get)
999 echo "$value"
1000 ;;
1001 update)
1002 crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value"
1003 rc=$?
1004 if [ $rc -ne 0 ]; then
1005 # Log the error but return success to avoid monitor failure if we can not update the attribute.
1006 ocf_log err "could not update etcd $attribute, error code: $rc"
1007 return $OCF_SUCCESS
1008 fi
1009 ;;
1010 *)
1011 ocf_log err "unsupported $action for attribute_node_revision"
1012 return "$OCF_ERR_GENERIC"
1013 ;;
1014 esac
1015 }
1016
1017 attribute_node_revision_peer()
1018 {
1019 local nodename
1020 nodename=$(get_peer_node_name)
1021 crm_attribute --query --type nodes --node "$nodename" --name "revision" | awk -F"value=" '{print $2}'
1022 }
1023
1024 # Converts a decimal number to hexadecimal format with validation
1025 # Args: $1 - decimal number (test for non-negative integer too)
1026 # Returns: 0 on success, OCF_ERR_GENERIC on invalid input
1027 # Outputs: hexadecimal representation to stdout
1028 decimal_to_hex() {
1029 local dec=$1
1030
1031 if ! echo "$dec" | grep -q "^[1-9][0-9]*$"; then
1032 ocf_log err "Invalid member ID format: '$dec' (expected decimal number)"
1033 return $OCF_ERR_GENERIC
1034 fi
1035
1036 printf "%x" "$dec"
1037 return $OCF_SUCCESS
1038 }
1039
1040 attribute_node_member_id()
1041 {
1042 local action="$1"
1043 local attribute="member_id"
1044
1045 if ! container_exists; then
1046 # we need a running container to execute etcdctl.
1047 return 0
1048 fi
1049
1050 case "$action" in
1051 get)
1052 # When we need this value at the agent startup we don't have a etcd
1053 # container running, so we always get this value from CIB
1054 crm_attribute --query --type nodes --node "$NODENAME" --name "$attribute" | awk -F"value=" '{print $2}'
1055 ;;
1056 update)
1057 local member_list_json
1058 member_list_json=$(get_member_list_json)
1059 ocf_log info "member list: $member_list_json"
1060 if [ -z "$member_list_json" ] ; then
1061 ocf_log err "could not get $attribute: could not get member list JSON"
1062 return $OCF_ERR_GENERIC
1063 fi
1064
1065 local value value_hex rc
1066 value=$(echo -n "$member_list_json" | jq -r ".header.member_id")
1067 rc=$?
1068 if [ $rc -ne 0 ]; then
1069 # Log the error but return success to avoid monitor failure if the file is not available yet.
1070 ocf_log err "could not get $attribute from member list JSON, error code: $rc"
1071 return $OCF_SUCCESS
1072 fi
1073
1074 # JSON member_id is decimal, while etcdctl command needs the hex version
1075 value_hex=$(decimal_to_hex "$value")
1076 rc=$?
1077 if [ $rc -ne 0 ]; then
1078 ocf_log err "could not convert decimal member_id '$value' to hex, error code: $rc"
1079 return $OCF_ERR_GENERIC
1080 fi
1081 crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value_hex"
1082 rc=$?
1083 if [ $rc -ne 0 ]; then
1084 # Log the error but return success to avoid monitor failure if we can not update the attribute.
1085 ocf_log err "could not update etcd $attribute, error code: $rc"
1086 return $OCF_SUCCESS
1087 fi
1088 ;;
1089 clear)
1090 crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --delete
1091 ;;
1092 *)
1093 ocf_log err "unsupported $action for attribute_node_member_id"
1094 return "$OCF_ERR_GENERIC"
1095 ;;
1096 esac
1097 }
1098
1099 # remove an etcd member identified by its IP from the member list
1100 remove_etcd_member_by_ip()
1101 {
1102 local rc
1103 local out
1104 local member_ip
1105 local endpoint_url
1106 local member_list_json
1107 local stale_member_id
1108 local stale_member_id_hex
1109
1110 member_ip=$1
1111 member_list_json=$(get_member_list_json)
1112 rc=$?
1113 if [ "$rc" -ne 0 ] ; then
1114 ocf_log err "could not remove etcd member. Failed to get member list, error code: $rc"
1115 return $OCF_ERR_GENERIC
1116 fi
1117
1118 stale_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$member_ip\")) | any).ID")
1119 if [ -z "$stale_member_id" ]; then
1120 ocf_log err "could not remove etcd member. Failed to find member ID"
1121 return $OCF_ERR_GENERIC
1122 fi
1123
1124 # JSON member_id is decimal, while etcdctl command needs the hex version
1125 if ! stale_member_id_hex=$(decimal_to_hex "$stale_member_id"); then
1126 ocf_log err "could not remove etcd member. Failed to convert member_id '$stale_member_id' into hex format"
1127 return $OCF_ERR_GENERIC
1128 fi
1129
1130 endpoint_url=$(ip_url $(attribute_node_ip get))
1131 out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" member remove "$stale_member_id_hex" 2>&1)
1132 rc=$?
1133 if [ "$rc" -ne 0 ] ; then
1134 ocf_log err "could not remove etcd member. etcdctl member remove command failed, error code: $rc, output: $out"
1135 return $OCF_ERR_GENERIC
1136 fi
1137
1138 ocf_log info "$out"
1139 return $OCF_SUCCESS
1140 }
1141
1142 add_member_as_learner()
1143 {
1144 local rc
1145 local member_name=$1
1146 local member_ip=$2
1147 local endpoint_url=$(ip_url $(attribute_node_ip get))
1148 local peer_url=$(ip_url $member_ip)
1149 local i
1150 local max_retries
1151 local out
1152
1153 i=0
1154 max_retries=3
1155 while [ "$i" -lt "$max_retries" ]; do
1156 i=$((i + 1))
1157 ocf_log info "adding $member_name ($member_ip) to the member list as learner (attempt $i of $max_retries)"
1158 out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" \
1159 member add "$member_name" \
1160 --peer-urls="$peer_url:2380" \
1161 --learner 2>&1)
1162 rc=$?
1163 if [ "$rc" -eq 0 ]; then
1164 ocf_log info "$out"
1165 attribute_learner_node update "$member_name"
1166 return $OCF_SUCCESS
1167 fi
1168 if echo "$out" | grep -q "Peer URLs already exists"; then
1169 # etcd data might have stale membership data
1170 ocf_log warn "could not add member: Peer URLs already exists"
1171 remove_etcd_member_by_ip "$member_ip"
1172 else
1173 ocf_log warn "could not add member: $out"
1174 fi
1175 done
1176
1177 ocf_log err "could not add $member_name as learner, error code $rc, etcdctl output: $out"
1178 return $OCF_ERR_GENERIC
1179 }
1180
1181 set_force_new_cluster()
1182 {
1183 local rc
1184 crm_attribute --lifetime reboot --node "$NODENAME" --name "force_new_cluster" --update "$NODENAME"
1185 rc=$?
1186 if [ $rc -ne 0 ]; then
1187 ocf_log err "could not set force_new_cluster attribute to $NODENAME"
1188 fi
1189 return $rc
1190 }
1191
1192 # get_force_new_cluster returns a space-separated list of nodes that have the force_new_cluster attribute set.
1193 # Return values:
1194 # - Exit code 0 with non-empty output: One or more nodes have the force_new_cluster attribute set
1195 # - Exit code 0 with empty output: No nodes have the force_new_cluster attribute set
1196 # - Exit code 1 with empty output: Error occurred while querying the cluster nodes
1197 get_force_new_cluster()
1198 {
1199 local node nodes value
1200 local holders=""
1201
1202 if ! nodes=$(crm_node -l | awk '{print $2}'); then
1203 ocf_log err "could not get force_new_cluster attribute, crm_node error code: $?"
1204 return 1
1205 fi
1206 if [ -z "$nodes" ]; then
1207 ocf_log err "could not get force_new_cluster attribute, the list of nodes is empty"
1208 return 1
1209 fi
1210
1211 for node in $nodes; do
1212 if ! value=$(crm_attribute --query --lifetime reboot --name "force_new_cluster" --node "$node" 2>/dev/null | awk -F'value=' '{print $2}' | tr -d "'"); then
1213 ocf_log err "could not get force_new_cluster attribute, crm_attribut error code: $?"
1214 return 1
1215 fi
1216 if [ -n "$value" ]; then
1217 holders="$holders$node "
1218 fi
1219 done
1220 echo "$holders"
1221 }
1222
1223
1224 clear_force_new_cluster()
1225 {
1226 # only the holder of "force_new_cluster" attribute can delete it
1227 if ! is_force_new_cluster; then
1228 ocf_log info "force_new_cluster unset or not owned by $NODENAME"
1229 return $OCF_SUCCESS
1230 fi
1231
1232 if ! crm_attribute --delete --lifetime reboot --node "$NODENAME" --name "force_new_cluster"; then
1233 ocf_log err "could not clear force_new_cluster attribute, error code: $?"
1234 return $OCF_ERR_GENERIC
1235 fi
1236
1237 ocf_log info "$NODENAME: force_new_cluster attribute cleared"
1238 return $OCF_SUCCESS
1239 }
1240
1241
1242 is_force_new_cluster()
1243 {
1244 # Return 0 if 'force_new_cluster' is set on the current node, 1 otherwise.
1245 local fnc_holders
1246
1247 if ! fnc_holders=$(get_force_new_cluster); then
1248 ocf_exit_reason "is_force_new_cluster: Failed to get force_new_cluster node holders"
1249 exit $OCF_ERR_GENERIC
1250 fi
1251
1252 if echo "$fnc_holders" | grep -q -w "$NODENAME"; then
1253 ocf_log debug "$NODENAME has force_new_cluster set"
1254 return 0
1255 fi
1256
1257 ocf_log debug "force_new_cluster attribute is not set on $NODENAME"
1258 return 1
1259 }
1260
1261 is_standalone()
1262 {
1263 local standalone_node
1264
1265 standalone_node=$(get_standalone_node)
1266 if [ -z "$standalone_node" ]; then
1267 ocf_log debug "no node running standalone"
1268 return 1
1269 fi
1270
1271 if [ "$NODENAME" = "$standalone_node" ]; then
1272 ocf_log debug "$NODENAME is set as standalone"
1273 return 0
1274 fi
1275 ocf_log debug "$NODENAME is set as learner"
1276 return 1
1277
1278 }
1279
1280 set_standalone_node()
1281 {
1282 local rc
1283
1284 ocf_log info "add $NODENAME as standalone"
1285 crm_attribute --name "standalone_node" --update "$NODENAME"
1286 rc=$?
1287 if [ $rc -ne 0 ]; then
1288 ocf_log err "could not set standalone_node attribute to $NODENAME"
1289 fi
1290 return $rc
1291 }
1292
1293 get_standalone_node()
1294 {
1295 crm_attribute --query --name "standalone_node" | awk -F"value=" '{print $2}'
1296 }
1297
1298 clear_standalone_node()
1299 {
1300 crm_attribute --name "standalone_node" --delete
1301 }
1302
1303
1304 # Promotes an etcd learner member to a voting member
1305 # Args: $1 - learner member ID in decimal format
1306 # Returns: OCF_SUCCESS (even on expected promotion failures), OCF_ERR_GENERIC on conversion errors
1307 # Note: Promotion failures are expected and logged as info (peer may not be up-to-date)
1308 promote_learner_member()
1309 {
1310 local learner_member_id=$1
1311
1312 # JSON member_id is decimal, while etcdctl command needs the hex version
1313 if ! learner_member_id_hex=$(decimal_to_hex "$learner_member_id"); then
1314 ocf_log err "could not convert decimal member_id '$learner_member_id' to hex, error code: $?"
1315 return $OCF_ERR_GENERIC
1316 fi
1317 if ! ocf_run podman exec "${CONTAINER}" etcdctl member promote "$learner_member_id_hex" 2>&1; then
1318 # promotion is expected to fail if the peer is not yet up-to-date
1319 ocf_log info "could not promote member $learner_member_id_hex, error code: $?"
1320 return $OCF_ERR_GENERIC
1321 fi
1322 ocf_log info "successfully promoted member '$learner_member_id_hex'"
1323 return $OCF_SUCCESS
1324 }
1325
1326 # Reconciles etcd cluster member states
1327 # Promotes learner members or clears standalone/learner attributes as needed
1328 # Args: $1 - member list JSON from etcdctl
1329 # Returns: OCF_SUCCESS on completion, OCF_ERR_GENERIC on errors
1330 # Note: Only operates when exactly 2 started members are present
1331 reconcile_member_state()
1332 {
1333 local rc
1334 local member_list_json="$1"
1335
1336 # count only the started members, which have the ".name" JSON field
1337 number_of_started_members=$(printf "%s" "$member_list_json" | jq -r ".members[].name | select(. != null)" | wc -l)
1338 if [ "$number_of_started_members" -ne 2 ]; then
1339 ocf_log info "could not clear standalone_node, nor learner_node properties: found $number_of_started_members members, need 2"
1340 return $OCF_SUCCESS
1341 fi
1342
1343 learner_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .isLearner==true ).ID")
1344 rc=$?
1345 if [ $rc -ne 0 ]; then
1346 ocf_log err "could not get isLearner field from member list, error code: $rc"
1347 return $rc
1348 fi
1349
1350 if [ -n "$learner_member_id" ]; then
1351 if ! promote_learner_member "$learner_member_id"; then
1352 return $OCF_ERR_GENERIC
1353 fi
1354 # promotion succeded: continue to clear standalone_node and learner_node
1355 fi
1356
1357 if ! clear_standalone_node; then
1358 ocf_log error "could not clear standalone_node attribute, error code: $?"
1359 return $OCF_ERR_GENERIC
1360 fi
1361 if ! attribute_learner_node clear; then
1362 ocf_log error "could not clear learner_node attribute, error code: $?"
1363 return $OCF_ERR_GENERIC
1364 fi
1365
1366 return $OCF_SUCCESS
1367 }
1368
1369 attribute_learner_node()
1370 {
1371 local action="$1"
1372 local value="$2"
1373 local attribute="learner_node"
1374
1375 case "$action" in
1376 get)
1377 crm_attribute --query --name "$attribute" | awk -F"value=" '{print $2}'
1378 ;;
1379 update)
1380 if ! crm_attribute --name "$attribute" --update "$value"; then
1381 rc="$?"
1382 ocf_log err "could not set $attribute to $value, error code: $rc"
1383 return "$rc"
1384 fi
1385 ;;
1386 clear)
1387 crm_attribute --name "$attribute" --delete
1388 ;;
1389 *)
1390 ocf_log err "unsupported $action for $attribute"
1391 return $OCF_ERR_GENERIC
1392 ;;
1393 esac
1394 }
1395
1396 is_learner()
1397 {
1398 if [ "$NODENAME" = "$(attribute_learner_node get)" ]; then
1399 return 0
1400 fi
1401 return 1
1402 }
1403
1404 get_peer_node_name() {
1405 crm_node -l | awk '{print $2}' | grep -v "$NODENAME"
1406 }
1407
1408 # Calculate the count of truly active resources by excluding those being stopped.
1409 # According to Pacemaker documentation, during "Post-notification (stop) /
1410 # Pre-notification (start)" transitions, the true active resource count should be:
1411 # Active resources = $OCF_RESKEY_CRM_meta_notify_active_resource
1412 # minus $OCF_RESKEY_CRM_meta_notify_stop_resource
1413 # This handles the case where a resource appears in both the active and stop lists
1414 # during rapid restart scenarios (e.g., process crash recovery).
1415 get_truly_active_resources_count() {
1416 local active_list="$OCF_RESKEY_CRM_meta_notify_active_resource"
1417 local stop_list="$OCF_RESKEY_CRM_meta_notify_stop_resource"
1418 local truly_active=""
1419
1420 # If no active resources, return 0
1421 # Use word count to handle whitespace-only values
1422 if [ "$(echo "$active_list" | wc -w)" -eq 0 ]; then
1423 echo "0"
1424 return
1425 fi
1426
1427 # If no resources being stopped, return count of active resources
1428 # Use word count to handle whitespace-only values
1429 if [ "$(echo "$stop_list" | wc -w)" -eq 0 ]; then
1430 echo "$active_list" | wc -w
1431 return
1432 fi
1433
1434 # Filter out resources that are being stopped from the active list
1435 for resource in $active_list; do
1436 local is_stopping=0
1437 for stop_resource in $stop_list; do
1438 if [ "$resource" = "$stop_resource" ]; then
1439 is_stopping=1
1440 break
1441 fi
1442 done
1443 if [ $is_stopping -eq 0 ]; then
1444 truly_active="$truly_active $resource"
1445 fi
1446 done
1447
1448 # Count the truly active resources (trim leading space and count words)
1449 echo "$truly_active" | wc -w
1450 }
1451
1452 get_all_etcd_endpoints() {
1453 for node in $(echo "$OCF_RESKEY_node_ip_map" | sed "s/\s//g;s/;/ /g"); do
1454 name=$(echo "$node" | cut -d: -f1)
1455 ip=$(echo "$node" | cut -d: -f2-) # Grab everything after the first : this covers ipv4/ipv6
1456 ipurl="$(ip_url $ip)"
1457 if [ -z "$name" ] || [ -z "$ip" ]; then
1458 ocf_exit_reason "name or ip missing for 1 or more nodes"
1459 exit $OCF_ERR_CONFIGURED
1460 fi
1461
1462 [ -z "$ALL_ETCD_ENDPOINTS" ] && ALL_ETCD_ENDPOINTS="$ipurl:2379" || ALL_ETCD_ENDPOINTS="$ALL_ETCD_ENDPOINTS,$ipurl:2379"
1463 done
1464 echo "$ALL_ETCD_ENDPOINTS"
1465 }
1466
1467 get_endpoint_status_json()
1468 {
1469 # Get the status of all endpoints
1470 local all_etcd_endpoints
1471
1472 all_etcd_endpoints=$(get_all_etcd_endpoints)
1473 podman exec "${CONTAINER}" etcdctl endpoint status --command-timeout="$MONITOR_ETCDCTL_TIMEOUT" --endpoints="$all_etcd_endpoints" -w json
1474 }
1475
1476 get_member_list_json() {
1477 # Get the list of members visible to the current node
1478 local this_node_endpoint
1479
1480 this_node_endpoint="$(ip_url $(attribute_node_ip get)):2379"
1481 podman exec "${CONTAINER}" etcdctl member list --command-timeout="$MONITOR_ETCDCTL_TIMEOUT" --endpoints="$this_node_endpoint" -w json
1482 }
1483
1484 detect_cluster_leadership_loss()
1485 {
1486 endpoint_status_json=$(get_endpoint_status_json)
1487 ocf_log info "endpoint status: $endpoint_status_json"
1488
1489 count_endpoints=$(printf "%s" "$endpoint_status_json" | jq -r ".[].Endpoint" | wc -l)
1490 if [ "$count_endpoints" -eq 1 ]; then
1491 ocf_log info "one endpoint only: checking status errors"
1492 endpoint_status_errors=$(printf "%s" "$endpoint_status_json" | jq -r ".[0].Status.errors")
1493 if echo "$endpoint_status_errors" | grep -q "no leader"; then
1494 set_force_new_cluster
1495 set_standalone_node
1496 ocf_exit_reason "$NODENAME must force a new cluster"
1497 return $OCF_ERR_GENERIC
1498 fi
1499 if [ "$endpoint_status_errors" != "null" ]; then
1500 ocf_log err "unmanaged endpoint status error: $endpoint_status_errors"
1501 fi
1502 fi
1503
1504 return $OCF_SUCCESS
1505 }
1506
1507
1508 # Manages etcd peer membership by detecting and handling missing or rejoining peers
1509 # Adds missing peers as learners and reconciles member states when peers rejoin
1510 # Args: $1 - member list JSON from etcdctl
1511 # Returns: OCF_SUCCESS on completion, OCF_ERR_GENERIC on errors
1512 # Note: Iterates through all peer nodes to ensure proper cluster membership
1513 manage_peer_membership()
1514 {
1515 local member_list_json="$1"
1516 local peer_ip_map_entry
1517 local peer_member_name
1518 local peer_member_ip
1519 local peer_member_id
1520
1521 # Get peer node name and IP
1522 peer_ip_map_entry=$(echo "$OCF_RESKEY_node_ip_map" | tr ';' '\n' | grep -vF "$NODENAME")
1523 if [ -z "$peer_ip_map_entry" ]; then
1524 ocf_exit_reason "manage_peer_membership: could not parse node_ip_map: '$OCF_RESKEY_node_ip_map'"
1525 exit $OCF_ERR_CONFIGURED
1526 fi
1527 peer_member_name=$(echo "$peer_ip_map_entry" | cut -d: -f1)
1528 peer_member_ip=$(echo "$peer_ip_map_entry" | cut -d: -f2-)
1529
1530 # Parsing the member list's json output to find a "learner" member.
1531 # Example of .members[] instance fields in member list json format:
1532 # {
1533 # "ID": <member ID>,
1534 # "name": "<node hostname>",
1535 # "peerURLs": [
1536 # "https://<node IP>:2380"
1537 # ],
1538 # "clientURLs": [
1539 # "https://<node IP>:2379"
1540 # ]
1541 # }
1542 # NOTE: voting members have a "name" field but no "isLearner" field,
1543 # while learner members have "isLearner": true (boolean) but no "name" field, so we search for peerURLs matching.
1544 peer_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$peer_member_ip\")) | any).ID")
1545 if [ -z "$peer_member_id" ]; then
1546 ocf_log info "$peer_member_name is not in the members list"
1547 add_member_as_learner "$peer_member_name" "$peer_member_ip"
1548 set_standalone_node
1549 return
1550 fi
1551
1552 # Ensure learner_node attribute is always set when we have a learner member
1553 local learner_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .isLearner==true ).ID")
1554 local current_learner_node=$(attribute_learner_node get)
1555 if [ -n "$learner_member_id" ] && [ -z "$current_learner_node" ]; then
1556 ocf_log debug "$peer_member_name found as learner in member list, but learner_node attribute was not set. Updating"
1557 attribute_learner_node update "$peer_member_name"
1558 return
1559 fi
1560
1561 ocf_log debug "$peer_member_name is in the members list by IP: $peer_member_ip"
1562 # Errors from reconcile_member_state are logged internally. Ignoring them here prevents stopping a healthy voter agent; critical local failures are caught by detect_cluster_leadership_loss.
1563 reconcile_member_state "$member_list_json"
1564 }
1565
1566 check_peer()
1567 {
1568 # Check peers endpoint status and locally accessible member list
1569 local member_list_json
1570
1571 # we need a running container to execute etcdctl.
1572 if ! container_exists; then
1573 return $OCF_SUCCESS
1574 fi
1575
1576 if ! member_list_json=$(get_member_list_json); then
1577 ocf_log info "podman failed to get member list"
1578 detect_cluster_leadership_loss
1579 return $?
1580 fi
1581
1582 manage_peer_membership "$member_list_json"
1583 return $OCF_SUCCESS
1584 }
1585
1586 podman_simple_status()
1587 {
1588 local rc
1589
1590 # simple status is implemented via podman exec
1591 # everything besides success is considered "not running"
1592 monitor_cmd_exec
1593 rc=$?
1594 if [ $rc -ne $OCF_SUCCESS ]; then
1595 rc=$OCF_NOT_RUNNING;
1596 fi
1597 return $rc
1598 }
1599
1600 # get_time_since_last_heartbeat returns the time in seconds since the heartbeat file was last updated.
1601 #
1602 # Returns: time in seconds since last heartbeat, or empty string if file doesn't exist
1603 get_time_since_last_heartbeat()
1604 {
1605 local last_heartbeat
1606
1607 if [ ! -f "$CONTAINER_HEARTBEAT_FILE" ]; then
1608 return
1609 fi
1610
1611 last_heartbeat=$(cat "$CONTAINER_HEARTBEAT_FILE")
1612 echo $(($(date +%s) - last_heartbeat))
1613 }
1614
1615 # container_health_check performs comprehensive health monitoring for the container.
1616 # This function allows coordinated failure handling where the agent waits for
1617 # peer-initiated cluster recovery in case of container failure.
1618 #
1619 # Uses a state file to track container state:
1620 # - Container running: Update state file with current epoch, return "healthy"
1621 # - Container not running + no state file: Return "not-running" (never checked before)
1622 # - Container not running + state file: Failure detected, check force_new_cluster
1623 # - If force_new_cluster set: Return "failed-restart-now"
1624 # - Otherwise: Return "failed-wait-for-peer"
1625 #
1626 # Returns: healthy, not-running, failed-restart-now, failed-wait-for-peer
1627
1628 container_health_check()
1629 {
1630 local rc
1631
1632 # We rely on running podman exec to monitor the container
1633 # state because that command seems to be less prone to
1634 # performance issue under IO load.
1635 #
1636 # For probes to work, we expect cmd_exec to be able to report
1637 # when a container is not running. Here, we're not interested
1638 # in distinguishing whether it's stopped or non existing
1639 # (there's function container_exists for that)
1640 # For monitor, however, we still need to know if it has stopped
1641 # recently (i.e. a failure), or not (fresh start)
1642 monitor_cmd_exec
1643 rc=$?
1644 if [ "$rc" -eq 0 ]; then
1645 # Container is running - update state file with current epoch
1646 local current_epoch
1647 current_epoch=$(date +%s)
1648 if ! echo "$current_epoch" > "$CONTAINER_HEARTBEAT_FILE"; then
1649 ocf_log warn "Failed to update container heartbeat file, error code: $?"
1650 # wait for peer to detect any real issue with the etcd cluster or wait for the
1651 # next monitor interval
1652 echo "failed-wait-for-peer"
1653 return
1654 fi
1655 echo "healthy"
1656 return
1657 fi
1658
1659 # Check if state file exists (was container running on last check?)
1660 if [ ! -f "$CONTAINER_HEARTBEAT_FILE" ]; then
1661 # No state file - container was never checked before
1662 ocf_log debug "Container ${CONTAINER} has no previous state"
1663 echo "not-running"
1664 # NOTE: this is where the probe is expected to exit, keeping the logic
1665 # quick and less prone to performance issue under IO load.
1666 return
1667 fi
1668
1669 # Could not execute monitor check command and state file exists - the container failed, check recovery status in this lifecycle
1670 local time_since_heartbeat
1671 time_since_heartbeat=$(get_time_since_last_heartbeat)
1672 ocf_log err "Container ${CONTAINER} failed (last healthy: ${time_since_heartbeat}s ago, error code: $rc)"
1673
1674 # Check if peer has set force_new_cluster for recovery
1675 local fnc_holders
1676 if ! fnc_holders=$(get_force_new_cluster); then
1677 ocf_log err "Could not detect peer-initiated recovery. Checking again in the next monitor cycle"
1678 echo "failed-wait-for-peer"
1679 return
1680 fi
1681
1682 local fnc_holder_count=$(echo "$fnc_holders" | wc -w)
1683 if [ "$fnc_holder_count" -gt 0 ]; then
1684 ocf_log info "force_new_cluster detected (set by: $fnc_holders), triggering restart"
1685 echo "failed-restart-now"
1686 return
1687 fi
1688
1689 echo "failed-wait-for-peer"
1690 }
1691
1692 podman_monitor()
1693 {
1694 local container_health_state
1695
1696 container_health_state=$(container_health_check)
1697 case "$container_health_state" in
1698 healthy)
1699 # Continue with normal monitoring
1700 ;;
1701 not-running)
1702 return $OCF_NOT_RUNNING
1703 ;;
1704 failed-restart-now)
1705 return $OCF_ERR_GENERIC
1706 ;;
1707 failed-wait-for-peer)
1708 # Continue running, waiting for peer recovery
1709 return $OCF_SUCCESS
1710 ;;
1711 *)
1712 ocf_log err "Unknown health state: $container_health_state"
1713 return $OCF_ERR_GENERIC
1714 ;;
1715 esac
1716
1717 # Check if certificate files have changed, if they have, etcd needs to be restarted
1718 if ! etcd_certificates_hash_manager "check"; then
1719 return $OCF_ERR_GENERIC
1720 fi
1721
1722 if is_learner; then
1723 ocf_log info "$NODENAME is learner. Cannot get member id"
1724 return "$OCF_SUCCESS"
1725 fi
1726 # Failing to cache data and check member list should not cause the
1727 # monitor operation to fail.
1728 # TODO: move this inside check_peers where we already query member list json
1729 attribute_node_member_id update
1730 if ! check_peer; then
1731 return $OCF_ERR_GENERIC
1732 fi
1733
1734 # node revision comes from the disk, so if it is not available is a fatal failure
1735 attribute_node_revision update
1736 return $?
1737 }
1738
1739 podman_create_mounts() {
1740 oldIFS="$IFS"
1741 IFS=","
1742 for directory in $OCF_RESKEY_mount_points; do
1743 mkdir -p "$directory"
1744 done
1745 IFS="$oldIFS"
1746 }
1747
1748 podman_container_id()
1749 {
1750 # Retrieve the container ID by doing a "podman ps" rather than
1751 # a "podman inspect", because the latter has performance issues
1752 # under IO load.
1753 # We could have run "podman start $CONTAINER" to get the ID back
1754 # but if the container is stopped, the command will return a
1755 # name instead of a container ID. This would break us.
1756 podman ps --no-trunc --format '{{.ID}} {{.Names}}' | grep -F -w -m1 "$CONTAINER" | cut -d' ' -f1
1757 }
1758
1759
1760 create_transient_drop_in_dependency()
1761 {
1762 local cid=$1
1763 local rc=$OCF_SUCCESS
1764
1765 if [ -z "$cid" ]; then
1766 ocf_exit_reason "Container ID not found for \"$CONTAINER\". Not creating drop-in dependency"
1767 return $OCF_ERR_GENERIC
1768 fi
1769
1770 ocf_log info "Creating drop-in dependency for \"$CONTAINER\" ($cid)"
1771 for scope in "libpod-$cid.scope.d" "libpod-conmon-$cid.scope.d"; do
1772 if [ $rc -eq $OCF_SUCCESS ] && [ ! -d /run/systemd/transient/"$scope" ]; then
1773 mkdir -p /run/systemd/transient/"$scope" && \
1774 printf "[Unit]\nBefore=pacemaker.service" > /run/systemd/transient/"$scope"/dep.conf && \
1775 chmod ago+r /run/systemd/transient/"$scope" /run/systemd/transient/"$scope"/dep.conf
1776 rc=$?
1777 fi
1778 done
1779
1780 if [ $rc -ne $OCF_SUCCESS ]; then
1781 ocf_log err "Could not create drop-in dependency for \"$CONTAINER\" ($cid)"
1782 else
1783 systemctl daemon-reload
1784 rc=$?
1785 if [ $rc -ne $OCF_SUCCESS ]; then
1786 ocf_log err "Could not refresh service definition after creating drop-in for \"$CONTAINER\""
1787 fi
1788 fi
1789
1790 return $rc
1791 }
1792
1793
1794 run_new_container()
1795 {
1796 local opts=$1
1797 local image=$2
1798 local cmd=$3
1799 local rc
1800
1801 ocf_log info "running container $CONTAINER for the first time"
1802 out=$(podman run $opts $image $cmd 2>&1)
1803 rc=$?
1804
1805 if [ -n "$out" ]; then
1806 out="$(echo "$out" | tr -s ' \t\r\n' ' ')"
1807 if [ $rc -eq 0 ]; then
1808 ocf_log info "$out"
1809 else
1810 ocf_log err "$out"
1811 fi
1812 fi
1813
1814 if [ $rc -eq 125 ]; then
1815 # If an internal podman error occurred, it might be because
1816 # the internal storage layer still references an old container
1817 # with the same name, even though podman itself thinks there
1818 # is no such container. If so, purge the storage layer to try
1819 # to clean the corruption and try again.
1820 if echo "$out" | grep -q "unknown.*flag"; then
1821 ocf_exit_reason "$out"
1822 return $rc
1823 fi
1824
1825 ocf_log warn "Internal podman error while creating new container $CONTAINER. Retrying."
1826 ocf_run podman rm --storage "$CONTAINER"
1827 ocf_run podman run $opts $image $cmd
1828 rc=$?
1829 elif [ $rc -eq 127 ]; then
1830 # rhbz#1972209: podman 3.0.x seems to be hit by a race
1831 # where the cgroup is not yet set up properly when the OCI
1832 # runtime configures the container. If that happens, recreate
1833 # the container as long as we get the same error code or
1834 # until start timeout preempts us.
1835 while [ $rc -eq 127 ] && (echo "$out" | grep -q "cgroup.*scope not found") ; do
1836 ocf_log warn "Internal podman error while assigning cgroup. Retrying."
1837 # Arbitrary sleep to prevent consuming all CPU while looping
1838 sleep 1
1839 podman rm -f "$CONTAINER"
1840 out=$(podman run $opts $image $cmd 2>&1)
1841 rc=$?
1842 done
1843 # Log the created container ID if it succeeded
1844 if [ $rc -eq 0 ]; then
1845 ocf_log info "$out"
1846 fi
1847 fi
1848
1849 return $rc
1850 }
1851
1852 compare_revision()
1853 {
1854 # Compare local revision (from disk) against peer revision (from CIB).
1855 # returns "older", "equal" or "newer"
1856 local revision
1857 local peer_node_name
1858 local peer_revision
1859
1860 revision=$(attribute_node_revision get)
1861 peer_revision=$(attribute_node_revision_peer)
1862
1863 if [ "$revision" = "" ] || [ "$revision" = "null" ] || [ "$peer_revision" = "" ] || [ "$peer_revision" = "null" ]; then
1864 ocf_log err "could not compare revisions: '$NODENAME' local revision='$revision', peer revision='$peer_revision'"
1865 return "$OCF_ERR_GENERIC"
1866 fi
1867
1868 if [ "$revision" -gt "$peer_revision" ]; then
1869 ocf_log info "$NODENAME revision: '$revision' is newer than peer revision: '$peer_revision'"
1870 echo "newer"
1871 elif [ "$revision" -eq "$peer_revision" ]; then
1872 ocf_log info "$NODENAME revision: '$revision' is equal to peer revision: '$peer_revision'"
1873 echo "equal"
1874 else
1875 ocf_log info "$NODENAME revision: '$revision' is older than peer revision: '$peer_revision'"
1876 echo "older"
1877 fi
1878 return "$OCF_SUCCESS"
1879 }
1880
1881 ensure_pod_manifest_exists()
1882 {
1883 local wait_timeout_sec=$((10 * 60))
1884 local poll_interval_sec=5
1885 local poll_retries=$((wait_timeout_sec/poll_interval_sec))
1886
1887 for try in $(seq "$poll_retries"); do
1888 if [ -f "$OCF_RESKEY_pod_manifest" ]; then
1889 ocf_log info "pod manifest ($OCF_RESKEY_pod_manifest) found"
1890 break
1891 fi
1892 ocf_log debug "pod manifest ($OCF_RESKEY_pod_manifest) does not exist yet: retry in $poll_interval_sec seconds."
1893 sleep "$poll_interval_sec"
1894 done
1895
1896 if [ ! -f "$OCF_RESKEY_pod_manifest" ]; then
1897 ocf_log err "pod manifest ($OCF_RESKEY_pod_manifest) still missing after $wait_timeout_sec seconds."
1898 return "$OCF_ERR_CONFIGURED"
1899 fi
1900
1901 return "$OCF_SUCCESS"
1902 }
1903
1904 filter_pod_manifest() {
1905 # Remove pod-version related fields from POD manifest
1906 local pod_manifest="$1"
1907 local temporary_file
1908 local jq_filter='del(.metadata.labels.revision) | .spec.containers[] |= ( .env |= map(select( .name != "ETCD_STATIC_POD_VERSION" ))) | .spec.volumes |= map( select( .name != "resource-dir" ))'
1909
1910 if ! temporary_file=$(mktemp); then
1911 ocf_log err "could not create temporary file for '$pod_manifest', error code: $?"
1912 return $OCF_ERR_GENERIC
1913 fi
1914 if ! jq "$jq_filter" "$pod_manifest" > "$temporary_file"; then
1915 ocf_log err "could not remove pod version related data from '$pod_manifest', error code: $?"
1916 return $OCF_ERR_GENERIC
1917 fi
1918 echo "$temporary_file"
1919 }
1920
1921 can_reuse_container() {
1922 # Decide whether to reuse the existing container or create a new one based on etcd pod manifest changes.
1923 # NOTE: explicitly ignore POD version and POD version related data, as the content might be the same even if the revision number has changed.
1924 local cp_rc
1925 local diff_rc
1926 local filtered_original_pod_manifest
1927 local filtered_copy_pod_manifest
1928
1929
1930 # If the container does not exist it cannot be reused
1931 if ! container_exists; then
1932 OCF_RESKEY_reuse=0
1933 return "$OCF_SUCCESS"
1934 fi
1935
1936 # If the manifest copy doesn't exist, we need a new container.
1937 if [ ! -f "$POD_MANIFEST_COPY" ]; then
1938 ocf_log info "a working copy of $OCF_RESKEY_pod_manifest was not found. A new etcd container will be created."
1939 OCF_RESKEY_reuse=0
1940 return "$OCF_SUCCESS"
1941 fi
1942 if ! filtered_original_pod_manifest=$(filter_pod_manifest "$OCF_RESKEY_pod_manifest"); then
1943 return $OCF_ERR_GENERIC
1944 fi
1945 if ! filtered_copy_pod_manifest=$(filter_pod_manifest "$POD_MANIFEST_COPY"); then
1946 return $OCF_ERR_GENERIC
1947 fi
1948
1949 ocf_log info "comparing $OCF_RESKEY_pod_manifest with local copy $POD_MANIFEST_COPY"
1950 ocf_run diff -s "$filtered_original_pod_manifest" "$filtered_copy_pod_manifest"
1951 diff_rc="$?"
1952 # clean up temporary files
1953 rm -f "$filtered_original_pod_manifest" "$filtered_copy_pod_manifest"
1954 case "$diff_rc" in
1955 0)
1956 ocf_log info "Reusing the existing etcd container"
1957 OCF_RESKEY_reuse=1
1958 ;;
1959 1)
1960 ocf_log info "Etcd pod manifest changes detected: creating a new etcd container to apply the changes"
1961 if ! ocf_run cp -p "$OCF_RESKEY_pod_manifest" "$POD_MANIFEST_COPY"; then
1962 cp_rc="$?"
1963 ocf_log err "Could not create a working copy of $OCF_RESKEY_pod_manifest, rc: $cp_rc"
1964 return "$OCF_ERR_GENERIC"
1965 fi
1966 ocf_log info "A working copy of $OCF_RESKEY_pod_manifest was created"
1967 OCF_RESKEY_reuse=0
1968 ;;
1969 *)
1970 ocf_log err "Could not check if etcd pod manifest has changed, diff rc: $diff_rc"
1971 return "$OCF_ERR_GENERIC"
1972 ;;
1973 esac
1974
1975 return "$OCF_SUCCESS"
1976 }
1977
1978 ensure_pod_manifest_copy_exists() {
1979 local cp_rc
1980
1981 if [ -f "$POD_MANIFEST_COPY" ]; then
1982 return "$OCF_SUCCESS"
1983 fi
1984
1985 # If the manifest copy doesn't exist, create it and ensure a new container.
1986 if ! ocf_run cp -p "$OCF_RESKEY_pod_manifest" "$POD_MANIFEST_COPY"; then
1987 cp_rc="$?"
1988 ocf_log err "Could not create a working copy of $OCF_RESKEY_pod_manifest, rc: $cp_rc"
1989 return "$OCF_ERR_GENERIC"
1990 fi
1991
1992 ocf_log info "a new working copy of $OCF_RESKEY_pod_manifest was created"
1993
1994 return "$OCF_SUCCESS"
1995 }
1996
1997 podman_start()
1998 {
1999 local cid
2000 local rc
2001 local etcd_pod_wait_timeout_sec=$((10 * 60))
2002 local etcd_pod_poll_interval_sec=10
2003 local etcd_pod_poll_retries=$((etcd_pod_wait_timeout_sec/etcd_pod_poll_interval_sec))
2004 local pod_was_running=false
2005
2006 ocf_log notice "podman-etcd start"
2007
2008 # Clear container health check state file
2009 if ! rm -f "$CONTAINER_HEARTBEAT_FILE"; then
2010 ocf_log err "could not delete container health check state file"
2011 fi
2012
2013 attribute_node_ip update
2014 attribute_node_cluster_id update
2015 attribute_node_revision update
2016
2017 # ensure the etcd pod is not running before starting the container
2018 ocf_log info "ensure etcd pod is not running (retries: $etcd_pod_poll_retries, interval: $etcd_pod_poll_interval_sec)"
2019 for try in $(seq $etcd_pod_poll_retries); do
2020 if ! etcd_pod_container_exists; then
2021 break
2022 fi
2023 ocf_log info "etcd pod running: retry in $etcd_pod_poll_interval_sec seconds."
2024 pod_was_running=true
2025 sleep $etcd_pod_poll_interval_sec
2026 done
2027 if etcd_pod_container_exists; then
2028 ocf_exit_reason "etcd pod is still running after $etcd_pod_wait_timeout_sec seconds."
2029 return $OCF_ERR_GENERIC
2030 fi
2031
2032 # Update the certificate hash after the container has started successfully
2033 # this is to ensure that the certificate hash is updated after a restart is initiated
2034 # by a cert rotation event from the monitor command.
2035 if ! etcd_certificates_hash_manager "update"; then
2036 ocf_exit_reason "etcd certificate hash manager failed to update the certificate hash"
2037 return $OCF_ERR_GENERIC
2038 fi
2039
2040 # check if the container has already started
2041 podman_simple_status
2042 if [ $? -eq $OCF_SUCCESS ]; then
2043 ocf_log info "the '$CONTAINER' has already started. Nothing to do"
2044 return "$OCF_SUCCESS"
2045 fi
2046
2047 if ! ensure_pod_manifest_exists; then
2048 ocf_exit_reason "could not find etcd pod manifest ($OCF_RESKEY_pod_manifest)"
2049 return "$OCF_ERR_GENERIC"
2050 fi
2051
2052 if ocf_is_true "$pod_was_running"; then
2053 ocf_log info "static pod was running: start normally"
2054 else
2055 local fnc_holders
2056 if ! fnc_holders=$(get_force_new_cluster); then
2057 ocf_exit_reason "Failed to get force_new_cluster node holders"
2058 return "$OCF_ERR_GENERIC"
2059 fi
2060
2061 local fnc_holder_count
2062 fnc_holder_count=$(echo "$fnc_holders" | wc -w)
2063 if [ "$fnc_holder_count" -gt 1 ]; then
2064 ocf_exit_reason "force_new_cluster attribute is set on multiple nodes ($fnc_holders)"
2065 return "$OCF_ERR_CONFIGURED"
2066 fi
2067
2068 if [ "$fnc_holder_count" -eq 1 ]; then
2069 if echo "$fnc_holders" | grep -q -w "$NODENAME"; then
2070 # Attribute is set on the local node.
2071 ocf_log notice "$NODENAME marked to force-new-cluster"
2072 JOIN_AS_LEARNER=false
2073 else
2074 # Attribute is set on a peer node.
2075 ocf_log info "$NODENAME shall join as learner because force_new_cluster is set on peer $fnc_holders"
2076 JOIN_AS_LEARNER=true
2077 fi
2078 else
2079 ocf_log info "no node is marked to force-new-cluster"
2080 # When the local agent starts, we can infer the cluster state by counting
2081 # how many agents are starting or already active:
2082 # - 1 active agent: it's the peer (we are just starting)
2083 # - 0 active agents, 1 starting: we are starting; the peer is not starting
2084 # - 0 active agents, 2 starting: both agents are starting simultaneously
2085 local active_resources_count
2086 active_resources_count=$(get_truly_active_resources_count)
2087 ocf_log info "found '$active_resources_count' active etcd resources (active: '$OCF_RESKEY_CRM_meta_notify_active_resource', stop: '$OCF_RESKEY_CRM_meta_notify_stop_resource')"
2088 case "$active_resources_count" in
2089 1)
2090 # is_standalone may return true here due to a restart: in the previous run,
2091 # this agent was the sole voter and the peer had not yet joined the member
2092 # list (learner_node unset). Since standalone_node was not cleared before
2093 # the restart, start normally to recover the previous cluster state.
2094 if is_standalone; then
2095 ocf_log info "peer active but not a voter: start normally to recover"
2096 elif [ "$(attribute_learner_node get)" = "$(get_peer_node_name)" ]; then
2097 ocf_log info "peer active but in learner mode: start normally to recover"
2098 else
2099 # If (A) we must join the peer's new cluster
2100 ocf_log info "peer is active standalone: joining as learner"
2101 JOIN_AS_LEARNER=true
2102 fi
2103 ;;
2104 0)
2105 # No active resources: clear any stale learner_node attribute from previous failed session
2106 ocf_log debug "clearing stale learner_node attribute (safe when active_resources_count=0)"
2107 attribute_learner_node clear
2108 # count how many agents are starting now
2109 local start_resources_count
2110 start_resources_count=$(echo "$OCF_RESKEY_CRM_meta_notify_start_resource" | wc -w)
2111 ocf_log info "found '$start_resources_count' starting etcd resources (meta notify environment variable: '$OCF_RESKEY_CRM_meta_notify_start_resource')"
2112
2113 # we need to compare the revisions in any of the following branches
2114 # so call the function only once here
2115 if ! revision_compare_result=$(compare_revision); then
2116 ocf_log err "could not compare revisions, error code: $?"
2117 return "$OCF_ERR_GENERIC"
2118 fi
2119 case "$start_resources_count" in
2120 1)
2121 ocf_log debug "peer not starting: ensure we can start a new cluster"
2122 if [ "$revision_compare_result" != "older" ]; then
2123 # If our revision is the same as or newer than the peer's last saved
2124 # revision, and the peer agent isn't currently starting, we can
2125 # restore e-quorum by forcing a new cluster.
2126 set_force_new_cluster
2127 else
2128 ocf_log err "local revision is older and peer is not starting: cannot start"
2129 ocf_exit_reason "local revision is older and peer is not starting: cannot start"
2130 return "$OCF_ERR_GENERIC"
2131 fi
2132 ;;
2133 2)
2134 # TODO: can we start "normally", regardless the revisions, if the container-id is the same on both nodes?
2135 ocf_log info "peer starting"
2136 if [ "$revision_compare_result" = "newer" ]; then
2137 set_force_new_cluster
2138 elif [ "$revision_compare_result" = "older" ]; then
2139 ocf_log info "$NODENAME shall join as learner"
2140 JOIN_AS_LEARNER=true
2141 else
2142 if [ "$(attribute_node_cluster_id get)" = "$(attribute_node_cluster_id_peer)" ]; then
2143 ocf_log info "same cluster_id and revision: start normal"
2144 else
2145 ocf_exit_reason "same revision but different cluster id"
2146 return "$OCF_ERR_CONFIGURED"
2147 fi
2148 fi
2149 ;;
2150 *)
2151 ocf_log err "Unexpected start resource count: $start_resources_count"
2152 podman_notify
2153 return "$OCF_ERR_GENERIC"
2154 ;;
2155 esac
2156 ;;
2157 *)
2158 ocf_log err "Unexpected active resource count: $active_resources_count"
2159 podman_notify
2160 return "$OCF_ERR_GENERIC"
2161 ;;
2162 esac
2163 fi
2164 fi
2165
2166 podman_create_mounts
2167 local run_opts="--detach --name=${CONTAINER} --replace"
2168
2169 run_opts="$run_opts --oom-score-adj=${OCF_RESKEY_oom}"
2170
2171 if ocf_is_true "$JOIN_AS_LEARNER"; then
2172 local wait_timeout_sec=$((2*60))
2173 local poll_interval_sec=5
2174 local retries=$(( wait_timeout_sec / poll_interval_sec ))
2175
2176 ocf_log info "ensure the leader node added $NODENAME as learner member before continuing (timeout: $wait_timeout_sec seconds)"
2177 for try in $(seq $retries); do
2178 learner_node=$(attribute_learner_node get)
2179 if [ "$NODENAME" != "$learner_node" ]; then
2180 ocf_log info "$NODENAME is not in the member list yet. Retry in $poll_interval_sec seconds."
2181 sleep $poll_interval_sec
2182 continue
2183 fi
2184 ocf_log info "learner node $learner_node in the member list"
2185 break
2186 done
2187 if [ "$NODENAME" != "$(attribute_learner_node get)" ]; then
2188 ocf_log err "wait for $NODENAME to be in the member list timed out"
2189 return "$OCF_ERR_GENERIC"
2190 fi
2191
2192 archive_data_folder
2193 if ! wipe_data_folder_for_learner; then
2194 return "$OCF_ERR_GENERIC"
2195 fi
2196 fi
2197
2198 ocf_log info "check for changes in pod manifest to decide if the container should be reused or replaced"
2199 if ! can_reuse_container ; then
2200 ocf_log err "could not determine etcd container reuse strategy"
2201 return $OCF_ERR_GENERIC
2202 fi
2203
2204 # Archive current container and its configuration before creating
2205 # new configuration files.
2206 if ! ocf_is_true "$OCF_RESKEY_reuse"; then
2207 # Log archive container failures but don't block, as the priority
2208 # is ensuring the etcd container starts successfully.
2209 archive_current_container
2210 fi
2211
2212 if ! ensure_pod_manifest_copy_exists; then
2213 return $OCF_ERR_GENERIC
2214 fi
2215
2216 if ! prepare_env; then
2217 ocf_log err "Could not prepare environment for podman"
2218 return $OCF_ERR_GENERIC
2219 fi
2220
2221 if ! generate_etcd_configuration; then
2222 ocf_log err "Could not generate etcd configuration"
2223 return $OCF_ERR_CONFIGURED
2224 fi
2225
2226 run_opts="$run_opts \
2227 --network=host \
2228 -v /etc/kubernetes/static-pod-resources/etcd-certs:/etc/kubernetes/static-pod-certs \
2229 -v /var/lib/etcd:/var/lib/etcd \
2230 --env ETCDCTL_CACERT=$SERVER_CACERT \
2231 --env ETCDCTL_CERT=$ETCD_PEER_CERT \
2232 --env ETCDCTL_KEY=$ETCD_PEER_KEY \
2233 --authfile=$OCF_RESKEY_authfile \
2234 --security-opt label=disable"
2235 if [ -n "$OCF_RESKEY_run_opts" ]; then
2236 run_opts="$run_opts $OCF_RESKEY_run_opts"
2237 fi
2238
2239 if [ -f "$ETCD_CONFIGURATION_FILE" ]; then
2240 ocf_log info "using etcd configuration file: $ETCD_CONFIGURATION_FILE"
2241 else
2242 ocf_log err "could not find $ETCD_CONFIGURATION_FILE"
2243 return "$OCF_ERR_GENERIC"
2244 fi
2245
2246 OCF_RESKEY_run_cmd="$OCF_RESKEY_run_cmd --config-file=$ETCD_CONFIGURATION_FILE"
2247 if [ -n "$OCF_RESKEY_run_cmd_opts" ]; then
2248 OCF_RESKEY_run_cmd="$OCF_RESKEY_run_cmd $OCF_RESKEY_run_cmd_opts"
2249 fi
2250
2251 if [ "$OCF_RESKEY_image" = "$OCF_RESKEY_image_default" ]; then
2252 # no container image provided via input parameters. Read it from the pod manifest.
2253 OCF_RESKEY_image=$(jq -r '.spec.containers[] | select( .name=="etcd").image' "$OCF_RESKEY_pod_manifest")
2254 ocf_log info "using container image ($OCF_RESKEY_image) from Pod manifest ($OCF_RESKEY_pod_manifest)"
2255 else
2256 # use the container image provided as input parameter
2257 ocf_log info "using container image ($OCF_RESKEY_image) via input parameters"
2258 fi
2259
2260 if [ $REQUIRE_IMAGE_PULL -eq 1 ]; then
2261 ocf_log notice "Beginning pull of image, ${OCF_RESKEY_image}"
2262 if ! podman pull --authfile="$OCF_RESKEY_authfile" "${OCF_RESKEY_image}"; then
2263 ocf_exit_reason "failed to pull image ${OCF_RESKEY_image}"
2264 return $OCF_ERR_GENERIC
2265 fi
2266 else
2267 ocf_log notice "Pull image not required, ${OCF_RESKEY_image}"
2268 fi
2269
2270 if ocf_is_true "$OCF_RESKEY_reuse" && container_exists; then
2271 ocf_log info "starting existing container $CONTAINER."
2272 ocf_run podman start "$CONTAINER"
2273 else
2274 ocf_log info "starting new container $CONTAINER."
2275 run_new_container "$run_opts" "$OCF_RESKEY_image" "$OCF_RESKEY_run_cmd"
2276 if [ $? -eq 125 ]; then
2277 return $OCF_ERR_GENERIC
2278 fi
2279 fi
2280 rc=$?
2281
2282 # if the container was stopped or didn't exist before, systemd
2283 # removed the libpod* scopes. So always try to recreate the drop-ins
2284 if [ $rc -eq 0 ] && ocf_is_true "$OCF_RESKEY_drop_in_dependency"; then
2285 cid=$(podman_container_id)
2286 create_transient_drop_in_dependency "$cid"
2287 rc=$?
2288 fi
2289
2290 if [ $rc -ne 0 ]; then
2291 ocf_exit_reason "podman failed to launch container (error code: $rc)"
2292 return $OCF_ERR_GENERIC
2293 fi
2294
2295 # wait for monitor to pass before declaring that the container is started
2296 while true; do
2297 podman_simple_status
2298 if [ $? -ne $OCF_SUCCESS ]; then
2299 ocf_exit_reason "Newly created podman container exited after start"
2300 ocf_run podman logs --tail 20 "${CONTAINER}"
2301 return $OCF_ERR_GENERIC
2302 fi
2303
2304 monitor_cmd_exec
2305 if [ $? -eq $OCF_SUCCESS ]; then
2306 ocf_log notice "Container $CONTAINER started successfully"
2307 if is_force_new_cluster; then
2308 clear_force_new_cluster
2309
2310 local peer_node_name
2311 local peer_node_ip
2312 peer_node_name="$(get_peer_node_name)"
2313 peer_node_ip="$(attribute_node_ip_peer)"
2314 if [ -n "$peer_node_name" ] && [ -n "$peer_node_ip" ]; then
2315 add_member_as_learner "$peer_node_name" "$peer_node_ip"
2316 set_standalone_node
2317 else
2318 ocf_log err "could not add peer as learner (peer node name: ${peer_node_name:-unknown}, peer ip: ${peer_node_ip:-unknown})"
2319 fi
2320 fi
2321 return $OCF_SUCCESS
2322 fi
2323
2324 ocf_exit_reason "waiting on monitor_cmd to pass after start"
2325 sleep 1
2326 done
2327 }
2328
2329 # leave_etcd_member_list removes the current node from the etcd member list during
2330 # shutdown to ensure clean cluster state.
2331 #
2332 # Skips removal if this is the standalone (last) node. When both nodes are stopping
2333 # concurrently, delays the second node to prevent simultaneous member removal that
2334 # could corrupt the etcd WAL.
2335 leave_etcd_member_list()
2336 {
2337 if ! member_id=$(attribute_node_member_id get); then
2338 ocf_log err "error leaving members list: could not get member-id"
2339 return
2340 fi
2341
2342 if is_standalone; then
2343 ocf_log info "last member. Not leaving the member list"
2344 return
2345 fi
2346
2347 local stopping_resources_count
2348 stopping_resources_count=$(echo "$OCF_RESKEY_CRM_meta_notify_stop_resource" | wc -w)
2349 ocf_log info "found '$stopping_resources_count' stopping etcd resources (stop: '$OCF_RESKEY_CRM_meta_notify_stop_resource')"
2350 if [ "$stopping_resources_count" -gt 1 ]; then
2351 # Prevent WAL corruption by delaying the alphabetically second node's member
2352 # removal when both nodes are stopping concurrently.
2353 local delayed_node
2354
2355 node_names_sorted=$(echo "$OCF_RESKEY_node_ip_map" | sed 's/:[^;]*//g; s/;/ /g' | tr ' ' '\n' | sort | tr '\n' ' ')
2356 delayed_node="$(echo "$node_names_sorted" | cut -d' ' -f2)"
2357
2358 if [ -z "$delayed_node" ]; then
2359 ocf_log warn "could not determine node to be delayed: not leaving the member list"
2360 return
2361 fi
2362
2363 if [ "$NODENAME" = "$delayed_node" ]; then
2364 ocf_log info "delaying stop for ${DELAY_SECOND_NODE_LEAVE_SEC}s to prevent simultaneous etcd member removal"
2365 sleep $DELAY_SECOND_NODE_LEAVE_SEC
2366 fi
2367 fi
2368
2369 # Ensure we're not the last active resource before leaving. The `standalone_node` property
2370 # may not be set if stop was called before monitor check, or after the delayed node waited.
2371 local active_resources_count
2372 active_resources_count=$(get_truly_active_resources_count)
2373 if [ "$active_resources_count" -lt 1 ]; then
2374 ocf_log info "last member. Not leaving the member list"
2375 return
2376 fi
2377
2378 ocf_log info "leaving members list as member with ID $member_id"
2379 local endpoint
2380 endpoint="$(ip_url $(attribute_node_ip get)):2379"
2381 ocf_run timeout 30 podman exec "$CONTAINER" etcdctl member remove "$member_id" --endpoints="$endpoint"
2382 rc=$?
2383 if [ $rc -ne 0 ]; then
2384 ocf_log err "error leaving members list, error code: $rc"
2385 fi
2386 }
2387
2388 podman_stop()
2389 {
2390 local timeout=60
2391 local rc
2392
2393 ocf_log notice "podman-etcd stop"
2394
2395 # Clear container health check state file
2396 if ! rm -f "$CONTAINER_HEARTBEAT_FILE"; then
2397 ocf_log err "could not delete container health check state file"
2398 fi
2399
2400 attribute_learner_node clear
2401 attribute_node_revision update
2402 attribute_node_cluster_id update
2403
2404 # Use podman inspect instead of podman exec (podman_simple_status) to check
2405 # container state. podman exec enters the container's process namespace and
2406 # hangs when etcd is unresponsive — the typical scenario that triggers a stop.
2407 local container_running
2408 container_running=$(podman inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null)
2409 if [ "$container_running" != "true" ]; then
2410 ocf_log info "could not leave members list: $CONTAINER container not running, running state: ${container_running}"
2411 attribute_node_member_id clear
2412 return $OCF_SUCCESS
2413 fi
2414
2415 leave_etcd_member_list
2416
2417 # clear node_member_id CIB attribute only after leaving the member list
2418 attribute_node_member_id clear
2419
2420 if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
2421 timeout=$(((OCF_RESKEY_CRM_meta_timeout/1000) -10 ))
2422 if [ $timeout -lt 10 ]; then
2423 timeout=10
2424 fi
2425 fi
2426
2427 if ocf_is_true "$OCF_RESKEY_force_kill"; then
2428 ocf_run podman kill "$CONTAINER"
2429 rc=$?
2430 else
2431 ocf_log info "waiting $timeout second[s] before killing container"
2432 ocf_run podman stop -t="$timeout" "$CONTAINER"
2433 rc=$?
2434 # on stop, systemd will automatically delete any transient
2435 # drop-in conf that has been created earlier
2436 fi
2437
2438 if [ $rc -ne 0 ]; then
2439 # If the stop failed, it could be because the controlling conmon
2440 # process died unexpectedly. If so, a generic error code is returned
2441 # but the associated container exit code is -1. If that's the case,
2442 # assume there's no failure and continue with the rm as usual.
2443 if [ $rc -eq 125 ] && \
2444 podman inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$CONTAINER" | grep -Eq '^(exited|stopped):-1$'; then
2445 ocf_log err "Container ${CONTAINER} had an unexpected stop outcome. Trying to remove it anyway."
2446 else
2447 ocf_exit_reason "Failed to stop container, ${CONTAINER}, based on image, ${OCF_RESKEY_image}."
2448 return $OCF_ERR_GENERIC
2449 fi
2450 fi
2451
2452 return $OCF_SUCCESS
2453 }
2454
2455 image_exists()
2456 {
2457 if [ "$OCF_RESKEY_image" = "$OCF_RESKEY_image_default" ]; then
2458 # the actual container image was not defined yet. Nor by
2459 # the user via OCF_RESKEY, nor by reading the Pod manifest
2460 return 0
2461 fi
2462 if podman image exists "${OCF_RESKEY_image}"; then
2463 # image found
2464 return 0
2465 fi
2466
2467 if ocf_is_true "$OCF_RESKEY_allow_pull"; then
2468 REQUIRE_IMAGE_PULL=1
2469 ocf_log notice "Image (${OCF_RESKEY_image}) does not exist locally but will be pulled during start"
2470 return 0
2471 fi
2472 # image not found.
2473 return 1
2474 }
2475
2476 podman_validate()
2477 {
2478 check_binary curl
2479 check_binary crictl
2480 check_binary oc
2481 check_binary podman
2482 check_binary jq
2483 check_binary tar
2484
2485 if [ -z "$OCF_RESKEY_node_ip_map" ]; then
2486 ocf_exit_reason "'node_ip_map' option is required"
2487 exit $OCF_ERR_CONFIGURED
2488 fi
2489
2490 if [ -z "$OCF_RESKEY_pod_manifest" ]; then
2491 ocf_exit_reason "'pod_manifest' option is required"
2492 exit $OCF_ERR_CONFIGURED
2493 fi
2494
2495 if [ -z "$OCF_RESKEY_image" ]; then
2496 ocf_exit_reason "'image' option is required"
2497 exit $OCF_ERR_CONFIGURED
2498 fi
2499
2500 if ! image_exists; then
2501 ocf_exit_reason "base image, ${OCF_RESKEY_image}, could not be found."
2502 exit $OCF_ERR_CONFIGURED
2503 fi
2504
2505 if [ "$OCF_RESKEY_oom" -lt -1000 ] || [ "$OCF_RESKEY_oom" -gt 1000 ]; then
2506 ocf_exit_reason "'oom' value ${OCF_RESKEY_oom} is out of range [-1000:1000]"
2507 exit $OCF_ERR_CONFIGURED
2508 fi
2509
2510 if ! echo "validation test" > "$ETCD_CERTS_HASH_FILE" \
2511 || ! cat "$ETCD_CERTS_HASH_FILE" >/dev/null 2>&1 \
2512 || ! rm "$ETCD_CERTS_HASH_FILE"; then
2513 ocf_exit_reason "cannot read/write to certificate hash file $ETCD_CERTS_HASH_FILE"
2514 exit $OCF_ERR_GENERIC
2515 fi
2516
2517 return $OCF_SUCCESS
2518 }
2519
2520 podman_notify()
2521 {
2522 ocf_log info "notify: type=${OCF_RESKEY_CRM_meta_notify_type}, operation=${OCF_RESKEY_CRM_meta_notify_operation}, nodes { active=[${OCF_RESKEY_CRM_meta_notify_active_uname}], start=[${OCF_RESKEY_CRM_meta_notify_start_uname}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_uname}] }, resources { active=[${OCF_RESKEY_CRM_meta_notify_active_resource}], start =[${OCF_RESKEY_CRM_meta_notify_start_resource}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_resource}] }"
2523 }
2524
2525 # TODO :
2526 # When a user starts plural clones in a node in globally-unique, a user cannot appoint plural name parameters.
2527 # When a user appoints reuse, the resource agent cannot connect plural clones with a container.
2528
2529 if ocf_is_true "$OCF_RESKEY_CRM_meta_globally_unique"; then
2530 if [ -n "$OCF_RESKEY_name" ]; then
2531 if [ -n "$OCF_RESKEY_CRM_meta_clone_node_max" ] && [ "$OCF_RESKEY_CRM_meta_clone_node_max" -ne 1 ]
2532 then
2533 ocf_exit_reason "Cannot make plural clones from the same name parameter."
2534 exit $OCF_ERR_CONFIGURED
2535 fi
2536 if [ -n "$OCF_RESKEY_CRM_meta_master_node_max" ] && [ "$OCF_RESKEY_CRM_meta_master_node_max" -ne 1 ]
2537 then
2538 ocf_exit_reason "Cannot make plural master from the same name parameter."
2539 exit $OCF_ERR_CONFIGURED
2540 fi
2541 fi
2542 : ${OCF_RESKEY_name=$(echo ${OCF_RESOURCE_INSTANCE} | tr ':' '-')}
2543 else
2544 : ${OCF_RESKEY_name=${OCF_RESOURCE_INSTANCE}}
2545 fi
2546
2547 CONTAINER=$OCF_RESKEY_name
2548 POD_MANIFEST_COPY="${OCF_RESKEY_config_location}/pod.yaml"
2549 ETCD_CONFIGURATION_FILE="${OCF_RESKEY_config_location}/config.yaml"
2550 ETCD_BACKUP_FILE="${OCF_RESKEY_backup_location}/config-previous.tar.gz"
2551 ETCD_MEMBER_DIR="/var/lib/etcd/member"
2552 ETCD_REVISION_JSON="/var/lib/etcd/revision.json"
2553 ETCD_REVISION_BUMP_PERCENTAGE=0.2
2554 ETCD_BUMP_REV_DEFAULT=1000000000
2555 ETCD_CERTS_HASH_FILE="${OCF_RESKEY_config_location}/certs.hash"
2556 # State file location: Uses HA_RSCTMP to ensure automatic cleanup on reboot.
2557 # This is intentional - reboots are controlled stops, not failures requiring detection.
2558 CONTAINER_HEARTBEAT_FILE=${HA_RSCTMP}/podman-container-last-running
2559 DELAY_SECOND_NODE_LEAVE_SEC=10
2560 # Shorter etcdctl command-timeout for monitor-path calls to prevent
2561 # consuming the 25s monitor budget. Non-monitor callers use the default 5s.
2562 MONITOR_ETCDCTL_TIMEOUT="3s"
2563
2564 # Note: we currently monitor podman containers by with the "podman exec"
2565 # command, so make sure that invocation is always valid by enforcing the
2566 # exec command to be non-empty
2567 : ${OCF_RESKEY_monitor_cmd:=/bin/true}
2568
2569 # When OCF_RESKEY_drop_in_dependency is not populated, we
2570 # look at another file-based way of enabling the option.
2571 # Otherwise, consider it disabled.
2572 if [ -z "$OCF_RESKEY_drop_in_dependency" ]; then
2573 if [ -f "/etc/sysconfig/podman_drop_in" ] || \
2574 [ -f "/etc/default/podman_drop_in" ]; then
2575 OCF_RESKEY_drop_in_dependency=yes
2576 fi
2577 fi
2578
2579
2580 case $__OCF_ACTION in
2581 meta-data) meta_data
2582 exit $OCF_SUCCESS;;
2583 usage|help) podman_usage
2584 exit $OCF_SUCCESS
2585 ;;
2586 esac
2587
2588 NODENAME=$(ocf_local_nodename)
2589 JOIN_AS_LEARNER=false
2590
2591 case $__OCF_ACTION in
2592 start)
2593 podman_validate || exit $?
2594 podman_start;;
2595 stop) podman_stop;;
2596 monitor) podman_monitor;;
2597 notify) podman_notify;;
2598 validate-all) podman_validate;;
2599 *) podman_usage
2600 exit $OCF_ERR_UNIMPLEMENTED
2601 ;;
2602 esac
2603 rc=$?
2604 ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
2605 exit $rc