| 1 |
#!/bin/sh |
| 2 |
# |
| 3 |
# The podman etcd HA resource agent creates and launches a etcd podman |
| 4 |
# container based off a supplied podman image. Containers managed by |
| 5 |
# this agent are both created and removed upon the agent's start and |
| 6 |
# stop actions. |
| 7 |
# |
| 8 |
# Based on the podman resource agent. |
| 9 |
# |
| 10 |
# Copyright (c) 2014 David Vossel <[email protected]> |
| 11 |
# Michele Baldessari <[email protected]> |
| 12 |
# All Rights Reserved. |
| 13 |
# |
| 14 |
# This program is free software; you can redistribute it and/or modify |
| 15 |
# it under the terms of version 2 of the GNU General Public License as |
| 16 |
# published by the Free Software Foundation. |
| 17 |
# |
| 18 |
# This program is distributed in the hope that it would be useful, but |
| 19 |
# WITHOUT ANY WARRANTY; without even the implied warranty of |
| 20 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| 21 |
# |
| 22 |
# Further, this software is distributed without any warranty that it is |
| 23 |
# free of the rightful claim of any third person regarding infringement |
| 24 |
# or the like. Any license provided herein, whether implied or |
| 25 |
# otherwise, applies only to this software file. Patent licenses, if |
| 26 |
# any, provided herein do not apply to combinations of this program with |
| 27 |
# other software, or any other product whatsoever. |
| 28 |
# |
| 29 |
# You should have received a copy of the GNU General Public License |
| 30 |
# along with this program; if not, write the Free Software Foundation, |
| 31 |
# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. |
| 32 |
# |
| 33 |
|
| 34 |
####################################################################### |
| 35 |
# Initialization: |
| 36 |
|
| 37 |
: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} |
| 38 |
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs |
| 39 |
|
| 40 |
# Parameter defaults |
| 41 |
OCF_RESKEY_image_default="default" |
| 42 |
OCF_RESKEY_pod_manifest_default="/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/external-etcd-pod/pod.yaml" |
| 43 |
OCF_RESKEY_etcd_certs_dir_default="/etc/kubernetes/static-pod-resources/etcd-certs" |
| 44 |
OCF_RESKEY_name_default="etcd" |
| 45 |
OCF_RESKEY_nic_default="br-ex" |
| 46 |
OCF_RESKEY_authfile_default="/var/lib/kubelet/config.json" |
| 47 |
OCF_RESKEY_allow_pull_default="1" |
| 48 |
OCF_RESKEY_reuse_default="0" |
| 49 |
OCF_RESKEY_oom_default="-997" |
| 50 |
OCF_RESKEY_config_location_default="/var/lib/etcd" |
| 51 |
OCF_RESKEY_backup_location_default="/var/lib/etcd" |
| 52 |
OCF_RESKEY_max_backup_snapshots_default="3" |
| 53 |
|
| 54 |
: ${OCF_RESKEY_image=${OCF_RESKEY_image_default}} |
| 55 |
: ${OCF_RESKEY_pod_manifest=${OCF_RESKEY_pod_manifest_default}} |
| 56 |
: ${OCF_RESKEY_etcd_certs_dir=${OCF_RESKEY_etcd_certs_dir_default}} |
| 57 |
: ${OCF_RESKEY_name=${OCF_RESKEY_name_default}} |
| 58 |
: ${OCF_RESKEY_nic=${OCF_RESKEY_nic_default}} |
| 59 |
: ${OCF_RESKEY_authfile=${OCF_RESKEY_authfile_default}} |
| 60 |
: ${OCF_RESKEY_allow_pull=${OCF_RESKEY_allow_pull_default}} |
| 61 |
: ${OCF_RESKEY_reuse=${OCF_RESKEY_reuse_default}} |
| 62 |
: ${OCF_RESKEY_oom=${OCF_RESKEY_oom_default}} |
| 63 |
: ${OCF_RESKEY_config_location=${OCF_RESKEY_config_location_default}} |
| 64 |
: ${OCF_RESKEY_backup_location=${OCF_RESKEY_backup_location_default}} |
| 65 |
: ${OCF_RESKEY_max_backup_snapshots=${OCF_RESKEY_max_backup_snapshots_default}} |
| 66 |
|
| 67 |
|
| 68 |
####################################################################### |
| 69 |
|
| 70 |
meta_data() |
| 71 |
{ |
| 72 |
cat <<END |
| 73 |
<?xml version="1.0"?> |
| 74 |
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> |
| 75 |
<resource-agent name="podman-etcd" version="1.0"> |
| 76 |
<version>1.0</version> |
| 77 |
|
| 78 |
<longdesc lang="en"> |
| 79 |
The podman-etcd HA resource agent creates and launches a etcd podman |
| 80 |
container based off a supplied podman image. Containers managed by |
| 81 |
this agent are both created and removed upon the agent's start and |
| 82 |
stop actions. |
| 83 |
</longdesc> |
| 84 |
<shortdesc lang="en">Podman etcd container resource agent.</shortdesc> |
| 85 |
|
| 86 |
<parameters> |
| 87 |
<parameter name="pod_manifest" required="0" unique="0"> |
| 88 |
<longdesc lang="en"> |
| 89 |
The Pod manifest with the configuration for Etcd. |
| 90 |
</longdesc> |
| 91 |
<shortdesc lang="en">Etcd pod manifest</shortdesc> |
| 92 |
<content type="string" default="${OCF_RESKEY_pod_manifest_default}"/> |
| 93 |
</parameter> |
| 94 |
|
| 95 |
<parameter name="etcd_certs_dir" required="0" unique="0"> |
| 96 |
<longdesc lang="en"> |
| 97 |
The Etcd certificates directory mounted into the etcd container. |
| 98 |
The agent will monitor this directory for changes and restart the etcd container if the certificates have changed. |
| 99 |
</longdesc> |
| 100 |
<shortdesc lang="en">Etcd certificates directory</shortdesc> |
| 101 |
<content type="string" default="${OCF_RESKEY_etcd_certs_dir_default}"/> |
| 102 |
</parameter> |
| 103 |
|
| 104 |
<parameter name="image" required="0" unique="0"> |
| 105 |
<longdesc lang="en"> |
| 106 |
The podman image to base this container off of. |
| 107 |
</longdesc> |
| 108 |
<shortdesc lang="en">podman image</shortdesc> |
| 109 |
<content type="string" default="${OCF_RESKEY_image_default}"/> |
| 110 |
</parameter> |
| 111 |
|
| 112 |
<parameter name="name" required="0" unique="0"> |
| 113 |
<longdesc lang="en"> |
| 114 |
The name to give the created container. By default this will |
| 115 |
be that resource's instance name. |
| 116 |
</longdesc> |
| 117 |
<shortdesc lang="en">podman container name</shortdesc> |
| 118 |
<content type="string" default="${OCF_RESKEY_name_default}"/> |
| 119 |
</parameter> |
| 120 |
|
| 121 |
<parameter name="node_ip_map" unique="0" required="1"> |
| 122 |
<longdesc lang="en"> |
| 123 |
A mapping of node names to IPs. |
| 124 |
|
| 125 |
This takes the form of: |
| 126 |
n1:ip1;n2:ip2 |
| 127 |
|
| 128 |
where the etcd container on n1 would have IP ip1 |
| 129 |
</longdesc> |
| 130 |
<shortdesc lang="en">Container node name to IP mapping</shortdesc> |
| 131 |
<content type="string"/> |
| 132 |
</parameter> |
| 133 |
|
| 134 |
<parameter name="nic" unique="0"> |
| 135 |
<longdesc lang="en"> |
| 136 |
Network interface to lookup interface for host. |
| 137 |
</longdesc> |
| 138 |
<shortdesc lang="en">Network interface</shortdesc> |
| 139 |
<content type="string" default="${OCF_RESKEY_nic_default}"/> |
| 140 |
</parameter> |
| 141 |
|
| 142 |
<parameter name="authfile" required="0" unique="0"> |
| 143 |
<longdesc lang="en"> |
| 144 |
Path of the authentication file. |
| 145 |
|
| 146 |
The file is created by podman login. |
| 147 |
</longdesc> |
| 148 |
<shortdesc lang="en">Path of the authentication file </shortdesc> |
| 149 |
<content type="string" default="${OCF_RESKEY_authfile_default}"/> |
| 150 |
</parameter> |
| 151 |
|
| 152 |
<parameter name="allow_pull" unique="0"> |
| 153 |
<longdesc lang="en"> |
| 154 |
Allow the image to be pulled from the configured podman registry when |
| 155 |
the image does not exist locally. NOTE, this can drastically increase |
| 156 |
the time required to start the container if the image repository is |
| 157 |
pulled over the network. |
| 158 |
</longdesc> |
| 159 |
<shortdesc lang="en">Allow pulling non-local images</shortdesc> |
| 160 |
<content type="boolean" default="${OCF_RESKEY_allow_pull_default}"/> |
| 161 |
</parameter> |
| 162 |
|
| 163 |
<parameter name="run_opts" required="0" unique="0"> |
| 164 |
<longdesc lang="en"> |
| 165 |
Add options to be appended to the 'podman run' command which is used |
| 166 |
when creating the container during the start action. This option allows |
| 167 |
users to do things such as setting a custom entry point and injecting |
| 168 |
environment variables into the newly created container. Note the '-d' |
| 169 |
option is supplied regardless of this value to force containers to run |
| 170 |
in the background. |
| 171 |
|
| 172 |
NOTE: Do not explicitly specify the --name argument in the run_opts. This |
| 173 |
agent will set --name using either the resource's instance or the name |
| 174 |
provided in the 'name' argument of this agent. |
| 175 |
|
| 176 |
</longdesc> |
| 177 |
<shortdesc lang="en">run options</shortdesc> |
| 178 |
<content type="string"/> |
| 179 |
</parameter> |
| 180 |
|
| 181 |
<parameter name="run_cmd" required="0" unique="0"> |
| 182 |
<longdesc lang="en"> |
| 183 |
Specify a command to launch within the container once |
| 184 |
it has initialized. |
| 185 |
</longdesc> |
| 186 |
<shortdesc lang="en">run command</shortdesc> |
| 187 |
<content type="string"/> |
| 188 |
</parameter> |
| 189 |
|
| 190 |
<parameter name="run_cmd_opts" required="0" unique="0"> |
| 191 |
<longdesc lang="en"> |
| 192 |
Options to be added to the 'run_cmd'. |
| 193 |
</longdesc> |
| 194 |
<shortdesc lang="en">run command options</shortdesc> |
| 195 |
<content type="string"/> |
| 196 |
</parameter> |
| 197 |
|
| 198 |
<parameter name="mount_points" required="0" unique="0"> |
| 199 |
<longdesc lang="en"> |
| 200 |
A comma separated list of directories that the container is expecting to use. |
| 201 |
The agent will ensure they exist by running 'mkdir -p' |
| 202 |
</longdesc> |
| 203 |
<shortdesc lang="en">Required mount points</shortdesc> |
| 204 |
<content type="string"/> |
| 205 |
</parameter> |
| 206 |
|
| 207 |
<parameter name="monitor_cmd" required="0" unique="0"> |
| 208 |
<longdesc lang="en"> |
| 209 |
Specify the full path of a command to launch within the container to check |
| 210 |
the health of the container. This command must return 0 to indicate that |
| 211 |
the container is healthy. A non-zero return code will indicate that the |
| 212 |
container has failed and should be recovered. |
| 213 |
|
| 214 |
Note: Using this method for monitoring processes inside a container |
| 215 |
is not recommended, as containerd tries to track processes running |
| 216 |
inside the container and does not deal well with many short-lived |
| 217 |
processes being spawned. Ensure that your container monitors its |
| 218 |
own processes and terminates on fatal error rather than invoking |
| 219 |
a command from the outside. |
| 220 |
</longdesc> |
| 221 |
<shortdesc lang="en">monitor command</shortdesc> |
| 222 |
<content type="string"/> |
| 223 |
</parameter> |
| 224 |
|
| 225 |
<parameter name="force_kill" required="0" unique="0"> |
| 226 |
<longdesc lang="en"> |
| 227 |
Kill a container immediately rather than waiting for it to gracefully |
| 228 |
shutdown |
| 229 |
</longdesc> |
| 230 |
<shortdesc lang="en">force kill</shortdesc> |
| 231 |
<content type="boolean"/> |
| 232 |
</parameter> |
| 233 |
|
| 234 |
<parameter name="reuse" required="0" unique="0"> |
| 235 |
<longdesc lang="en"> |
| 236 |
Allow the container to be reused once it is stopped. By default, |
| 237 |
containers get removed once they are stopped. Enable this option |
| 238 |
to have the particular one persist when this happens. |
| 239 |
</longdesc> |
| 240 |
<shortdesc lang="en">reuse container</shortdesc> |
| 241 |
<content type="boolean" default="${OCF_RESKEY_reuse_default}"/> |
| 242 |
</parameter> |
| 243 |
|
| 244 |
<parameter name="drop_in_dependency" required="0" unique="0"> |
| 245 |
<longdesc lang="en"> |
| 246 |
Use transient drop-in files to add extra dependencies to the systemd |
| 247 |
scopes associated to the container. During reboot, this prevents systemd |
| 248 |
to stop the container before pacemaker. |
| 249 |
</longdesc> |
| 250 |
<shortdesc lang="en">drop-in dependency</shortdesc> |
| 251 |
<content type="boolean"/> |
| 252 |
</parameter> |
| 253 |
|
| 254 |
<parameter name="oom" required="0" unique="0"> |
| 255 |
<longdesc lang="en"> |
| 256 |
Tune the host's Out-Of-Memory (OOM) preferences for containers (accepts values from -1000 to 1000). |
| 257 |
Default to same OOM score as system-node-critical |
| 258 |
https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#node-out-of-memory-behavior |
| 259 |
</longdesc> |
| 260 |
<shortdesc lang="en">OOM for container</shortdesc> |
| 261 |
<content type="integer" default="${OCF_RESKEY_oom_default}"/> |
| 262 |
</parameter> |
| 263 |
|
| 264 |
<parameter name="config_location" required="0" unique="0"> |
| 265 |
<longdesc lang="en"> |
| 266 |
The directory where the resource agent stores its state files, such as the generated etcd configuration and a copy of the pod manifest. |
| 267 |
</longdesc> |
| 268 |
<shortdesc lang="en">Resource agent state directory</shortdesc> |
| 269 |
<content type="string" default="${OCF_RESKEY_config_location_default}"/> |
| 270 |
</parameter> |
| 271 |
|
| 272 |
<parameter name="backup_location" required="0" unique="0"> |
| 273 |
<longdesc lang="en"> |
| 274 |
The directory where the resource agent stores its backups. |
| 275 |
</longdesc> |
| 276 |
<shortdesc lang="en">Resource agent backup directory</shortdesc> |
| 277 |
<content type="string" default="${OCF_RESKEY_backup_location_default}"/> |
| 278 |
</parameter> |
| 279 |
|
| 280 |
<parameter name="max_backup_snapshots" required="0" unique="0"> |
| 281 |
<longdesc lang="en"> |
| 282 |
Maximum number of etcd database snapshots to retain. When a new snapshot is created, |
| 283 |
older snapshots will be automatically removed to maintain this limit. This helps |
| 284 |
control storage usage while ensuring recent backups are available for recovery. |
| 285 |
Set max_backup_snapshots=0 to disable backups. |
| 286 |
</longdesc> |
| 287 |
<shortdesc lang="en">Maximum number of backup snapshots to retain</shortdesc> |
| 288 |
<content type="integer" default="${OCF_RESKEY_max_backup_snapshots_default}"/> |
| 289 |
</parameter> |
| 290 |
|
| 291 |
</parameters> |
| 292 |
|
| 293 |
<actions> |
| 294 |
<action name="start" timeout="600s" /> |
| 295 |
<action name="stop" timeout="90s" /> |
| 296 |
<action name="monitor" timeout="25s" interval="30s" depth="0" /> |
| 297 |
<action name="meta-data" timeout="5s" /> |
| 298 |
<action name="validate-all" timeout="30s" /> |
| 299 |
</actions> |
| 300 |
</resource-agent> |
| 301 |
END |
| 302 |
} |
| 303 |
|
| 304 |
####################################################################### |
| 305 |
REQUIRE_IMAGE_PULL=0 |
| 306 |
|
| 307 |
podman_usage() |
| 308 |
{ |
| 309 |
cat <<END |
| 310 |
usage: $0 {start|stop|monitor|validate-all|meta-data} |
| 311 |
|
| 312 |
Expects to have a fully populated OCF RA-compliant environment set. |
| 313 |
END |
| 314 |
} |
| 315 |
|
| 316 |
etcd_certificates_hash_manager() |
| 317 |
{ |
| 318 |
local action="$1" |
| 319 |
local current_hash |
| 320 |
local stored_hash |
| 321 |
|
| 322 |
# If the certs directory doesn't exist, consider it unchanged |
| 323 |
if [ ! -d "$OCF_RESKEY_etcd_certs_dir" ]; then |
| 324 |
ocf_log warn "certificates directory $OCF_RESKEY_etcd_certs_dir does not exist, skipping certificate monitoring" |
| 325 |
return $OCF_SUCCESS |
| 326 |
fi |
| 327 |
|
| 328 |
# Calculate hash of all certificate files, ignore key files to avoid accidental disclosure of sensitive information |
| 329 |
# we only need to monitor the certificate files to detect changes. |
| 330 |
if ! current_hash=$(find "$OCF_RESKEY_etcd_certs_dir" -type f \( -name "*.crt" \) -exec sha256sum {} \; | sort | sha256sum | cut -d' ' -f1); then |
| 331 |
ocf_log err "failed to calculate certificate files hash" |
| 332 |
return $OCF_ERR_GENERIC |
| 333 |
fi |
| 334 |
|
| 335 |
# If no stored hash exists, create one and return success |
| 336 |
if [ ! -f "$ETCD_CERTS_HASH_FILE" ]; then |
| 337 |
echo "$current_hash" > "$ETCD_CERTS_HASH_FILE" |
| 338 |
ocf_log info "created initial certificate hash: $current_hash" |
| 339 |
return $OCF_SUCCESS |
| 340 |
fi |
| 341 |
|
| 342 |
case "$action" in |
| 343 |
"update") |
| 344 |
if ! echo "$current_hash" > "$ETCD_CERTS_HASH_FILE"; then |
| 345 |
ocf_log err "failed to update certificate hash file $ETCD_CERTS_HASH_FILE" |
| 346 |
fi |
| 347 |
ocf_log info "updated certificate hash: $current_hash" |
| 348 |
;; |
| 349 |
"check") |
| 350 |
if ! stored_hash=$(cat "$ETCD_CERTS_HASH_FILE"); then |
| 351 |
ocf_log err "failed to read stored certificate hash from $ETCD_CERTS_HASH_FILE" |
| 352 |
# This should not happen but if for some reason we can not read the stored hash, |
| 353 |
# use the current hash and log the error but allow etcd to run as long as possible. |
| 354 |
stored_hash="$current_hash" |
| 355 |
fi |
| 356 |
if [ "$current_hash" != "$stored_hash" ]; then |
| 357 |
ocf_exit_reason "$NODENAME etcd certificate files have changed (stored: $stored_hash, current: $current_hash)" |
| 358 |
return $OCF_ERR_GENERIC |
| 359 |
fi |
| 360 |
;; |
| 361 |
*) |
| 362 |
ocf_log err "unsupported action: $action" |
| 363 |
return $OCF_ERR_GENERIC |
| 364 |
;; |
| 365 |
esac |
| 366 |
|
| 367 |
return $OCF_SUCCESS |
| 368 |
} |
| 369 |
|
| 370 |
monitor_cmd_exec() |
| 371 |
{ |
| 372 |
local rc=$OCF_SUCCESS |
| 373 |
local out |
| 374 |
local attempt |
| 375 |
# 3 attempts × 5s = 15s worst case, fits within the 25s monitor timeout. |
| 376 |
# The health check normally completes in <1s; the 5s per-attempt timeout |
| 377 |
# is a safety net for when the container's process namespace is slow. |
| 378 |
local max_attempts=3 |
| 379 |
local attempt_timeout=5 |
| 380 |
|
| 381 |
for attempt in $(seq 1 $max_attempts); do |
| 382 |
out=$(timeout $attempt_timeout podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1) |
| 383 |
rc=$? |
| 384 |
# 125: no container with name or ID ${CONTAINER} found |
| 385 |
# 126: container state improper (not running) |
| 386 |
# 127: any other error |
| 387 |
# 255: podman 2+: container not running |
| 388 |
case "$rc" in |
| 389 |
125|126|255) |
| 390 |
return $OCF_NOT_RUNNING |
| 391 |
;; |
| 392 |
0) |
| 393 |
ocf_log debug "monitor cmd passed: exit code = $rc" |
| 394 |
return $OCF_SUCCESS |
| 395 |
;; |
| 396 |
*) |
| 397 |
ocf_log warn "monitor cmd failed (rc=$rc), output: $out" |
| 398 |
rc=$OCF_ERR_GENERIC |
| 399 |
;; |
| 400 |
esac |
| 401 |
done |
| 402 |
|
| 403 |
return $rc |
| 404 |
} |
| 405 |
|
| 406 |
container_exists() |
| 407 |
{ |
| 408 |
local rc |
| 409 |
local out |
| 410 |
|
| 411 |
out=$(podman exec ${CONTAINER} $OCF_RESKEY_monitor_cmd 2>&1) |
| 412 |
rc=$? |
| 413 |
# 125: no container with name or ID ${CONTAINER} found |
| 414 |
if [ $rc -ne 125 ]; then |
| 415 |
return 0 |
| 416 |
fi |
| 417 |
return 1 |
| 418 |
} |
| 419 |
|
| 420 |
# archive_current_container archives the current |
| 421 |
# podman etcd container and its configuration files. |
| 422 |
archive_current_container() |
| 423 |
{ |
| 424 |
# don't attempt to archive a container that doesn't exist |
| 425 |
if ! container_exists; then |
| 426 |
return |
| 427 |
fi |
| 428 |
|
| 429 |
# delete any container named "*-previous", or we won't be able to archive the current container. |
| 430 |
if podman inspect "${CONTAINER}-previous" >/dev/null 2>&1; then |
| 431 |
ocf_log info "removing old archived container '$CONTAINER-previous'" |
| 432 |
if ! ocf_run podman rm --volumes --force "$CONTAINER-previous"; then |
| 433 |
ocf_log warn "could not remove old archived container (podman rm failed, error code: $?). Won't be able to archive current container" |
| 434 |
return |
| 435 |
fi |
| 436 |
fi |
| 437 |
|
| 438 |
ocf_log info "archiving '$CONTAINER' container as '$CONTAINER-previous' for debugging purposes" |
| 439 |
if ! ocf_run podman rename "$CONTAINER" "$CONTAINER-previous"; then |
| 440 |
ocf_log err "could not archive container '$CONTAINER', error code: $?" |
| 441 |
return |
| 442 |
fi |
| 443 |
|
| 444 |
# archive corresponding etcd configuration files |
| 445 |
local files_to_archive="" |
| 446 |
for file in "$OCF_RESKEY_authfile" "$POD_MANIFEST_COPY" "$ETCD_CONFIGURATION_FILE" "$ETCD_CERTS_HASH_FILE"; do |
| 447 |
if [ -f "$file" ]; then |
| 448 |
files_to_archive="$files_to_archive $file" |
| 449 |
else |
| 450 |
ocf_log warn "file '$file' is missing and won't be archived" |
| 451 |
fi |
| 452 |
done |
| 453 |
|
| 454 |
if [ -z "$files_to_archive" ]; then |
| 455 |
ocf_log warn "could not find any file to archive." |
| 456 |
return |
| 457 |
fi |
| 458 |
|
| 459 |
# NOTE: tar will override any existing archive as wanted |
| 460 |
# shellcheck disable=SC2086 |
| 461 |
if ! ocf_run tar --create --verbose --gzip --file "$ETCD_BACKUP_FILE" $files_to_archive; then |
| 462 |
ocf_log warn "container archived successfully, but configuration backup failed (error: $?). Container debugging available, but without matching configuration files" |
| 463 |
else |
| 464 |
ocf_log info "container configuration also archived in '$ETCD_BACKUP_FILE'" |
| 465 |
fi |
| 466 |
} |
| 467 |
|
| 468 |
# Correctly wraps an ipv6 in [] for url otherwise use return normal ipv4 address. |
| 469 |
ip_url() { |
| 470 |
local ip_addr=$1 |
| 471 |
local value |
| 472 |
if echo "$ip_addr" | grep -q ":" ; then |
| 473 |
value="[$ip_addr]" |
| 474 |
else |
| 475 |
value="$ip_addr" |
| 476 |
fi |
| 477 |
echo "https://$value" |
| 478 |
} |
| 479 |
|
| 480 |
attribute_node_ip() |
| 481 |
{ |
| 482 |
local action="$1" |
| 483 |
local attribute="node_ip" |
| 484 |
local ip_addr name |
| 485 |
|
| 486 |
# TODO: We can retrieve both the local and peer IP addresses from this map, which eliminates the need to use CIB to share them between nodes |
| 487 |
for node in $(echo "$OCF_RESKEY_node_ip_map" | sed "s/\s//g;s/;/ /g"); do |
| 488 |
name=$(echo "$node" | cut -d: -f1) |
| 489 |
# ignore other nodes |
| 490 |
if [ "$name" != "$NODENAME" ]; then |
| 491 |
continue |
| 492 |
fi |
| 493 |
ip_addr=$(echo "$node" | cut -d: -f2-) # Grab everything after the first : this covers ipv4/ipv6 |
| 494 |
done |
| 495 |
|
| 496 |
if [ -z "$ip_addr" ]; then |
| 497 |
ocf_log err "could not get local ip address from node_ip_map: '$OCF_RESKEY_node_ip_map'" |
| 498 |
return 1 |
| 499 |
fi |
| 500 |
|
| 501 |
case "$action" in |
| 502 |
get) |
| 503 |
echo "$ip_addr" |
| 504 |
;; |
| 505 |
update) |
| 506 |
if ! crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$ip_addr"; then |
| 507 |
rc="$?" |
| 508 |
ocf_log err "could not set $attribute to $ip_addr, error code: $rc" |
| 509 |
return "$rc" |
| 510 |
fi |
| 511 |
;; |
| 512 |
clear) |
| 513 |
crm_attribute --name "$attribute" --delete |
| 514 |
;; |
| 515 |
*) |
| 516 |
ocf_log err "unsupported $action for $attribute" |
| 517 |
return $OCF_ERR_GENERIC |
| 518 |
;; |
| 519 |
esac |
| 520 |
} |
| 521 |
|
| 522 |
attribute_node_ip_peer() { |
| 523 |
local peer_name |
| 524 |
peer_name=$(get_peer_node_name) |
| 525 |
crm_attribute --query --name "node_ip" --node "$peer_name" | awk -F"value=" '{print $2}' |
| 526 |
} |
| 527 |
|
| 528 |
get_env_from_manifest() { |
| 529 |
local env_var_name="$1" |
| 530 |
local env_var_value |
| 531 |
|
| 532 |
# The agent waits for the manifest to exist before starting, so the |
| 533 |
# file should exist already, but this check is included for robustness. |
| 534 |
if [ ! -f "$OCF_RESKEY_pod_manifest" ]; then |
| 535 |
ocf_log err "external etcd pod manifest ($OCF_RESKEY_pod_manifest) not found" |
| 536 |
exit "$OCF_ERR_INSTALLED" |
| 537 |
fi |
| 538 |
|
| 539 |
env_var_value=$(jq -r ".spec.containers[].env[] | select( .name == \"$env_var_name\" ).value" "$OCF_RESKEY_pod_manifest") |
| 540 |
rc=$? |
| 541 |
if [ $rc -ne 0 ]; then |
| 542 |
ocf_log err "could not find environment variable $env_var_name in etcd pod manifest, error code: $rc" |
| 543 |
exit "$OCF_ERR_INSTALLED" |
| 544 |
fi |
| 545 |
|
| 546 |
ocf_log debug "ETCD pod environment variable $env_var_name: $env_var_value" |
| 547 |
|
| 548 |
echo "$env_var_value" |
| 549 |
} |
| 550 |
|
| 551 |
# etcd configuration file expects duration to be expressed in nanoseconds |
| 552 |
convert_duration_in_nanoseconds() { |
| 553 |
local duration=$1 |
| 554 |
local value unit nanoseconds |
| 555 |
|
| 556 |
if [ -z "$duration" ]; then |
| 557 |
ocf_log err "convert_duration_in_nanoseconds: no duration provided" |
| 558 |
return 1 |
| 559 |
fi |
| 560 |
|
| 561 |
if ! echo "$duration" | grep -qE '^[0-9]+[numµ]?s$'; then |
| 562 |
ocf_log err "convert_duration_in_nanoseconds: invalid duration format \"$duration\". Expected format: <number><unit> where unit is one of s, ms, us, µs, ns" |
| 563 |
return 1 |
| 564 |
fi |
| 565 |
|
| 566 |
# Extract numeric value and unit from duration string |
| 567 |
value=$(echo "$duration" | sed 's/[^0-9]*$//') |
| 568 |
unit=$(echo "$duration" | sed 's/^[0-9]*//') |
| 569 |
|
| 570 |
case "$unit" in |
| 571 |
ns) |
| 572 |
nanoseconds=$value |
| 573 |
;; |
| 574 |
us|µs) |
| 575 |
nanoseconds=$((value * 1000)) |
| 576 |
;; |
| 577 |
ms) |
| 578 |
nanoseconds=$((value * 1000000)) |
| 579 |
;; |
| 580 |
s) |
| 581 |
nanoseconds=$((value * 1000000000)) |
| 582 |
;; |
| 583 |
*) |
| 584 |
# this should not happen as the input is already validated |
| 585 |
ocf_log err "convert_duration_in_nanoseconds: unknown duration unit \"$unit\"" |
| 586 |
return 1 |
| 587 |
;; |
| 588 |
esac |
| 589 |
|
| 590 |
echo "$nanoseconds" |
| 591 |
} |
| 592 |
|
| 593 |
prepare_env() { |
| 594 |
local name ip ipurl standalone_node |
| 595 |
|
| 596 |
NODEIP="$(attribute_node_ip get)" |
| 597 |
NODEIPURL=$(ip_url $NODEIP) |
| 598 |
|
| 599 |
if is_force_new_cluster; then |
| 600 |
ALL_ETCD_ENDPOINTS="$NODEIPURL:2379" |
| 601 |
ETCD_INITIAL_CLUSTER_STATE="new" |
| 602 |
ETCD_INITIAL_CLUSTER="$NODENAME=$NODEIPURL:2380" |
| 603 |
else |
| 604 |
ETCD_INITIAL_CLUSTER_STATE="existing" |
| 605 |
for node in $(echo "$OCF_RESKEY_node_ip_map" | sed "s/\s//g;s/;/ /g"); do |
| 606 |
name=$(echo "$node" | cut -d: -f1) |
| 607 |
ip=$(echo "$node" | cut -d: -f2-) # Grab everything after the first : this covers ipv4/ipv6 |
| 608 |
ipurl="$(ip_url $ip)" |
| 609 |
if [ -z "$name" ] || [ -z "$ip" ]; then |
| 610 |
ocf_exit_reason "name or ip missing for 1 or more nodes" |
| 611 |
exit $OCF_ERR_CONFIGURED |
| 612 |
fi |
| 613 |
|
| 614 |
[ -z "$ALL_ETCD_ENDPOINTS" ] && ALL_ETCD_ENDPOINTS="$ipurl:2379" || ALL_ETCD_ENDPOINTS="$ALL_ETCD_ENDPOINTS,$ipurl:2379" |
| 615 |
[ -z "$ETCD_INITIAL_CLUSTER" ] && ETCD_INITIAL_CLUSTER="$name=$ipurl:2380" || ETCD_INITIAL_CLUSTER="$ETCD_INITIAL_CLUSTER,$name=$ipurl:2380" |
| 616 |
done |
| 617 |
fi |
| 618 |
|
| 619 |
ETCD_CIPHER_SUITES=$(get_env_from_manifest "ETCD_CIPHER_SUITES") |
| 620 |
ETCD_DATA_DIR=$(get_env_from_manifest "ETCD_DATA_DIR") |
| 621 |
if [ ! -d "$ETCD_DATA_DIR" ]; then |
| 622 |
ocf_log err "could not find data-dir at path \"$ETCD_DATA_DIR\"" |
| 623 |
return "$OCF_ERR_ARGS" |
| 624 |
else |
| 625 |
ocf_log info "using data-dir: $ETCD_DATA_DIR" |
| 626 |
fi |
| 627 |
ETCD_ELECTION_TIMEOUT=$(get_env_from_manifest "ETCD_ELECTION_TIMEOUT") |
| 628 |
ETCD_ENABLE_PPROF=$(get_env_from_manifest "ETCD_ENABLE_PPROF") |
| 629 |
ETCD_WARNING_APPLY_DURATION=$(get_env_from_manifest "ETCD_WARNING_APPLY_DURATION") |
| 630 |
ETCD_WATCH_PROGRESS_NOTIFY_INTERVAL=$(get_env_from_manifest "ETCD_WATCH_PROGRESS_NOTIFY_INTERVAL") |
| 631 |
ETCD_HEARTBEAT_INTERVAL=$(get_env_from_manifest "ETCD_HEARTBEAT_INTERVAL") |
| 632 |
ETCD_QUOTA_BACKEND_BYTES=$(get_env_from_manifest "ETCD_QUOTA_BACKEND_BYTES") |
| 633 |
ETCD_SOCKET_REUSE_ADDRESS=$(get_env_from_manifest "ETCD_SOCKET_REUSE_ADDRESS") |
| 634 |
|
| 635 |
SERVER_CACERT=$(get_env_from_manifest "ETCDCTL_CACERT") |
| 636 |
ETCD_PEER_CERT=$(get_env_from_manifest "ETCDCTL_CERT") |
| 637 |
ETCD_PEER_KEY=$(get_env_from_manifest "ETCDCTL_KEY") |
| 638 |
|
| 639 |
LISTEN_CLIENT_URLS="0.0.0.0" |
| 640 |
LISTEN_PEER_URLS="0.0.0.0" |
| 641 |
LISTEN_METRICS_URLS="0.0.0.0" |
| 642 |
|
| 643 |
return $OCF_SUCCESS |
| 644 |
} |
| 645 |
|
| 646 |
compute_bump_revision() { |
| 647 |
local rc |
| 648 |
|
| 649 |
# Same logic used by cluster-etcd-operator quorum-restore-pod utility. |
| 650 |
# see https://github.com/openshift/cluster-etcd-operator/blob/215998939f5223da916622c71fd07d17656faf6b/bindata/etcd/quorum-restore-pod.yaml#L26-L34 |
| 651 |
# set a default value: 1bn would be an etcd running at 1000 writes/s for about eleven days. |
| 652 |
BUMP_REV=$ETCD_BUMP_REV_DEFAULT |
| 653 |
if [ ! -f "${ETCD_REVISION_JSON}" ]; then |
| 654 |
ocf_log err "could not compute bump revision: ${ETCD_REVISION_JSON} not found. Defaulting to ${ETCD_BUMP_REV_DEFAULT} revision bump" |
| 655 |
return |
| 656 |
fi |
| 657 |
|
| 658 |
# this will bump by the amount of 20% of the last known live revision. |
| 659 |
if ! COMPUTED_BUMP=$(jq -r "(.maxRaftIndex*${ETCD_REVISION_BUMP_PERCENTAGE}|floor)" "${ETCD_REVISION_JSON}"); then |
| 660 |
ocf_log err "could not compute maxRaftIndex for bump revision, jq error code: $?. Defaulting to ${ETCD_BUMP_REV_DEFAULT} revision bump" |
| 661 |
return |
| 662 |
fi |
| 663 |
|
| 664 |
if [ -z "${COMPUTED_BUMP}" ] || [ "${COMPUTED_BUMP}" -le 0 ] || [ "${COMPUTED_BUMP}" -gt "${ETCD_BUMP_REV_DEFAULT}" ]; then |
| 665 |
ocf_log err "computed bump revision (${COMPUTED_BUMP}) is invalid. Defaulting to ${ETCD_BUMP_REV_DEFAULT} revision bump" |
| 666 |
return |
| 667 |
fi |
| 668 |
|
| 669 |
BUMP_REV="${COMPUTED_BUMP}" |
| 670 |
ocf_log info "bumping etcd revisions by ${BUMP_REV}" |
| 671 |
} |
| 672 |
|
| 673 |
generate_etcd_configuration() { |
| 674 |
if is_force_new_cluster; then |
| 675 |
compute_bump_revision |
| 676 |
# The embedded newline is required for correct YAML formatting. |
| 677 |
FORCE_NEW_CLUSTER_CONFIG="force-new-cluster: true |
| 678 |
force-new-cluster-bump-amount: $BUMP_REV" |
| 679 |
else |
| 680 |
FORCE_NEW_CLUSTER_CONFIG="force-new-cluster: false" |
| 681 |
fi |
| 682 |
|
| 683 |
# the space indentation for client-transport-security and peer-transport-security |
| 684 |
# is required for correct YAML formatting. |
| 685 |
# TODO: replace flags deprecated in Etcd v3.6 |
| 686 |
cat > "$ETCD_CONFIGURATION_FILE" << EOF |
| 687 |
logger: zap |
| 688 |
log-level: info |
| 689 |
snapshot-count: 10000 |
| 690 |
name: $NODENAME |
| 691 |
data-dir: $ETCD_DATA_DIR |
| 692 |
$FORCE_NEW_CLUSTER_CONFIG |
| 693 |
socket-reuse-address: $ETCD_SOCKET_REUSE_ADDRESS |
| 694 |
election-timeout: $ETCD_ELECTION_TIMEOUT |
| 695 |
enable-pprof: $ETCD_ENABLE_PPROF |
| 696 |
heartbeat-interval: $ETCD_HEARTBEAT_INTERVAL |
| 697 |
quota-backend-bytes: $ETCD_QUOTA_BACKEND_BYTES |
| 698 |
initial-advertise-peer-urls: "$NODEIPURL:2380" |
| 699 |
listen-peer-urls: "$(ip_url ${LISTEN_PEER_URLS}):2380" |
| 700 |
listen-client-urls: "$(ip_url ${LISTEN_CLIENT_URLS}):2379,unixs://${NODEIP}:0" |
| 701 |
initial-cluster: $ETCD_INITIAL_CLUSTER |
| 702 |
initial-cluster-state: $ETCD_INITIAL_CLUSTER_STATE |
| 703 |
client-transport-security: |
| 704 |
cert-file: /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-${NODENAME}.crt |
| 705 |
key-file: /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-${NODENAME}.key |
| 706 |
client-cert-auth: true |
| 707 |
trusted-ca-file: $SERVER_CACERT |
| 708 |
peer-transport-security: |
| 709 |
cert-file: $ETCD_PEER_CERT |
| 710 |
key-file: $ETCD_PEER_KEY |
| 711 |
client-cert-auth: true |
| 712 |
trusted-ca-file: $SERVER_CACERT |
| 713 |
advertise-client-urls: "$NODEIPURL:2379" |
| 714 |
listen-metrics-urls: "$(ip_url ${LISTEN_METRICS_URLS}):9978" |
| 715 |
metrics: extensive |
| 716 |
experimental-initial-corrupt-check: true |
| 717 |
experimental-max-learners: 1 |
| 718 |
experimental-warning-apply-duration: $(convert_duration_in_nanoseconds "$ETCD_WARNING_APPLY_DURATION") |
| 719 |
experimental-watch-progress-notify-interval: $(convert_duration_in_nanoseconds "$ETCD_WATCH_PROGRESS_NOTIFY_INTERVAL") |
| 720 |
EOF |
| 721 |
rc=$? |
| 722 |
if [ $rc -ne 0 ]; then |
| 723 |
ocf_log err "could not create etcd configuration, 'cat' error code: $rc" |
| 724 |
return $OCF_ERR_CONFIGURED |
| 725 |
fi |
| 726 |
|
| 727 |
# Append cipher suites from the env variable where the entries are comma separated. |
| 728 |
{ |
| 729 |
if [ -n "$ETCD_CIPHER_SUITES" ]; then |
| 730 |
echo "cipher-suites:" |
| 731 |
echo "$ETCD_CIPHER_SUITES" | tr ',' '\n' | while read -r cipher; do |
| 732 |
echo " - \"$cipher\"" |
| 733 |
done |
| 734 |
fi |
| 735 |
} >> "$ETCD_CONFIGURATION_FILE" |
| 736 |
rc=$? |
| 737 |
if [ $rc -ne 0 ]; then |
| 738 |
ocf_log err "could not append cipher suites to etcd configuration, error code: $rc" |
| 739 |
return $OCF_ERR_CONFIGURED |
| 740 |
fi |
| 741 |
|
| 742 |
return $OCF_SUCCESS |
| 743 |
} |
| 744 |
|
| 745 |
# Remove etcd member directory to allow the node to rejoin the cluster as a learner. |
| 746 |
# |
| 747 |
# When a node rejoins an etcd cluster, it must start fresh as a learner to prevent |
| 748 |
# data inconsistencies. This function removes the member directory and syncs to disk. |
| 749 |
# |
| 750 |
# Returns: |
| 751 |
# OCF_SUCCESS - Member directory successfully removed |
| 752 |
# OCF_ERR_GENERIC - Failed to remove member directory (critical error) |
| 753 |
wipe_data_folder_for_learner() |
| 754 |
{ |
| 755 |
ocf_log info "deleting etcd member directory ($ETCD_MEMBER_DIR) to enable learner rejoin" |
| 756 |
if ! rm -rf "$ETCD_MEMBER_DIR"; then |
| 757 |
ocf_log err "could not delete etcd member directory ($ETCD_MEMBER_DIR), error code: $?" |
| 758 |
return $OCF_ERR_GENERIC |
| 759 |
fi |
| 760 |
sync |
| 761 |
return $OCF_SUCCESS |
| 762 |
} |
| 763 |
|
| 764 |
|
| 765 |
# Calculate available disk space in bytes for a given directory. |
| 766 |
# |
| 767 |
# This function queries the filesystem and returns available space in bytes. |
| 768 |
# It converts df output (KB) to bytes for consistent size comparisons. |
| 769 |
# |
| 770 |
# Arguments: |
| 771 |
# $1 - Target directory path to check |
| 772 |
# |
| 773 |
# Returns: |
| 774 |
# OCF_SUCCESS - Available space in bytes (via stdout) |
| 775 |
# OCF_ERR_GENERIC - Failed to determine available space (error message via stdout) |
| 776 |
get_available_space_in_directory() |
| 777 |
{ |
| 778 |
local target_dir=$1 |
| 779 |
local available_space_kb |
| 780 |
local available_space_bytes |
| 781 |
|
| 782 |
available_space_kb=$(df -P "$target_dir" | awk 'NR==2 {print $4}' 2>&1) |
| 783 |
|
| 784 |
# Validate output is numeric |
| 785 |
if ! echo "$available_space_kb" | grep -q '^[0-9]\+$'; then |
| 786 |
echo "df command failed or returned invalid value: $available_space_kb" |
| 787 |
return $OCF_ERR_GENERIC |
| 788 |
fi |
| 789 |
|
| 790 |
available_space_bytes=$((available_space_kb*1024)) |
| 791 |
echo "$available_space_bytes" |
| 792 |
return $OCF_SUCCESS |
| 793 |
} |
| 794 |
|
| 795 |
# Archive etcd database with backup and cleanup |
| 796 |
# |
| 797 |
# This function creates a backup copy of the etcd database, validates it, and |
| 798 |
# removes old backups according to the retention policy. Backups are optional |
| 799 |
# and can be disabled by setting max_backup_snapshots=0. |
| 800 |
# |
| 801 |
# Error handling strategy: |
| 802 |
# All backup failures return OCF_SUCCESS to prevent blocking cluster recovery. |
| 803 |
# Backups are beneficial but not critical for recovery operations. |
| 804 |
# |
| 805 |
# NOTE: This function cannot use etcdctl/etcdutl utilities because the etcd |
| 806 |
# server is not running when this backup is performed. |
| 807 |
archive_data_folder() |
| 808 |
{ |
| 809 |
local backup_dir="$OCF_RESKEY_backup_location" |
| 810 |
local etcd_db_path="$ETCD_MEMBER_DIR/snap/db" |
| 811 |
|
| 812 |
if [ "$OCF_RESKEY_max_backup_snapshots" -eq 0 ]; then |
| 813 |
ocf_log debug "etcd backup disabled (max_backup_snapshots=0)" |
| 814 |
return $OCF_SUCCESS |
| 815 |
fi |
| 816 |
|
| 817 |
# Check if the etcd database file exists |
| 818 |
if [ ! -f "$etcd_db_path" ]; then |
| 819 |
ocf_log warn "backup skipped: etcd database file not found at '$etcd_db_path'" |
| 820 |
return $OCF_SUCCESS |
| 821 |
fi |
| 822 |
|
| 823 |
# Ensure backup directory exists |
| 824 |
if [ ! -d "$backup_dir" ]; then |
| 825 |
ocf_log debug "creating backup directory: '$backup_dir'" |
| 826 |
if ! mkdir -p "$backup_dir"; then |
| 827 |
ocf_log warn "backup skipped: failed to create backup directory '$backup_dir'" |
| 828 |
return $OCF_SUCCESS |
| 829 |
fi |
| 830 |
fi |
| 831 |
|
| 832 |
ocf_log debug "checking disk space: backup_dir=$backup_dir" |
| 833 |
local available_space_bytes |
| 834 |
if ! available_space_bytes=$(get_available_space_in_directory "$backup_dir"); then |
| 835 |
ocf_log warn "backup skipped: could not compute available disk space in '$backup_dir', error msg: $available_space_bytes" |
| 836 |
return $OCF_SUCCESS |
| 837 |
fi |
| 838 |
|
| 839 |
local required_space_bytes |
| 840 |
required_space_bytes=$(stat -c %s "$etcd_db_path" 2>&1) |
| 841 |
if ! echo "$required_space_bytes" | grep -q '^[0-9]\+$'; then |
| 842 |
ocf_log warn "backup skipped: could not compute etcd database size at '$etcd_db_path', error msg: $required_space_bytes" |
| 843 |
return $OCF_SUCCESS |
| 844 |
fi |
| 845 |
|
| 846 |
if [ "$required_space_bytes" -gt "$available_space_bytes" ]; then |
| 847 |
ocf_log warn "backup skipped: insufficient disk space (required: ${required_space_bytes}B, available: ${available_space_bytes}B)" |
| 848 |
return $OCF_SUCCESS |
| 849 |
fi |
| 850 |
|
| 851 |
# Generate timestamp and backup filename |
| 852 |
local timestamp |
| 853 |
timestamp=$(date +%Y%m%d-%H%M%S) |
| 854 |
|
| 855 |
local backup_file |
| 856 |
backup_file="$backup_dir/snapshot-$timestamp.db" |
| 857 |
|
| 858 |
ocf_log info "creating etcd database backup: '$backup_file'" |
| 859 |
|
| 860 |
# Create the backup by copying the database file (enable Copy-on-Write copy) |
| 861 |
if ! cp --reflink=auto "$etcd_db_path" "$backup_file"; then |
| 862 |
ocf_log warn "backup creation failed: could not copy '$etcd_db_path' to '$backup_file', error code: $?" |
| 863 |
return $OCF_SUCCESS |
| 864 |
fi |
| 865 |
|
| 866 |
# Validate the backup file exists and has the expected size |
| 867 |
if [ ! -f "$backup_file" ]; then |
| 868 |
ocf_log warn "backup validation failed: snapshot file '$backup_file' does not exist" |
| 869 |
return $OCF_SUCCESS |
| 870 |
fi |
| 871 |
|
| 872 |
local backup_size_bytes |
| 873 |
backup_size_bytes=$(stat -c %s "$backup_file" 2>/dev/null || echo "0") |
| 874 |
if [ "$backup_size_bytes" -ne "$required_space_bytes" ]; then |
| 875 |
ocf_log warn "backup validation failed: size mismatch (expected: ${required_space_bytes}B, got: ${backup_size_bytes}B)" |
| 876 |
rm -f "$backup_file" |
| 877 |
return $OCF_SUCCESS |
| 878 |
fi |
| 879 |
|
| 880 |
ocf_log info "backup created successfully: $backup_file (${backup_size_bytes}B)" |
| 881 |
|
| 882 |
# Cleanup old backups based on retention policy |
| 883 |
cleanup_old_backups "$backup_dir" |
| 884 |
|
| 885 |
return $OCF_SUCCESS |
| 886 |
} |
| 887 |
|
| 888 |
cleanup_old_backups() |
| 889 |
{ |
| 890 |
local backup_dir="$1" |
| 891 |
local max_snapshots="$OCF_RESKEY_max_backup_snapshots" |
| 892 |
local backup_count |
| 893 |
local backups_to_remove |
| 894 |
local old_backups |
| 895 |
|
| 896 |
# Validate max_snapshots is a positive integer |
| 897 |
if ! echo "$max_snapshots" | grep -q '^[1-9][0-9]*$'; then |
| 898 |
ocf_log warn "invalid max_backup_snapshots value. Positive integer expected, got '$max_snapshots' instead, skipping cleanup" |
| 899 |
return $OCF_SUCCESS |
| 900 |
fi |
| 901 |
|
| 902 |
# Count existing backup files |
| 903 |
backup_count=$(find "$backup_dir" -maxdepth 1 -name "snapshot-*.db" -type f 2>/dev/null | wc -l) |
| 904 |
|
| 905 |
if [ "$backup_count" -le "$max_snapshots" ]; then |
| 906 |
ocf_log info "backup count ($backup_count) is within retention limit ($max_snapshots), no cleanup needed" |
| 907 |
return $OCF_SUCCESS |
| 908 |
fi |
| 909 |
|
| 910 |
# Calculate how many backups to remove |
| 911 |
backups_to_remove=$((backup_count - max_snapshots)) |
| 912 |
ocf_log info "removing $backups_to_remove old backup(s) to maintain retention limit of $max_snapshots" |
| 913 |
|
| 914 |
# Find oldest backups sorted by modification time |
| 915 |
# -t sorts by modification time, -r reverses (oldest first) |
| 916 |
# -print0 and -0 handle filenames with spaces/special characters |
| 917 |
old_backups=$(find "$backup_dir" -maxdepth 1 -name "snapshot-*.db" -type f -print0 2>/dev/null | \ |
| 918 |
xargs -0 -r ls -tr | \ |
| 919 |
head -n "$backups_to_remove") |
| 920 |
|
| 921 |
if [ -n "$old_backups" ]; then |
| 922 |
ocf_log info "removing old backups: $old_backups" |
| 923 |
if ! echo "$old_backups" | xargs -r rm -f; then |
| 924 |
ocf_log warn "failed to remove some old backups, error code: $?" |
| 925 |
fi |
| 926 |
fi |
| 927 |
|
| 928 |
return $OCF_SUCCESS |
| 929 |
} |
| 930 |
|
| 931 |
etcd_pod_container_exists() { |
| 932 |
local count_matches |
| 933 |
# Check whether the etcd pod exists on the same node (including stopped/exited containers) |
| 934 |
count_matches=$(crictl pods --label app=etcd -q | xargs -I {} crictl ps -a --pod {} -o json | jq -r '.containers[].metadata | select ( .name == "etcd" ).name' | wc -l) |
| 935 |
if [ "$count_matches" -eq 1 ]; then |
| 936 |
# etcd pod found |
| 937 |
return 0 |
| 938 |
fi |
| 939 |
# etcd pod not found |
| 940 |
return 1 |
| 941 |
} |
| 942 |
|
| 943 |
attribute_node_cluster_id() |
| 944 |
{ |
| 945 |
local action="$1" |
| 946 |
local value |
| 947 |
local rc |
| 948 |
value=$(jq -r ".clusterId" "$ETCD_REVISION_JSON") |
| 949 |
rc=$? |
| 950 |
if [ $rc -ne 0 ]; then |
| 951 |
# Log the error but return success to avoid monitor failure if the file is not available yet. |
| 952 |
#This should not block cluster recovery. |
| 953 |
ocf_log err "could not get cluster_id, error code: $rc" |
| 954 |
return $OCF_SUCCESS |
| 955 |
fi |
| 956 |
|
| 957 |
case "$action" in |
| 958 |
get) |
| 959 |
echo "$value" |
| 960 |
;; |
| 961 |
update) |
| 962 |
crm_attribute --type nodes --node "$NODENAME" --name "cluster_id" --update "$value" |
| 963 |
rc=$? |
| 964 |
if [ $rc -ne 0 ]; then |
| 965 |
# Log the error but return success to avoid monitor failure if we can not update the attribute. |
| 966 |
ocf_log err "could not update cluster_id, error code: $rc" |
| 967 |
return $OCF_SUCCESS |
| 968 |
fi |
| 969 |
;; |
| 970 |
*) |
| 971 |
ocf_log err "unsupported $action for attribute_node_cluster_id" |
| 972 |
return $OCF_ERR_GENERIC |
| 973 |
;; |
| 974 |
esac |
| 975 |
} |
| 976 |
|
| 977 |
attribute_node_cluster_id_peer() |
| 978 |
{ |
| 979 |
local nodename |
| 980 |
|
| 981 |
nodename=$(get_peer_node_name) |
| 982 |
crm_attribute --query --type nodes --node "$nodename" --name "cluster_id" | awk -F"value=" '{print $2}' |
| 983 |
} |
| 984 |
|
| 985 |
attribute_node_revision() |
| 986 |
{ |
| 987 |
local action="$1" |
| 988 |
local value |
| 989 |
local attribute="revision" |
| 990 |
|
| 991 |
if ! value=$(jq -r ".maxRaftIndex" "$ETCD_REVISION_JSON"); then |
| 992 |
rc=$? |
| 993 |
ocf_log err "could not get $attribute, error code: $rc" |
| 994 |
return "$rc" |
| 995 |
fi |
| 996 |
|
| 997 |
case "$action" in |
| 998 |
get) |
| 999 |
echo "$value" |
| 1000 |
;; |
| 1001 |
update) |
| 1002 |
crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value" |
| 1003 |
rc=$? |
| 1004 |
if [ $rc -ne 0 ]; then |
| 1005 |
# Log the error but return success to avoid monitor failure if we can not update the attribute. |
| 1006 |
ocf_log err "could not update etcd $attribute, error code: $rc" |
| 1007 |
return $OCF_SUCCESS |
| 1008 |
fi |
| 1009 |
;; |
| 1010 |
*) |
| 1011 |
ocf_log err "unsupported $action for attribute_node_revision" |
| 1012 |
return "$OCF_ERR_GENERIC" |
| 1013 |
;; |
| 1014 |
esac |
| 1015 |
} |
| 1016 |
|
| 1017 |
attribute_node_revision_peer() |
| 1018 |
{ |
| 1019 |
local nodename |
| 1020 |
nodename=$(get_peer_node_name) |
| 1021 |
crm_attribute --query --type nodes --node "$nodename" --name "revision" | awk -F"value=" '{print $2}' |
| 1022 |
} |
| 1023 |
|
| 1024 |
# Converts a decimal number to hexadecimal format with validation |
| 1025 |
# Args: $1 - decimal number (test for non-negative integer too) |
| 1026 |
# Returns: 0 on success, OCF_ERR_GENERIC on invalid input |
| 1027 |
# Outputs: hexadecimal representation to stdout |
| 1028 |
decimal_to_hex() { |
| 1029 |
local dec=$1 |
| 1030 |
|
| 1031 |
if ! echo "$dec" | grep -q "^[1-9][0-9]*$"; then |
| 1032 |
ocf_log err "Invalid member ID format: '$dec' (expected decimal number)" |
| 1033 |
return $OCF_ERR_GENERIC |
| 1034 |
fi |
| 1035 |
|
| 1036 |
printf "%x" "$dec" |
| 1037 |
return $OCF_SUCCESS |
| 1038 |
} |
| 1039 |
|
| 1040 |
attribute_node_member_id() |
| 1041 |
{ |
| 1042 |
local action="$1" |
| 1043 |
local attribute="member_id" |
| 1044 |
|
| 1045 |
if ! container_exists; then |
| 1046 |
# we need a running container to execute etcdctl. |
| 1047 |
return 0 |
| 1048 |
fi |
| 1049 |
|
| 1050 |
case "$action" in |
| 1051 |
get) |
| 1052 |
# When we need this value at the agent startup we don't have a etcd |
| 1053 |
# container running, so we always get this value from CIB |
| 1054 |
crm_attribute --query --type nodes --node "$NODENAME" --name "$attribute" | awk -F"value=" '{print $2}' |
| 1055 |
;; |
| 1056 |
update) |
| 1057 |
local member_list_json |
| 1058 |
member_list_json=$(get_member_list_json) |
| 1059 |
ocf_log info "member list: $member_list_json" |
| 1060 |
if [ -z "$member_list_json" ] ; then |
| 1061 |
ocf_log err "could not get $attribute: could not get member list JSON" |
| 1062 |
return $OCF_ERR_GENERIC |
| 1063 |
fi |
| 1064 |
|
| 1065 |
local value value_hex rc |
| 1066 |
value=$(echo -n "$member_list_json" | jq -r ".header.member_id") |
| 1067 |
rc=$? |
| 1068 |
if [ $rc -ne 0 ]; then |
| 1069 |
# Log the error but return success to avoid monitor failure if the file is not available yet. |
| 1070 |
ocf_log err "could not get $attribute from member list JSON, error code: $rc" |
| 1071 |
return $OCF_SUCCESS |
| 1072 |
fi |
| 1073 |
|
| 1074 |
# JSON member_id is decimal, while etcdctl command needs the hex version |
| 1075 |
value_hex=$(decimal_to_hex "$value") |
| 1076 |
rc=$? |
| 1077 |
if [ $rc -ne 0 ]; then |
| 1078 |
ocf_log err "could not convert decimal member_id '$value' to hex, error code: $rc" |
| 1079 |
return $OCF_ERR_GENERIC |
| 1080 |
fi |
| 1081 |
crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --update "$value_hex" |
| 1082 |
rc=$? |
| 1083 |
if [ $rc -ne 0 ]; then |
| 1084 |
# Log the error but return success to avoid monitor failure if we can not update the attribute. |
| 1085 |
ocf_log err "could not update etcd $attribute, error code: $rc" |
| 1086 |
return $OCF_SUCCESS |
| 1087 |
fi |
| 1088 |
;; |
| 1089 |
clear) |
| 1090 |
crm_attribute --type nodes --node "$NODENAME" --name "$attribute" --delete |
| 1091 |
;; |
| 1092 |
*) |
| 1093 |
ocf_log err "unsupported $action for attribute_node_member_id" |
| 1094 |
return "$OCF_ERR_GENERIC" |
| 1095 |
;; |
| 1096 |
esac |
| 1097 |
} |
| 1098 |
|
| 1099 |
# remove an etcd member identified by its IP from the member list |
| 1100 |
remove_etcd_member_by_ip() |
| 1101 |
{ |
| 1102 |
local rc |
| 1103 |
local out |
| 1104 |
local member_ip |
| 1105 |
local endpoint_url |
| 1106 |
local member_list_json |
| 1107 |
local stale_member_id |
| 1108 |
local stale_member_id_hex |
| 1109 |
|
| 1110 |
member_ip=$1 |
| 1111 |
member_list_json=$(get_member_list_json) |
| 1112 |
rc=$? |
| 1113 |
if [ "$rc" -ne 0 ] ; then |
| 1114 |
ocf_log err "could not remove etcd member. Failed to get member list, error code: $rc" |
| 1115 |
return $OCF_ERR_GENERIC |
| 1116 |
fi |
| 1117 |
|
| 1118 |
stale_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$member_ip\")) | any).ID") |
| 1119 |
if [ -z "$stale_member_id" ]; then |
| 1120 |
ocf_log err "could not remove etcd member. Failed to find member ID" |
| 1121 |
return $OCF_ERR_GENERIC |
| 1122 |
fi |
| 1123 |
|
| 1124 |
# JSON member_id is decimal, while etcdctl command needs the hex version |
| 1125 |
if ! stale_member_id_hex=$(decimal_to_hex "$stale_member_id"); then |
| 1126 |
ocf_log err "could not remove etcd member. Failed to convert member_id '$stale_member_id' into hex format" |
| 1127 |
return $OCF_ERR_GENERIC |
| 1128 |
fi |
| 1129 |
|
| 1130 |
endpoint_url=$(ip_url $(attribute_node_ip get)) |
| 1131 |
out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" member remove "$stale_member_id_hex" 2>&1) |
| 1132 |
rc=$? |
| 1133 |
if [ "$rc" -ne 0 ] ; then |
| 1134 |
ocf_log err "could not remove etcd member. etcdctl member remove command failed, error code: $rc, output: $out" |
| 1135 |
return $OCF_ERR_GENERIC |
| 1136 |
fi |
| 1137 |
|
| 1138 |
ocf_log info "$out" |
| 1139 |
return $OCF_SUCCESS |
| 1140 |
} |
| 1141 |
|
| 1142 |
add_member_as_learner() |
| 1143 |
{ |
| 1144 |
local rc |
| 1145 |
local member_name=$1 |
| 1146 |
local member_ip=$2 |
| 1147 |
local endpoint_url=$(ip_url $(attribute_node_ip get)) |
| 1148 |
local peer_url=$(ip_url $member_ip) |
| 1149 |
local i |
| 1150 |
local max_retries |
| 1151 |
local out |
| 1152 |
|
| 1153 |
i=0 |
| 1154 |
max_retries=3 |
| 1155 |
while [ "$i" -lt "$max_retries" ]; do |
| 1156 |
i=$((i + 1)) |
| 1157 |
ocf_log info "adding $member_name ($member_ip) to the member list as learner (attempt $i of $max_retries)" |
| 1158 |
out=$(podman exec "${CONTAINER}" etcdctl --endpoints="$endpoint_url:2379" \ |
| 1159 |
member add "$member_name" \ |
| 1160 |
--peer-urls="$peer_url:2380" \ |
| 1161 |
--learner 2>&1) |
| 1162 |
rc=$? |
| 1163 |
if [ "$rc" -eq 0 ]; then |
| 1164 |
ocf_log info "$out" |
| 1165 |
attribute_learner_node update "$member_name" |
| 1166 |
return $OCF_SUCCESS |
| 1167 |
fi |
| 1168 |
if echo "$out" | grep -q "Peer URLs already exists"; then |
| 1169 |
# etcd data might have stale membership data |
| 1170 |
ocf_log warn "could not add member: Peer URLs already exists" |
| 1171 |
remove_etcd_member_by_ip "$member_ip" |
| 1172 |
else |
| 1173 |
ocf_log warn "could not add member: $out" |
| 1174 |
fi |
| 1175 |
done |
| 1176 |
|
| 1177 |
ocf_log err "could not add $member_name as learner, error code $rc, etcdctl output: $out" |
| 1178 |
return $OCF_ERR_GENERIC |
| 1179 |
} |
| 1180 |
|
| 1181 |
set_force_new_cluster() |
| 1182 |
{ |
| 1183 |
local rc |
| 1184 |
crm_attribute --lifetime reboot --node "$NODENAME" --name "force_new_cluster" --update "$NODENAME" |
| 1185 |
rc=$? |
| 1186 |
if [ $rc -ne 0 ]; then |
| 1187 |
ocf_log err "could not set force_new_cluster attribute to $NODENAME" |
| 1188 |
fi |
| 1189 |
return $rc |
| 1190 |
} |
| 1191 |
|
| 1192 |
# get_force_new_cluster returns a space-separated list of nodes that have the force_new_cluster attribute set. |
| 1193 |
# Return values: |
| 1194 |
# - Exit code 0 with non-empty output: One or more nodes have the force_new_cluster attribute set |
| 1195 |
# - Exit code 0 with empty output: No nodes have the force_new_cluster attribute set |
| 1196 |
# - Exit code 1 with empty output: Error occurred while querying the cluster nodes |
| 1197 |
get_force_new_cluster() |
| 1198 |
{ |
| 1199 |
local node nodes value |
| 1200 |
local holders="" |
| 1201 |
|
| 1202 |
if ! nodes=$(crm_node -l | awk '{print $2}'); then |
| 1203 |
ocf_log err "could not get force_new_cluster attribute, crm_node error code: $?" |
| 1204 |
return 1 |
| 1205 |
fi |
| 1206 |
if [ -z "$nodes" ]; then |
| 1207 |
ocf_log err "could not get force_new_cluster attribute, the list of nodes is empty" |
| 1208 |
return 1 |
| 1209 |
fi |
| 1210 |
|
| 1211 |
for node in $nodes; do |
| 1212 |
if ! value=$(crm_attribute --query --lifetime reboot --name "force_new_cluster" --node "$node" 2>/dev/null | awk -F'value=' '{print $2}' | tr -d "'"); then |
| 1213 |
ocf_log err "could not get force_new_cluster attribute, crm_attribut error code: $?" |
| 1214 |
return 1 |
| 1215 |
fi |
| 1216 |
if [ -n "$value" ]; then |
| 1217 |
holders="$holders$node " |
| 1218 |
fi |
| 1219 |
done |
| 1220 |
echo "$holders" |
| 1221 |
} |
| 1222 |
|
| 1223 |
|
| 1224 |
clear_force_new_cluster() |
| 1225 |
{ |
| 1226 |
# only the holder of "force_new_cluster" attribute can delete it |
| 1227 |
if ! is_force_new_cluster; then |
| 1228 |
ocf_log info "force_new_cluster unset or not owned by $NODENAME" |
| 1229 |
return $OCF_SUCCESS |
| 1230 |
fi |
| 1231 |
|
| 1232 |
if ! crm_attribute --delete --lifetime reboot --node "$NODENAME" --name "force_new_cluster"; then |
| 1233 |
ocf_log err "could not clear force_new_cluster attribute, error code: $?" |
| 1234 |
return $OCF_ERR_GENERIC |
| 1235 |
fi |
| 1236 |
|
| 1237 |
ocf_log info "$NODENAME: force_new_cluster attribute cleared" |
| 1238 |
return $OCF_SUCCESS |
| 1239 |
} |
| 1240 |
|
| 1241 |
|
| 1242 |
is_force_new_cluster() |
| 1243 |
{ |
| 1244 |
# Return 0 if 'force_new_cluster' is set on the current node, 1 otherwise. |
| 1245 |
local fnc_holders |
| 1246 |
|
| 1247 |
if ! fnc_holders=$(get_force_new_cluster); then |
| 1248 |
ocf_exit_reason "is_force_new_cluster: Failed to get force_new_cluster node holders" |
| 1249 |
exit $OCF_ERR_GENERIC |
| 1250 |
fi |
| 1251 |
|
| 1252 |
if echo "$fnc_holders" | grep -q -w "$NODENAME"; then |
| 1253 |
ocf_log debug "$NODENAME has force_new_cluster set" |
| 1254 |
return 0 |
| 1255 |
fi |
| 1256 |
|
| 1257 |
ocf_log debug "force_new_cluster attribute is not set on $NODENAME" |
| 1258 |
return 1 |
| 1259 |
} |
| 1260 |
|
| 1261 |
is_standalone() |
| 1262 |
{ |
| 1263 |
local standalone_node |
| 1264 |
|
| 1265 |
standalone_node=$(get_standalone_node) |
| 1266 |
if [ -z "$standalone_node" ]; then |
| 1267 |
ocf_log debug "no node running standalone" |
| 1268 |
return 1 |
| 1269 |
fi |
| 1270 |
|
| 1271 |
if [ "$NODENAME" = "$standalone_node" ]; then |
| 1272 |
ocf_log debug "$NODENAME is set as standalone" |
| 1273 |
return 0 |
| 1274 |
fi |
| 1275 |
ocf_log debug "$NODENAME is set as learner" |
| 1276 |
return 1 |
| 1277 |
|
| 1278 |
} |
| 1279 |
|
| 1280 |
set_standalone_node() |
| 1281 |
{ |
| 1282 |
local rc |
| 1283 |
|
| 1284 |
ocf_log info "add $NODENAME as standalone" |
| 1285 |
crm_attribute --name "standalone_node" --update "$NODENAME" |
| 1286 |
rc=$? |
| 1287 |
if [ $rc -ne 0 ]; then |
| 1288 |
ocf_log err "could not set standalone_node attribute to $NODENAME" |
| 1289 |
fi |
| 1290 |
return $rc |
| 1291 |
} |
| 1292 |
|
| 1293 |
get_standalone_node() |
| 1294 |
{ |
| 1295 |
crm_attribute --query --name "standalone_node" | awk -F"value=" '{print $2}' |
| 1296 |
} |
| 1297 |
|
| 1298 |
clear_standalone_node() |
| 1299 |
{ |
| 1300 |
crm_attribute --name "standalone_node" --delete |
| 1301 |
} |
| 1302 |
|
| 1303 |
|
| 1304 |
# Promotes an etcd learner member to a voting member |
| 1305 |
# Args: $1 - learner member ID in decimal format |
| 1306 |
# Returns: OCF_SUCCESS (even on expected promotion failures), OCF_ERR_GENERIC on conversion errors |
| 1307 |
# Note: Promotion failures are expected and logged as info (peer may not be up-to-date) |
| 1308 |
promote_learner_member() |
| 1309 |
{ |
| 1310 |
local learner_member_id=$1 |
| 1311 |
|
| 1312 |
# JSON member_id is decimal, while etcdctl command needs the hex version |
| 1313 |
if ! learner_member_id_hex=$(decimal_to_hex "$learner_member_id"); then |
| 1314 |
ocf_log err "could not convert decimal member_id '$learner_member_id' to hex, error code: $?" |
| 1315 |
return $OCF_ERR_GENERIC |
| 1316 |
fi |
| 1317 |
if ! ocf_run podman exec "${CONTAINER}" etcdctl member promote "$learner_member_id_hex" 2>&1; then |
| 1318 |
# promotion is expected to fail if the peer is not yet up-to-date |
| 1319 |
ocf_log info "could not promote member $learner_member_id_hex, error code: $?" |
| 1320 |
return $OCF_ERR_GENERIC |
| 1321 |
fi |
| 1322 |
ocf_log info "successfully promoted member '$learner_member_id_hex'" |
| 1323 |
return $OCF_SUCCESS |
| 1324 |
} |
| 1325 |
|
| 1326 |
# Reconciles etcd cluster member states |
| 1327 |
# Promotes learner members or clears standalone/learner attributes as needed |
| 1328 |
# Args: $1 - member list JSON from etcdctl |
| 1329 |
# Returns: OCF_SUCCESS on completion, OCF_ERR_GENERIC on errors |
| 1330 |
# Note: Only operates when exactly 2 started members are present |
| 1331 |
reconcile_member_state() |
| 1332 |
{ |
| 1333 |
local rc |
| 1334 |
local member_list_json="$1" |
| 1335 |
|
| 1336 |
# count only the started members, which have the ".name" JSON field |
| 1337 |
number_of_started_members=$(printf "%s" "$member_list_json" | jq -r ".members[].name | select(. != null)" | wc -l) |
| 1338 |
if [ "$number_of_started_members" -ne 2 ]; then |
| 1339 |
ocf_log info "could not clear standalone_node, nor learner_node properties: found $number_of_started_members members, need 2" |
| 1340 |
return $OCF_SUCCESS |
| 1341 |
fi |
| 1342 |
|
| 1343 |
learner_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .isLearner==true ).ID") |
| 1344 |
rc=$? |
| 1345 |
if [ $rc -ne 0 ]; then |
| 1346 |
ocf_log err "could not get isLearner field from member list, error code: $rc" |
| 1347 |
return $rc |
| 1348 |
fi |
| 1349 |
|
| 1350 |
if [ -n "$learner_member_id" ]; then |
| 1351 |
if ! promote_learner_member "$learner_member_id"; then |
| 1352 |
return $OCF_ERR_GENERIC |
| 1353 |
fi |
| 1354 |
# promotion succeded: continue to clear standalone_node and learner_node |
| 1355 |
fi |
| 1356 |
|
| 1357 |
if ! clear_standalone_node; then |
| 1358 |
ocf_log error "could not clear standalone_node attribute, error code: $?" |
| 1359 |
return $OCF_ERR_GENERIC |
| 1360 |
fi |
| 1361 |
if ! attribute_learner_node clear; then |
| 1362 |
ocf_log error "could not clear learner_node attribute, error code: $?" |
| 1363 |
return $OCF_ERR_GENERIC |
| 1364 |
fi |
| 1365 |
|
| 1366 |
return $OCF_SUCCESS |
| 1367 |
} |
| 1368 |
|
| 1369 |
attribute_learner_node() |
| 1370 |
{ |
| 1371 |
local action="$1" |
| 1372 |
local value="$2" |
| 1373 |
local attribute="learner_node" |
| 1374 |
|
| 1375 |
case "$action" in |
| 1376 |
get) |
| 1377 |
crm_attribute --query --name "$attribute" | awk -F"value=" '{print $2}' |
| 1378 |
;; |
| 1379 |
update) |
| 1380 |
if ! crm_attribute --name "$attribute" --update "$value"; then |
| 1381 |
rc="$?" |
| 1382 |
ocf_log err "could not set $attribute to $value, error code: $rc" |
| 1383 |
return "$rc" |
| 1384 |
fi |
| 1385 |
;; |
| 1386 |
clear) |
| 1387 |
crm_attribute --name "$attribute" --delete |
| 1388 |
;; |
| 1389 |
*) |
| 1390 |
ocf_log err "unsupported $action for $attribute" |
| 1391 |
return $OCF_ERR_GENERIC |
| 1392 |
;; |
| 1393 |
esac |
| 1394 |
} |
| 1395 |
|
| 1396 |
is_learner() |
| 1397 |
{ |
| 1398 |
if [ "$NODENAME" = "$(attribute_learner_node get)" ]; then |
| 1399 |
return 0 |
| 1400 |
fi |
| 1401 |
return 1 |
| 1402 |
} |
| 1403 |
|
| 1404 |
get_peer_node_name() { |
| 1405 |
crm_node -l | awk '{print $2}' | grep -v "$NODENAME" |
| 1406 |
} |
| 1407 |
|
| 1408 |
# Calculate the count of truly active resources by excluding those being stopped. |
| 1409 |
# According to Pacemaker documentation, during "Post-notification (stop) / |
| 1410 |
# Pre-notification (start)" transitions, the true active resource count should be: |
| 1411 |
# Active resources = $OCF_RESKEY_CRM_meta_notify_active_resource |
| 1412 |
# minus $OCF_RESKEY_CRM_meta_notify_stop_resource |
| 1413 |
# This handles the case where a resource appears in both the active and stop lists |
| 1414 |
# during rapid restart scenarios (e.g., process crash recovery). |
| 1415 |
get_truly_active_resources_count() { |
| 1416 |
local active_list="$OCF_RESKEY_CRM_meta_notify_active_resource" |
| 1417 |
local stop_list="$OCF_RESKEY_CRM_meta_notify_stop_resource" |
| 1418 |
local truly_active="" |
| 1419 |
|
| 1420 |
# If no active resources, return 0 |
| 1421 |
# Use word count to handle whitespace-only values |
| 1422 |
if [ "$(echo "$active_list" | wc -w)" -eq 0 ]; then |
| 1423 |
echo "0" |
| 1424 |
return |
| 1425 |
fi |
| 1426 |
|
| 1427 |
# If no resources being stopped, return count of active resources |
| 1428 |
# Use word count to handle whitespace-only values |
| 1429 |
if [ "$(echo "$stop_list" | wc -w)" -eq 0 ]; then |
| 1430 |
echo "$active_list" | wc -w |
| 1431 |
return |
| 1432 |
fi |
| 1433 |
|
| 1434 |
# Filter out resources that are being stopped from the active list |
| 1435 |
for resource in $active_list; do |
| 1436 |
local is_stopping=0 |
| 1437 |
for stop_resource in $stop_list; do |
| 1438 |
if [ "$resource" = "$stop_resource" ]; then |
| 1439 |
is_stopping=1 |
| 1440 |
break |
| 1441 |
fi |
| 1442 |
done |
| 1443 |
if [ $is_stopping -eq 0 ]; then |
| 1444 |
truly_active="$truly_active $resource" |
| 1445 |
fi |
| 1446 |
done |
| 1447 |
|
| 1448 |
# Count the truly active resources (trim leading space and count words) |
| 1449 |
echo "$truly_active" | wc -w |
| 1450 |
} |
| 1451 |
|
| 1452 |
get_all_etcd_endpoints() { |
| 1453 |
for node in $(echo "$OCF_RESKEY_node_ip_map" | sed "s/\s//g;s/;/ /g"); do |
| 1454 |
name=$(echo "$node" | cut -d: -f1) |
| 1455 |
ip=$(echo "$node" | cut -d: -f2-) # Grab everything after the first : this covers ipv4/ipv6 |
| 1456 |
ipurl="$(ip_url $ip)" |
| 1457 |
if [ -z "$name" ] || [ -z "$ip" ]; then |
| 1458 |
ocf_exit_reason "name or ip missing for 1 or more nodes" |
| 1459 |
exit $OCF_ERR_CONFIGURED |
| 1460 |
fi |
| 1461 |
|
| 1462 |
[ -z "$ALL_ETCD_ENDPOINTS" ] && ALL_ETCD_ENDPOINTS="$ipurl:2379" || ALL_ETCD_ENDPOINTS="$ALL_ETCD_ENDPOINTS,$ipurl:2379" |
| 1463 |
done |
| 1464 |
echo "$ALL_ETCD_ENDPOINTS" |
| 1465 |
} |
| 1466 |
|
| 1467 |
get_endpoint_status_json() |
| 1468 |
{ |
| 1469 |
# Get the status of all endpoints |
| 1470 |
local all_etcd_endpoints |
| 1471 |
|
| 1472 |
all_etcd_endpoints=$(get_all_etcd_endpoints) |
| 1473 |
podman exec "${CONTAINER}" etcdctl endpoint status --command-timeout="$MONITOR_ETCDCTL_TIMEOUT" --endpoints="$all_etcd_endpoints" -w json |
| 1474 |
} |
| 1475 |
|
| 1476 |
get_member_list_json() { |
| 1477 |
# Get the list of members visible to the current node |
| 1478 |
local this_node_endpoint |
| 1479 |
|
| 1480 |
this_node_endpoint="$(ip_url $(attribute_node_ip get)):2379" |
| 1481 |
podman exec "${CONTAINER}" etcdctl member list --command-timeout="$MONITOR_ETCDCTL_TIMEOUT" --endpoints="$this_node_endpoint" -w json |
| 1482 |
} |
| 1483 |
|
| 1484 |
detect_cluster_leadership_loss() |
| 1485 |
{ |
| 1486 |
endpoint_status_json=$(get_endpoint_status_json) |
| 1487 |
ocf_log info "endpoint status: $endpoint_status_json" |
| 1488 |
|
| 1489 |
count_endpoints=$(printf "%s" "$endpoint_status_json" | jq -r ".[].Endpoint" | wc -l) |
| 1490 |
if [ "$count_endpoints" -eq 1 ]; then |
| 1491 |
ocf_log info "one endpoint only: checking status errors" |
| 1492 |
endpoint_status_errors=$(printf "%s" "$endpoint_status_json" | jq -r ".[0].Status.errors") |
| 1493 |
if echo "$endpoint_status_errors" | grep -q "no leader"; then |
| 1494 |
set_force_new_cluster |
| 1495 |
set_standalone_node |
| 1496 |
ocf_exit_reason "$NODENAME must force a new cluster" |
| 1497 |
return $OCF_ERR_GENERIC |
| 1498 |
fi |
| 1499 |
if [ "$endpoint_status_errors" != "null" ]; then |
| 1500 |
ocf_log err "unmanaged endpoint status error: $endpoint_status_errors" |
| 1501 |
fi |
| 1502 |
fi |
| 1503 |
|
| 1504 |
return $OCF_SUCCESS |
| 1505 |
} |
| 1506 |
|
| 1507 |
|
| 1508 |
# Manages etcd peer membership by detecting and handling missing or rejoining peers |
| 1509 |
# Adds missing peers as learners and reconciles member states when peers rejoin |
| 1510 |
# Args: $1 - member list JSON from etcdctl |
| 1511 |
# Returns: OCF_SUCCESS on completion, OCF_ERR_GENERIC on errors |
| 1512 |
# Note: Iterates through all peer nodes to ensure proper cluster membership |
| 1513 |
manage_peer_membership() |
| 1514 |
{ |
| 1515 |
local member_list_json="$1" |
| 1516 |
local peer_ip_map_entry |
| 1517 |
local peer_member_name |
| 1518 |
local peer_member_ip |
| 1519 |
local peer_member_id |
| 1520 |
|
| 1521 |
# Get peer node name and IP |
| 1522 |
peer_ip_map_entry=$(echo "$OCF_RESKEY_node_ip_map" | tr ';' '\n' | grep -vF "$NODENAME") |
| 1523 |
if [ -z "$peer_ip_map_entry" ]; then |
| 1524 |
ocf_exit_reason "manage_peer_membership: could not parse node_ip_map: '$OCF_RESKEY_node_ip_map'" |
| 1525 |
exit $OCF_ERR_CONFIGURED |
| 1526 |
fi |
| 1527 |
peer_member_name=$(echo "$peer_ip_map_entry" | cut -d: -f1) |
| 1528 |
peer_member_ip=$(echo "$peer_ip_map_entry" | cut -d: -f2-) |
| 1529 |
|
| 1530 |
# Parsing the member list's json output to find a "learner" member. |
| 1531 |
# Example of .members[] instance fields in member list json format: |
| 1532 |
# { |
| 1533 |
# "ID": <member ID>, |
| 1534 |
# "name": "<node hostname>", |
| 1535 |
# "peerURLs": [ |
| 1536 |
# "https://<node IP>:2380" |
| 1537 |
# ], |
| 1538 |
# "clientURLs": [ |
| 1539 |
# "https://<node IP>:2379" |
| 1540 |
# ] |
| 1541 |
# } |
| 1542 |
# NOTE: voting members have a "name" field but no "isLearner" field, |
| 1543 |
# while learner members have "isLearner": true (boolean) but no "name" field, so we search for peerURLs matching. |
| 1544 |
peer_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .peerURLs | map(test(\"$peer_member_ip\")) | any).ID") |
| 1545 |
if [ -z "$peer_member_id" ]; then |
| 1546 |
ocf_log info "$peer_member_name is not in the members list" |
| 1547 |
add_member_as_learner "$peer_member_name" "$peer_member_ip" |
| 1548 |
set_standalone_node |
| 1549 |
return |
| 1550 |
fi |
| 1551 |
|
| 1552 |
# Ensure learner_node attribute is always set when we have a learner member |
| 1553 |
local learner_member_id=$(printf "%s" "$member_list_json" | jq -r ".members[] | select( .isLearner==true ).ID") |
| 1554 |
local current_learner_node=$(attribute_learner_node get) |
| 1555 |
if [ -n "$learner_member_id" ] && [ -z "$current_learner_node" ]; then |
| 1556 |
ocf_log debug "$peer_member_name found as learner in member list, but learner_node attribute was not set. Updating" |
| 1557 |
attribute_learner_node update "$peer_member_name" |
| 1558 |
return |
| 1559 |
fi |
| 1560 |
|
| 1561 |
ocf_log debug "$peer_member_name is in the members list by IP: $peer_member_ip" |
| 1562 |
# Errors from reconcile_member_state are logged internally. Ignoring them here prevents stopping a healthy voter agent; critical local failures are caught by detect_cluster_leadership_loss. |
| 1563 |
reconcile_member_state "$member_list_json" |
| 1564 |
} |
| 1565 |
|
| 1566 |
check_peer() |
| 1567 |
{ |
| 1568 |
# Check peers endpoint status and locally accessible member list |
| 1569 |
local member_list_json |
| 1570 |
|
| 1571 |
# we need a running container to execute etcdctl. |
| 1572 |
if ! container_exists; then |
| 1573 |
return $OCF_SUCCESS |
| 1574 |
fi |
| 1575 |
|
| 1576 |
if ! member_list_json=$(get_member_list_json); then |
| 1577 |
ocf_log info "podman failed to get member list" |
| 1578 |
detect_cluster_leadership_loss |
| 1579 |
return $? |
| 1580 |
fi |
| 1581 |
|
| 1582 |
manage_peer_membership "$member_list_json" |
| 1583 |
return $OCF_SUCCESS |
| 1584 |
} |
| 1585 |
|
| 1586 |
podman_simple_status() |
| 1587 |
{ |
| 1588 |
local rc |
| 1589 |
|
| 1590 |
# simple status is implemented via podman exec |
| 1591 |
# everything besides success is considered "not running" |
| 1592 |
monitor_cmd_exec |
| 1593 |
rc=$? |
| 1594 |
if [ $rc -ne $OCF_SUCCESS ]; then |
| 1595 |
rc=$OCF_NOT_RUNNING; |
| 1596 |
fi |
| 1597 |
return $rc |
| 1598 |
} |
| 1599 |
|
| 1600 |
# get_time_since_last_heartbeat returns the time in seconds since the heartbeat file was last updated. |
| 1601 |
# |
| 1602 |
# Returns: time in seconds since last heartbeat, or empty string if file doesn't exist |
| 1603 |
get_time_since_last_heartbeat() |
| 1604 |
{ |
| 1605 |
local last_heartbeat |
| 1606 |
|
| 1607 |
if [ ! -f "$CONTAINER_HEARTBEAT_FILE" ]; then |
| 1608 |
return |
| 1609 |
fi |
| 1610 |
|
| 1611 |
last_heartbeat=$(cat "$CONTAINER_HEARTBEAT_FILE") |
| 1612 |
echo $(($(date +%s) - last_heartbeat)) |
| 1613 |
} |
| 1614 |
|
| 1615 |
# container_health_check performs comprehensive health monitoring for the container. |
| 1616 |
# This function allows coordinated failure handling where the agent waits for |
| 1617 |
# peer-initiated cluster recovery in case of container failure. |
| 1618 |
# |
| 1619 |
# Uses a state file to track container state: |
| 1620 |
# - Container running: Update state file with current epoch, return "healthy" |
| 1621 |
# - Container not running + no state file: Return "not-running" (never checked before) |
| 1622 |
# - Container not running + state file: Failure detected, check force_new_cluster |
| 1623 |
# - If force_new_cluster set: Return "failed-restart-now" |
| 1624 |
# - Otherwise: Return "failed-wait-for-peer" |
| 1625 |
# |
| 1626 |
# Returns: healthy, not-running, failed-restart-now, failed-wait-for-peer |
| 1627 |
|
| 1628 |
container_health_check() |
| 1629 |
{ |
| 1630 |
local rc |
| 1631 |
|
| 1632 |
# We rely on running podman exec to monitor the container |
| 1633 |
# state because that command seems to be less prone to |
| 1634 |
# performance issue under IO load. |
| 1635 |
# |
| 1636 |
# For probes to work, we expect cmd_exec to be able to report |
| 1637 |
# when a container is not running. Here, we're not interested |
| 1638 |
# in distinguishing whether it's stopped or non existing |
| 1639 |
# (there's function container_exists for that) |
| 1640 |
# For monitor, however, we still need to know if it has stopped |
| 1641 |
# recently (i.e. a failure), or not (fresh start) |
| 1642 |
monitor_cmd_exec |
| 1643 |
rc=$? |
| 1644 |
if [ "$rc" -eq 0 ]; then |
| 1645 |
# Container is running - update state file with current epoch |
| 1646 |
local current_epoch |
| 1647 |
current_epoch=$(date +%s) |
| 1648 |
if ! echo "$current_epoch" > "$CONTAINER_HEARTBEAT_FILE"; then |
| 1649 |
ocf_log warn "Failed to update container heartbeat file, error code: $?" |
| 1650 |
# wait for peer to detect any real issue with the etcd cluster or wait for the |
| 1651 |
# next monitor interval |
| 1652 |
echo "failed-wait-for-peer" |
| 1653 |
return |
| 1654 |
fi |
| 1655 |
echo "healthy" |
| 1656 |
return |
| 1657 |
fi |
| 1658 |
|
| 1659 |
# Check if state file exists (was container running on last check?) |
| 1660 |
if [ ! -f "$CONTAINER_HEARTBEAT_FILE" ]; then |
| 1661 |
# No state file - container was never checked before |
| 1662 |
ocf_log debug "Container ${CONTAINER} has no previous state" |
| 1663 |
echo "not-running" |
| 1664 |
# NOTE: this is where the probe is expected to exit, keeping the logic |
| 1665 |
# quick and less prone to performance issue under IO load. |
| 1666 |
return |
| 1667 |
fi |
| 1668 |
|
| 1669 |
# Could not execute monitor check command and state file exists - the container failed, check recovery status in this lifecycle |
| 1670 |
local time_since_heartbeat |
| 1671 |
time_since_heartbeat=$(get_time_since_last_heartbeat) |
| 1672 |
ocf_log err "Container ${CONTAINER} failed (last healthy: ${time_since_heartbeat}s ago, error code: $rc)" |
| 1673 |
|
| 1674 |
# Check if peer has set force_new_cluster for recovery |
| 1675 |
local fnc_holders |
| 1676 |
if ! fnc_holders=$(get_force_new_cluster); then |
| 1677 |
ocf_log err "Could not detect peer-initiated recovery. Checking again in the next monitor cycle" |
| 1678 |
echo "failed-wait-for-peer" |
| 1679 |
return |
| 1680 |
fi |
| 1681 |
|
| 1682 |
local fnc_holder_count=$(echo "$fnc_holders" | wc -w) |
| 1683 |
if [ "$fnc_holder_count" -gt 0 ]; then |
| 1684 |
ocf_log info "force_new_cluster detected (set by: $fnc_holders), triggering restart" |
| 1685 |
echo "failed-restart-now" |
| 1686 |
return |
| 1687 |
fi |
| 1688 |
|
| 1689 |
echo "failed-wait-for-peer" |
| 1690 |
} |
| 1691 |
|
| 1692 |
podman_monitor() |
| 1693 |
{ |
| 1694 |
local container_health_state |
| 1695 |
|
| 1696 |
container_health_state=$(container_health_check) |
| 1697 |
case "$container_health_state" in |
| 1698 |
healthy) |
| 1699 |
# Continue with normal monitoring |
| 1700 |
;; |
| 1701 |
not-running) |
| 1702 |
return $OCF_NOT_RUNNING |
| 1703 |
;; |
| 1704 |
failed-restart-now) |
| 1705 |
return $OCF_ERR_GENERIC |
| 1706 |
;; |
| 1707 |
failed-wait-for-peer) |
| 1708 |
# Continue running, waiting for peer recovery |
| 1709 |
return $OCF_SUCCESS |
| 1710 |
;; |
| 1711 |
*) |
| 1712 |
ocf_log err "Unknown health state: $container_health_state" |
| 1713 |
return $OCF_ERR_GENERIC |
| 1714 |
;; |
| 1715 |
esac |
| 1716 |
|
| 1717 |
# Check if certificate files have changed, if they have, etcd needs to be restarted |
| 1718 |
if ! etcd_certificates_hash_manager "check"; then |
| 1719 |
return $OCF_ERR_GENERIC |
| 1720 |
fi |
| 1721 |
|
| 1722 |
if is_learner; then |
| 1723 |
ocf_log info "$NODENAME is learner. Cannot get member id" |
| 1724 |
return "$OCF_SUCCESS" |
| 1725 |
fi |
| 1726 |
# Failing to cache data and check member list should not cause the |
| 1727 |
# monitor operation to fail. |
| 1728 |
# TODO: move this inside check_peers where we already query member list json |
| 1729 |
attribute_node_member_id update |
| 1730 |
if ! check_peer; then |
| 1731 |
return $OCF_ERR_GENERIC |
| 1732 |
fi |
| 1733 |
|
| 1734 |
# node revision comes from the disk, so if it is not available is a fatal failure |
| 1735 |
attribute_node_revision update |
| 1736 |
return $? |
| 1737 |
} |
| 1738 |
|
| 1739 |
podman_create_mounts() { |
| 1740 |
oldIFS="$IFS" |
| 1741 |
IFS="," |
| 1742 |
for directory in $OCF_RESKEY_mount_points; do |
| 1743 |
mkdir -p "$directory" |
| 1744 |
done |
| 1745 |
IFS="$oldIFS" |
| 1746 |
} |
| 1747 |
|
| 1748 |
podman_container_id() |
| 1749 |
{ |
| 1750 |
# Retrieve the container ID by doing a "podman ps" rather than |
| 1751 |
# a "podman inspect", because the latter has performance issues |
| 1752 |
# under IO load. |
| 1753 |
# We could have run "podman start $CONTAINER" to get the ID back |
| 1754 |
# but if the container is stopped, the command will return a |
| 1755 |
# name instead of a container ID. This would break us. |
| 1756 |
podman ps --no-trunc --format '{{.ID}} {{.Names}}' | grep -F -w -m1 "$CONTAINER" | cut -d' ' -f1 |
| 1757 |
} |
| 1758 |
|
| 1759 |
|
| 1760 |
create_transient_drop_in_dependency() |
| 1761 |
{ |
| 1762 |
local cid=$1 |
| 1763 |
local rc=$OCF_SUCCESS |
| 1764 |
|
| 1765 |
if [ -z "$cid" ]; then |
| 1766 |
ocf_exit_reason "Container ID not found for \"$CONTAINER\". Not creating drop-in dependency" |
| 1767 |
return $OCF_ERR_GENERIC |
| 1768 |
fi |
| 1769 |
|
| 1770 |
ocf_log info "Creating drop-in dependency for \"$CONTAINER\" ($cid)" |
| 1771 |
for scope in "libpod-$cid.scope.d" "libpod-conmon-$cid.scope.d"; do |
| 1772 |
if [ $rc -eq $OCF_SUCCESS ] && [ ! -d /run/systemd/transient/"$scope" ]; then |
| 1773 |
mkdir -p /run/systemd/transient/"$scope" && \ |
| 1774 |
printf "[Unit]\nBefore=pacemaker.service" > /run/systemd/transient/"$scope"/dep.conf && \ |
| 1775 |
chmod ago+r /run/systemd/transient/"$scope" /run/systemd/transient/"$scope"/dep.conf |
| 1776 |
rc=$? |
| 1777 |
fi |
| 1778 |
done |
| 1779 |
|
| 1780 |
if [ $rc -ne $OCF_SUCCESS ]; then |
| 1781 |
ocf_log err "Could not create drop-in dependency for \"$CONTAINER\" ($cid)" |
| 1782 |
else |
| 1783 |
systemctl daemon-reload |
| 1784 |
rc=$? |
| 1785 |
if [ $rc -ne $OCF_SUCCESS ]; then |
| 1786 |
ocf_log err "Could not refresh service definition after creating drop-in for \"$CONTAINER\"" |
| 1787 |
fi |
| 1788 |
fi |
| 1789 |
|
| 1790 |
return $rc |
| 1791 |
} |
| 1792 |
|
| 1793 |
|
| 1794 |
run_new_container() |
| 1795 |
{ |
| 1796 |
local opts=$1 |
| 1797 |
local image=$2 |
| 1798 |
local cmd=$3 |
| 1799 |
local rc |
| 1800 |
|
| 1801 |
ocf_log info "running container $CONTAINER for the first time" |
| 1802 |
out=$(podman run $opts $image $cmd 2>&1) |
| 1803 |
rc=$? |
| 1804 |
|
| 1805 |
if [ -n "$out" ]; then |
| 1806 |
out="$(echo "$out" | tr -s ' \t\r\n' ' ')" |
| 1807 |
if [ $rc -eq 0 ]; then |
| 1808 |
ocf_log info "$out" |
| 1809 |
else |
| 1810 |
ocf_log err "$out" |
| 1811 |
fi |
| 1812 |
fi |
| 1813 |
|
| 1814 |
if [ $rc -eq 125 ]; then |
| 1815 |
# If an internal podman error occurred, it might be because |
| 1816 |
# the internal storage layer still references an old container |
| 1817 |
# with the same name, even though podman itself thinks there |
| 1818 |
# is no such container. If so, purge the storage layer to try |
| 1819 |
# to clean the corruption and try again. |
| 1820 |
if echo "$out" | grep -q "unknown.*flag"; then |
| 1821 |
ocf_exit_reason "$out" |
| 1822 |
return $rc |
| 1823 |
fi |
| 1824 |
|
| 1825 |
ocf_log warn "Internal podman error while creating new container $CONTAINER. Retrying." |
| 1826 |
ocf_run podman rm --storage "$CONTAINER" |
| 1827 |
ocf_run podman run $opts $image $cmd |
| 1828 |
rc=$? |
| 1829 |
elif [ $rc -eq 127 ]; then |
| 1830 |
# rhbz#1972209: podman 3.0.x seems to be hit by a race |
| 1831 |
# where the cgroup is not yet set up properly when the OCI |
| 1832 |
# runtime configures the container. If that happens, recreate |
| 1833 |
# the container as long as we get the same error code or |
| 1834 |
# until start timeout preempts us. |
| 1835 |
while [ $rc -eq 127 ] && (echo "$out" | grep -q "cgroup.*scope not found") ; do |
| 1836 |
ocf_log warn "Internal podman error while assigning cgroup. Retrying." |
| 1837 |
# Arbitrary sleep to prevent consuming all CPU while looping |
| 1838 |
sleep 1 |
| 1839 |
podman rm -f "$CONTAINER" |
| 1840 |
out=$(podman run $opts $image $cmd 2>&1) |
| 1841 |
rc=$? |
| 1842 |
done |
| 1843 |
# Log the created container ID if it succeeded |
| 1844 |
if [ $rc -eq 0 ]; then |
| 1845 |
ocf_log info "$out" |
| 1846 |
fi |
| 1847 |
fi |
| 1848 |
|
| 1849 |
return $rc |
| 1850 |
} |
| 1851 |
|
| 1852 |
compare_revision() |
| 1853 |
{ |
| 1854 |
# Compare local revision (from disk) against peer revision (from CIB). |
| 1855 |
# returns "older", "equal" or "newer" |
| 1856 |
local revision |
| 1857 |
local peer_node_name |
| 1858 |
local peer_revision |
| 1859 |
|
| 1860 |
revision=$(attribute_node_revision get) |
| 1861 |
peer_revision=$(attribute_node_revision_peer) |
| 1862 |
|
| 1863 |
if [ "$revision" = "" ] || [ "$revision" = "null" ] || [ "$peer_revision" = "" ] || [ "$peer_revision" = "null" ]; then |
| 1864 |
ocf_log err "could not compare revisions: '$NODENAME' local revision='$revision', peer revision='$peer_revision'" |
| 1865 |
return "$OCF_ERR_GENERIC" |
| 1866 |
fi |
| 1867 |
|
| 1868 |
if [ "$revision" -gt "$peer_revision" ]; then |
| 1869 |
ocf_log info "$NODENAME revision: '$revision' is newer than peer revision: '$peer_revision'" |
| 1870 |
echo "newer" |
| 1871 |
elif [ "$revision" -eq "$peer_revision" ]; then |
| 1872 |
ocf_log info "$NODENAME revision: '$revision' is equal to peer revision: '$peer_revision'" |
| 1873 |
echo "equal" |
| 1874 |
else |
| 1875 |
ocf_log info "$NODENAME revision: '$revision' is older than peer revision: '$peer_revision'" |
| 1876 |
echo "older" |
| 1877 |
fi |
| 1878 |
return "$OCF_SUCCESS" |
| 1879 |
} |
| 1880 |
|
| 1881 |
ensure_pod_manifest_exists() |
| 1882 |
{ |
| 1883 |
local wait_timeout_sec=$((10 * 60)) |
| 1884 |
local poll_interval_sec=5 |
| 1885 |
local poll_retries=$((wait_timeout_sec/poll_interval_sec)) |
| 1886 |
|
| 1887 |
for try in $(seq "$poll_retries"); do |
| 1888 |
if [ -f "$OCF_RESKEY_pod_manifest" ]; then |
| 1889 |
ocf_log info "pod manifest ($OCF_RESKEY_pod_manifest) found" |
| 1890 |
break |
| 1891 |
fi |
| 1892 |
ocf_log debug "pod manifest ($OCF_RESKEY_pod_manifest) does not exist yet: retry in $poll_interval_sec seconds." |
| 1893 |
sleep "$poll_interval_sec" |
| 1894 |
done |
| 1895 |
|
| 1896 |
if [ ! -f "$OCF_RESKEY_pod_manifest" ]; then |
| 1897 |
ocf_log err "pod manifest ($OCF_RESKEY_pod_manifest) still missing after $wait_timeout_sec seconds." |
| 1898 |
return "$OCF_ERR_CONFIGURED" |
| 1899 |
fi |
| 1900 |
|
| 1901 |
return "$OCF_SUCCESS" |
| 1902 |
} |
| 1903 |
|
| 1904 |
filter_pod_manifest() { |
| 1905 |
# Remove pod-version related fields from POD manifest |
| 1906 |
local pod_manifest="$1" |
| 1907 |
local temporary_file |
| 1908 |
local jq_filter='del(.metadata.labels.revision) | .spec.containers[] |= ( .env |= map(select( .name != "ETCD_STATIC_POD_VERSION" ))) | .spec.volumes |= map( select( .name != "resource-dir" ))' |
| 1909 |
|
| 1910 |
if ! temporary_file=$(mktemp); then |
| 1911 |
ocf_log err "could not create temporary file for '$pod_manifest', error code: $?" |
| 1912 |
return $OCF_ERR_GENERIC |
| 1913 |
fi |
| 1914 |
if ! jq "$jq_filter" "$pod_manifest" > "$temporary_file"; then |
| 1915 |
ocf_log err "could not remove pod version related data from '$pod_manifest', error code: $?" |
| 1916 |
return $OCF_ERR_GENERIC |
| 1917 |
fi |
| 1918 |
echo "$temporary_file" |
| 1919 |
} |
| 1920 |
|
| 1921 |
can_reuse_container() { |
| 1922 |
# Decide whether to reuse the existing container or create a new one based on etcd pod manifest changes. |
| 1923 |
# NOTE: explicitly ignore POD version and POD version related data, as the content might be the same even if the revision number has changed. |
| 1924 |
local cp_rc |
| 1925 |
local diff_rc |
| 1926 |
local filtered_original_pod_manifest |
| 1927 |
local filtered_copy_pod_manifest |
| 1928 |
|
| 1929 |
|
| 1930 |
# If the container does not exist it cannot be reused |
| 1931 |
if ! container_exists; then |
| 1932 |
OCF_RESKEY_reuse=0 |
| 1933 |
return "$OCF_SUCCESS" |
| 1934 |
fi |
| 1935 |
|
| 1936 |
# If the manifest copy doesn't exist, we need a new container. |
| 1937 |
if [ ! -f "$POD_MANIFEST_COPY" ]; then |
| 1938 |
ocf_log info "a working copy of $OCF_RESKEY_pod_manifest was not found. A new etcd container will be created." |
| 1939 |
OCF_RESKEY_reuse=0 |
| 1940 |
return "$OCF_SUCCESS" |
| 1941 |
fi |
| 1942 |
if ! filtered_original_pod_manifest=$(filter_pod_manifest "$OCF_RESKEY_pod_manifest"); then |
| 1943 |
return $OCF_ERR_GENERIC |
| 1944 |
fi |
| 1945 |
if ! filtered_copy_pod_manifest=$(filter_pod_manifest "$POD_MANIFEST_COPY"); then |
| 1946 |
return $OCF_ERR_GENERIC |
| 1947 |
fi |
| 1948 |
|
| 1949 |
ocf_log info "comparing $OCF_RESKEY_pod_manifest with local copy $POD_MANIFEST_COPY" |
| 1950 |
ocf_run diff -s "$filtered_original_pod_manifest" "$filtered_copy_pod_manifest" |
| 1951 |
diff_rc="$?" |
| 1952 |
# clean up temporary files |
| 1953 |
rm -f "$filtered_original_pod_manifest" "$filtered_copy_pod_manifest" |
| 1954 |
case "$diff_rc" in |
| 1955 |
0) |
| 1956 |
ocf_log info "Reusing the existing etcd container" |
| 1957 |
OCF_RESKEY_reuse=1 |
| 1958 |
;; |
| 1959 |
1) |
| 1960 |
ocf_log info "Etcd pod manifest changes detected: creating a new etcd container to apply the changes" |
| 1961 |
if ! ocf_run cp -p "$OCF_RESKEY_pod_manifest" "$POD_MANIFEST_COPY"; then |
| 1962 |
cp_rc="$?" |
| 1963 |
ocf_log err "Could not create a working copy of $OCF_RESKEY_pod_manifest, rc: $cp_rc" |
| 1964 |
return "$OCF_ERR_GENERIC" |
| 1965 |
fi |
| 1966 |
ocf_log info "A working copy of $OCF_RESKEY_pod_manifest was created" |
| 1967 |
OCF_RESKEY_reuse=0 |
| 1968 |
;; |
| 1969 |
*) |
| 1970 |
ocf_log err "Could not check if etcd pod manifest has changed, diff rc: $diff_rc" |
| 1971 |
return "$OCF_ERR_GENERIC" |
| 1972 |
;; |
| 1973 |
esac |
| 1974 |
|
| 1975 |
return "$OCF_SUCCESS" |
| 1976 |
} |
| 1977 |
|
| 1978 |
ensure_pod_manifest_copy_exists() { |
| 1979 |
local cp_rc |
| 1980 |
|
| 1981 |
if [ -f "$POD_MANIFEST_COPY" ]; then |
| 1982 |
return "$OCF_SUCCESS" |
| 1983 |
fi |
| 1984 |
|
| 1985 |
# If the manifest copy doesn't exist, create it and ensure a new container. |
| 1986 |
if ! ocf_run cp -p "$OCF_RESKEY_pod_manifest" "$POD_MANIFEST_COPY"; then |
| 1987 |
cp_rc="$?" |
| 1988 |
ocf_log err "Could not create a working copy of $OCF_RESKEY_pod_manifest, rc: $cp_rc" |
| 1989 |
return "$OCF_ERR_GENERIC" |
| 1990 |
fi |
| 1991 |
|
| 1992 |
ocf_log info "a new working copy of $OCF_RESKEY_pod_manifest was created" |
| 1993 |
|
| 1994 |
return "$OCF_SUCCESS" |
| 1995 |
} |
| 1996 |
|
| 1997 |
podman_start() |
| 1998 |
{ |
| 1999 |
local cid |
| 2000 |
local rc |
| 2001 |
local etcd_pod_wait_timeout_sec=$((10 * 60)) |
| 2002 |
local etcd_pod_poll_interval_sec=10 |
| 2003 |
local etcd_pod_poll_retries=$((etcd_pod_wait_timeout_sec/etcd_pod_poll_interval_sec)) |
| 2004 |
local pod_was_running=false |
| 2005 |
|
| 2006 |
ocf_log notice "podman-etcd start" |
| 2007 |
|
| 2008 |
# Clear container health check state file |
| 2009 |
if ! rm -f "$CONTAINER_HEARTBEAT_FILE"; then |
| 2010 |
ocf_log err "could not delete container health check state file" |
| 2011 |
fi |
| 2012 |
|
| 2013 |
attribute_node_ip update |
| 2014 |
attribute_node_cluster_id update |
| 2015 |
attribute_node_revision update |
| 2016 |
|
| 2017 |
# ensure the etcd pod is not running before starting the container |
| 2018 |
ocf_log info "ensure etcd pod is not running (retries: $etcd_pod_poll_retries, interval: $etcd_pod_poll_interval_sec)" |
| 2019 |
for try in $(seq $etcd_pod_poll_retries); do |
| 2020 |
if ! etcd_pod_container_exists; then |
| 2021 |
break |
| 2022 |
fi |
| 2023 |
ocf_log info "etcd pod running: retry in $etcd_pod_poll_interval_sec seconds." |
| 2024 |
pod_was_running=true |
| 2025 |
sleep $etcd_pod_poll_interval_sec |
| 2026 |
done |
| 2027 |
if etcd_pod_container_exists; then |
| 2028 |
ocf_exit_reason "etcd pod is still running after $etcd_pod_wait_timeout_sec seconds." |
| 2029 |
return $OCF_ERR_GENERIC |
| 2030 |
fi |
| 2031 |
|
| 2032 |
# Update the certificate hash after the container has started successfully |
| 2033 |
# this is to ensure that the certificate hash is updated after a restart is initiated |
| 2034 |
# by a cert rotation event from the monitor command. |
| 2035 |
if ! etcd_certificates_hash_manager "update"; then |
| 2036 |
ocf_exit_reason "etcd certificate hash manager failed to update the certificate hash" |
| 2037 |
return $OCF_ERR_GENERIC |
| 2038 |
fi |
| 2039 |
|
| 2040 |
# check if the container has already started |
| 2041 |
podman_simple_status |
| 2042 |
if [ $? -eq $OCF_SUCCESS ]; then |
| 2043 |
ocf_log info "the '$CONTAINER' has already started. Nothing to do" |
| 2044 |
return "$OCF_SUCCESS" |
| 2045 |
fi |
| 2046 |
|
| 2047 |
if ! ensure_pod_manifest_exists; then |
| 2048 |
ocf_exit_reason "could not find etcd pod manifest ($OCF_RESKEY_pod_manifest)" |
| 2049 |
return "$OCF_ERR_GENERIC" |
| 2050 |
fi |
| 2051 |
|
| 2052 |
if ocf_is_true "$pod_was_running"; then |
| 2053 |
ocf_log info "static pod was running: start normally" |
| 2054 |
else |
| 2055 |
local fnc_holders |
| 2056 |
if ! fnc_holders=$(get_force_new_cluster); then |
| 2057 |
ocf_exit_reason "Failed to get force_new_cluster node holders" |
| 2058 |
return "$OCF_ERR_GENERIC" |
| 2059 |
fi |
| 2060 |
|
| 2061 |
local fnc_holder_count |
| 2062 |
fnc_holder_count=$(echo "$fnc_holders" | wc -w) |
| 2063 |
if [ "$fnc_holder_count" -gt 1 ]; then |
| 2064 |
ocf_exit_reason "force_new_cluster attribute is set on multiple nodes ($fnc_holders)" |
| 2065 |
return "$OCF_ERR_CONFIGURED" |
| 2066 |
fi |
| 2067 |
|
| 2068 |
if [ "$fnc_holder_count" -eq 1 ]; then |
| 2069 |
if echo "$fnc_holders" | grep -q -w "$NODENAME"; then |
| 2070 |
# Attribute is set on the local node. |
| 2071 |
ocf_log notice "$NODENAME marked to force-new-cluster" |
| 2072 |
JOIN_AS_LEARNER=false |
| 2073 |
else |
| 2074 |
# Attribute is set on a peer node. |
| 2075 |
ocf_log info "$NODENAME shall join as learner because force_new_cluster is set on peer $fnc_holders" |
| 2076 |
JOIN_AS_LEARNER=true |
| 2077 |
fi |
| 2078 |
else |
| 2079 |
ocf_log info "no node is marked to force-new-cluster" |
| 2080 |
# When the local agent starts, we can infer the cluster state by counting |
| 2081 |
# how many agents are starting or already active: |
| 2082 |
# - 1 active agent: it's the peer (we are just starting) |
| 2083 |
# - 0 active agents, 1 starting: we are starting; the peer is not starting |
| 2084 |
# - 0 active agents, 2 starting: both agents are starting simultaneously |
| 2085 |
local active_resources_count |
| 2086 |
active_resources_count=$(get_truly_active_resources_count) |
| 2087 |
ocf_log info "found '$active_resources_count' active etcd resources (active: '$OCF_RESKEY_CRM_meta_notify_active_resource', stop: '$OCF_RESKEY_CRM_meta_notify_stop_resource')" |
| 2088 |
case "$active_resources_count" in |
| 2089 |
1) |
| 2090 |
# is_standalone may return true here due to a restart: in the previous run, |
| 2091 |
# this agent was the sole voter and the peer had not yet joined the member |
| 2092 |
# list (learner_node unset). Since standalone_node was not cleared before |
| 2093 |
# the restart, start normally to recover the previous cluster state. |
| 2094 |
if is_standalone; then |
| 2095 |
ocf_log info "peer active but not a voter: start normally to recover" |
| 2096 |
elif [ "$(attribute_learner_node get)" = "$(get_peer_node_name)" ]; then |
| 2097 |
ocf_log info "peer active but in learner mode: start normally to recover" |
| 2098 |
else |
| 2099 |
# If (A) we must join the peer's new cluster |
| 2100 |
ocf_log info "peer is active standalone: joining as learner" |
| 2101 |
JOIN_AS_LEARNER=true |
| 2102 |
fi |
| 2103 |
;; |
| 2104 |
0) |
| 2105 |
# No active resources: clear any stale learner_node attribute from previous failed session |
| 2106 |
ocf_log debug "clearing stale learner_node attribute (safe when active_resources_count=0)" |
| 2107 |
attribute_learner_node clear |
| 2108 |
# count how many agents are starting now |
| 2109 |
local start_resources_count |
| 2110 |
start_resources_count=$(echo "$OCF_RESKEY_CRM_meta_notify_start_resource" | wc -w) |
| 2111 |
ocf_log info "found '$start_resources_count' starting etcd resources (meta notify environment variable: '$OCF_RESKEY_CRM_meta_notify_start_resource')" |
| 2112 |
|
| 2113 |
# we need to compare the revisions in any of the following branches |
| 2114 |
# so call the function only once here |
| 2115 |
if ! revision_compare_result=$(compare_revision); then |
| 2116 |
ocf_log err "could not compare revisions, error code: $?" |
| 2117 |
return "$OCF_ERR_GENERIC" |
| 2118 |
fi |
| 2119 |
case "$start_resources_count" in |
| 2120 |
1) |
| 2121 |
ocf_log debug "peer not starting: ensure we can start a new cluster" |
| 2122 |
if [ "$revision_compare_result" != "older" ]; then |
| 2123 |
# If our revision is the same as or newer than the peer's last saved |
| 2124 |
# revision, and the peer agent isn't currently starting, we can |
| 2125 |
# restore e-quorum by forcing a new cluster. |
| 2126 |
set_force_new_cluster |
| 2127 |
else |
| 2128 |
ocf_log err "local revision is older and peer is not starting: cannot start" |
| 2129 |
ocf_exit_reason "local revision is older and peer is not starting: cannot start" |
| 2130 |
return "$OCF_ERR_GENERIC" |
| 2131 |
fi |
| 2132 |
;; |
| 2133 |
2) |
| 2134 |
# TODO: can we start "normally", regardless the revisions, if the container-id is the same on both nodes? |
| 2135 |
ocf_log info "peer starting" |
| 2136 |
if [ "$revision_compare_result" = "newer" ]; then |
| 2137 |
set_force_new_cluster |
| 2138 |
elif [ "$revision_compare_result" = "older" ]; then |
| 2139 |
ocf_log info "$NODENAME shall join as learner" |
| 2140 |
JOIN_AS_LEARNER=true |
| 2141 |
else |
| 2142 |
if [ "$(attribute_node_cluster_id get)" = "$(attribute_node_cluster_id_peer)" ]; then |
| 2143 |
ocf_log info "same cluster_id and revision: start normal" |
| 2144 |
else |
| 2145 |
ocf_exit_reason "same revision but different cluster id" |
| 2146 |
return "$OCF_ERR_CONFIGURED" |
| 2147 |
fi |
| 2148 |
fi |
| 2149 |
;; |
| 2150 |
*) |
| 2151 |
ocf_log err "Unexpected start resource count: $start_resources_count" |
| 2152 |
podman_notify |
| 2153 |
return "$OCF_ERR_GENERIC" |
| 2154 |
;; |
| 2155 |
esac |
| 2156 |
;; |
| 2157 |
*) |
| 2158 |
ocf_log err "Unexpected active resource count: $active_resources_count" |
| 2159 |
podman_notify |
| 2160 |
return "$OCF_ERR_GENERIC" |
| 2161 |
;; |
| 2162 |
esac |
| 2163 |
fi |
| 2164 |
fi |
| 2165 |
|
| 2166 |
podman_create_mounts |
| 2167 |
local run_opts="--detach --name=${CONTAINER} --replace" |
| 2168 |
|
| 2169 |
run_opts="$run_opts --oom-score-adj=${OCF_RESKEY_oom}" |
| 2170 |
|
| 2171 |
if ocf_is_true "$JOIN_AS_LEARNER"; then |
| 2172 |
local wait_timeout_sec=$((2*60)) |
| 2173 |
local poll_interval_sec=5 |
| 2174 |
local retries=$(( wait_timeout_sec / poll_interval_sec )) |
| 2175 |
|
| 2176 |
ocf_log info "ensure the leader node added $NODENAME as learner member before continuing (timeout: $wait_timeout_sec seconds)" |
| 2177 |
for try in $(seq $retries); do |
| 2178 |
learner_node=$(attribute_learner_node get) |
| 2179 |
if [ "$NODENAME" != "$learner_node" ]; then |
| 2180 |
ocf_log info "$NODENAME is not in the member list yet. Retry in $poll_interval_sec seconds." |
| 2181 |
sleep $poll_interval_sec |
| 2182 |
continue |
| 2183 |
fi |
| 2184 |
ocf_log info "learner node $learner_node in the member list" |
| 2185 |
break |
| 2186 |
done |
| 2187 |
if [ "$NODENAME" != "$(attribute_learner_node get)" ]; then |
| 2188 |
ocf_log err "wait for $NODENAME to be in the member list timed out" |
| 2189 |
return "$OCF_ERR_GENERIC" |
| 2190 |
fi |
| 2191 |
|
| 2192 |
archive_data_folder |
| 2193 |
if ! wipe_data_folder_for_learner; then |
| 2194 |
return "$OCF_ERR_GENERIC" |
| 2195 |
fi |
| 2196 |
fi |
| 2197 |
|
| 2198 |
ocf_log info "check for changes in pod manifest to decide if the container should be reused or replaced" |
| 2199 |
if ! can_reuse_container ; then |
| 2200 |
ocf_log err "could not determine etcd container reuse strategy" |
| 2201 |
return $OCF_ERR_GENERIC |
| 2202 |
fi |
| 2203 |
|
| 2204 |
# Archive current container and its configuration before creating |
| 2205 |
# new configuration files. |
| 2206 |
if ! ocf_is_true "$OCF_RESKEY_reuse"; then |
| 2207 |
# Log archive container failures but don't block, as the priority |
| 2208 |
# is ensuring the etcd container starts successfully. |
| 2209 |
archive_current_container |
| 2210 |
fi |
| 2211 |
|
| 2212 |
if ! ensure_pod_manifest_copy_exists; then |
| 2213 |
return $OCF_ERR_GENERIC |
| 2214 |
fi |
| 2215 |
|
| 2216 |
if ! prepare_env; then |
| 2217 |
ocf_log err "Could not prepare environment for podman" |
| 2218 |
return $OCF_ERR_GENERIC |
| 2219 |
fi |
| 2220 |
|
| 2221 |
if ! generate_etcd_configuration; then |
| 2222 |
ocf_log err "Could not generate etcd configuration" |
| 2223 |
return $OCF_ERR_CONFIGURED |
| 2224 |
fi |
| 2225 |
|
| 2226 |
run_opts="$run_opts \ |
| 2227 |
--network=host \ |
| 2228 |
-v /etc/kubernetes/static-pod-resources/etcd-certs:/etc/kubernetes/static-pod-certs \ |
| 2229 |
-v /var/lib/etcd:/var/lib/etcd \ |
| 2230 |
--env ETCDCTL_CACERT=$SERVER_CACERT \ |
| 2231 |
--env ETCDCTL_CERT=$ETCD_PEER_CERT \ |
| 2232 |
--env ETCDCTL_KEY=$ETCD_PEER_KEY \ |
| 2233 |
--authfile=$OCF_RESKEY_authfile \ |
| 2234 |
--security-opt label=disable" |
| 2235 |
if [ -n "$OCF_RESKEY_run_opts" ]; then |
| 2236 |
run_opts="$run_opts $OCF_RESKEY_run_opts" |
| 2237 |
fi |
| 2238 |
|
| 2239 |
if [ -f "$ETCD_CONFIGURATION_FILE" ]; then |
| 2240 |
ocf_log info "using etcd configuration file: $ETCD_CONFIGURATION_FILE" |
| 2241 |
else |
| 2242 |
ocf_log err "could not find $ETCD_CONFIGURATION_FILE" |
| 2243 |
return "$OCF_ERR_GENERIC" |
| 2244 |
fi |
| 2245 |
|
| 2246 |
OCF_RESKEY_run_cmd="$OCF_RESKEY_run_cmd --config-file=$ETCD_CONFIGURATION_FILE" |
| 2247 |
if [ -n "$OCF_RESKEY_run_cmd_opts" ]; then |
| 2248 |
OCF_RESKEY_run_cmd="$OCF_RESKEY_run_cmd $OCF_RESKEY_run_cmd_opts" |
| 2249 |
fi |
| 2250 |
|
| 2251 |
if [ "$OCF_RESKEY_image" = "$OCF_RESKEY_image_default" ]; then |
| 2252 |
# no container image provided via input parameters. Read it from the pod manifest. |
| 2253 |
OCF_RESKEY_image=$(jq -r '.spec.containers[] | select( .name=="etcd").image' "$OCF_RESKEY_pod_manifest") |
| 2254 |
ocf_log info "using container image ($OCF_RESKEY_image) from Pod manifest ($OCF_RESKEY_pod_manifest)" |
| 2255 |
else |
| 2256 |
# use the container image provided as input parameter |
| 2257 |
ocf_log info "using container image ($OCF_RESKEY_image) via input parameters" |
| 2258 |
fi |
| 2259 |
|
| 2260 |
if [ $REQUIRE_IMAGE_PULL -eq 1 ]; then |
| 2261 |
ocf_log notice "Beginning pull of image, ${OCF_RESKEY_image}" |
| 2262 |
if ! podman pull --authfile="$OCF_RESKEY_authfile" "${OCF_RESKEY_image}"; then |
| 2263 |
ocf_exit_reason "failed to pull image ${OCF_RESKEY_image}" |
| 2264 |
return $OCF_ERR_GENERIC |
| 2265 |
fi |
| 2266 |
else |
| 2267 |
ocf_log notice "Pull image not required, ${OCF_RESKEY_image}" |
| 2268 |
fi |
| 2269 |
|
| 2270 |
if ocf_is_true "$OCF_RESKEY_reuse" && container_exists; then |
| 2271 |
ocf_log info "starting existing container $CONTAINER." |
| 2272 |
ocf_run podman start "$CONTAINER" |
| 2273 |
else |
| 2274 |
ocf_log info "starting new container $CONTAINER." |
| 2275 |
run_new_container "$run_opts" "$OCF_RESKEY_image" "$OCF_RESKEY_run_cmd" |
| 2276 |
if [ $? -eq 125 ]; then |
| 2277 |
return $OCF_ERR_GENERIC |
| 2278 |
fi |
| 2279 |
fi |
| 2280 |
rc=$? |
| 2281 |
|
| 2282 |
# if the container was stopped or didn't exist before, systemd |
| 2283 |
# removed the libpod* scopes. So always try to recreate the drop-ins |
| 2284 |
if [ $rc -eq 0 ] && ocf_is_true "$OCF_RESKEY_drop_in_dependency"; then |
| 2285 |
cid=$(podman_container_id) |
| 2286 |
create_transient_drop_in_dependency "$cid" |
| 2287 |
rc=$? |
| 2288 |
fi |
| 2289 |
|
| 2290 |
if [ $rc -ne 0 ]; then |
| 2291 |
ocf_exit_reason "podman failed to launch container (error code: $rc)" |
| 2292 |
return $OCF_ERR_GENERIC |
| 2293 |
fi |
| 2294 |
|
| 2295 |
# wait for monitor to pass before declaring that the container is started |
| 2296 |
while true; do |
| 2297 |
podman_simple_status |
| 2298 |
if [ $? -ne $OCF_SUCCESS ]; then |
| 2299 |
ocf_exit_reason "Newly created podman container exited after start" |
| 2300 |
ocf_run podman logs --tail 20 "${CONTAINER}" |
| 2301 |
return $OCF_ERR_GENERIC |
| 2302 |
fi |
| 2303 |
|
| 2304 |
monitor_cmd_exec |
| 2305 |
if [ $? -eq $OCF_SUCCESS ]; then |
| 2306 |
ocf_log notice "Container $CONTAINER started successfully" |
| 2307 |
if is_force_new_cluster; then |
| 2308 |
clear_force_new_cluster |
| 2309 |
|
| 2310 |
local peer_node_name |
| 2311 |
local peer_node_ip |
| 2312 |
peer_node_name="$(get_peer_node_name)" |
| 2313 |
peer_node_ip="$(attribute_node_ip_peer)" |
| 2314 |
if [ -n "$peer_node_name" ] && [ -n "$peer_node_ip" ]; then |
| 2315 |
add_member_as_learner "$peer_node_name" "$peer_node_ip" |
| 2316 |
set_standalone_node |
| 2317 |
else |
| 2318 |
ocf_log err "could not add peer as learner (peer node name: ${peer_node_name:-unknown}, peer ip: ${peer_node_ip:-unknown})" |
| 2319 |
fi |
| 2320 |
fi |
| 2321 |
return $OCF_SUCCESS |
| 2322 |
fi |
| 2323 |
|
| 2324 |
ocf_exit_reason "waiting on monitor_cmd to pass after start" |
| 2325 |
sleep 1 |
| 2326 |
done |
| 2327 |
} |
| 2328 |
|
| 2329 |
# leave_etcd_member_list removes the current node from the etcd member list during |
| 2330 |
# shutdown to ensure clean cluster state. |
| 2331 |
# |
| 2332 |
# Skips removal if this is the standalone (last) node. When both nodes are stopping |
| 2333 |
# concurrently, delays the second node to prevent simultaneous member removal that |
| 2334 |
# could corrupt the etcd WAL. |
| 2335 |
leave_etcd_member_list() |
| 2336 |
{ |
| 2337 |
if ! member_id=$(attribute_node_member_id get); then |
| 2338 |
ocf_log err "error leaving members list: could not get member-id" |
| 2339 |
return |
| 2340 |
fi |
| 2341 |
|
| 2342 |
if is_standalone; then |
| 2343 |
ocf_log info "last member. Not leaving the member list" |
| 2344 |
return |
| 2345 |
fi |
| 2346 |
|
| 2347 |
local stopping_resources_count |
| 2348 |
stopping_resources_count=$(echo "$OCF_RESKEY_CRM_meta_notify_stop_resource" | wc -w) |
| 2349 |
ocf_log info "found '$stopping_resources_count' stopping etcd resources (stop: '$OCF_RESKEY_CRM_meta_notify_stop_resource')" |
| 2350 |
if [ "$stopping_resources_count" -gt 1 ]; then |
| 2351 |
# Prevent WAL corruption by delaying the alphabetically second node's member |
| 2352 |
# removal when both nodes are stopping concurrently. |
| 2353 |
local delayed_node |
| 2354 |
|
| 2355 |
node_names_sorted=$(echo "$OCF_RESKEY_node_ip_map" | sed 's/:[^;]*//g; s/;/ /g' | tr ' ' '\n' | sort | tr '\n' ' ') |
| 2356 |
delayed_node="$(echo "$node_names_sorted" | cut -d' ' -f2)" |
| 2357 |
|
| 2358 |
if [ -z "$delayed_node" ]; then |
| 2359 |
ocf_log warn "could not determine node to be delayed: not leaving the member list" |
| 2360 |
return |
| 2361 |
fi |
| 2362 |
|
| 2363 |
if [ "$NODENAME" = "$delayed_node" ]; then |
| 2364 |
ocf_log info "delaying stop for ${DELAY_SECOND_NODE_LEAVE_SEC}s to prevent simultaneous etcd member removal" |
| 2365 |
sleep $DELAY_SECOND_NODE_LEAVE_SEC |
| 2366 |
fi |
| 2367 |
fi |
| 2368 |
|
| 2369 |
# Ensure we're not the last active resource before leaving. The `standalone_node` property |
| 2370 |
# may not be set if stop was called before monitor check, or after the delayed node waited. |
| 2371 |
local active_resources_count |
| 2372 |
active_resources_count=$(get_truly_active_resources_count) |
| 2373 |
if [ "$active_resources_count" -lt 1 ]; then |
| 2374 |
ocf_log info "last member. Not leaving the member list" |
| 2375 |
return |
| 2376 |
fi |
| 2377 |
|
| 2378 |
ocf_log info "leaving members list as member with ID $member_id" |
| 2379 |
local endpoint |
| 2380 |
endpoint="$(ip_url $(attribute_node_ip get)):2379" |
| 2381 |
ocf_run timeout 30 podman exec "$CONTAINER" etcdctl member remove "$member_id" --endpoints="$endpoint" |
| 2382 |
rc=$? |
| 2383 |
if [ $rc -ne 0 ]; then |
| 2384 |
ocf_log err "error leaving members list, error code: $rc" |
| 2385 |
fi |
| 2386 |
} |
| 2387 |
|
| 2388 |
podman_stop() |
| 2389 |
{ |
| 2390 |
local timeout=60 |
| 2391 |
local rc |
| 2392 |
|
| 2393 |
ocf_log notice "podman-etcd stop" |
| 2394 |
|
| 2395 |
# Clear container health check state file |
| 2396 |
if ! rm -f "$CONTAINER_HEARTBEAT_FILE"; then |
| 2397 |
ocf_log err "could not delete container health check state file" |
| 2398 |
fi |
| 2399 |
|
| 2400 |
attribute_learner_node clear |
| 2401 |
attribute_node_revision update |
| 2402 |
attribute_node_cluster_id update |
| 2403 |
|
| 2404 |
# Use podman inspect instead of podman exec (podman_simple_status) to check |
| 2405 |
# container state. podman exec enters the container's process namespace and |
| 2406 |
# hangs when etcd is unresponsive — the typical scenario that triggers a stop. |
| 2407 |
local container_running |
| 2408 |
container_running=$(podman inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null) |
| 2409 |
if [ "$container_running" != "true" ]; then |
| 2410 |
ocf_log info "could not leave members list: $CONTAINER container not running, running state: ${container_running}" |
| 2411 |
attribute_node_member_id clear |
| 2412 |
return $OCF_SUCCESS |
| 2413 |
fi |
| 2414 |
|
| 2415 |
leave_etcd_member_list |
| 2416 |
|
| 2417 |
# clear node_member_id CIB attribute only after leaving the member list |
| 2418 |
attribute_node_member_id clear |
| 2419 |
|
| 2420 |
if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then |
| 2421 |
timeout=$(((OCF_RESKEY_CRM_meta_timeout/1000) -10 )) |
| 2422 |
if [ $timeout -lt 10 ]; then |
| 2423 |
timeout=10 |
| 2424 |
fi |
| 2425 |
fi |
| 2426 |
|
| 2427 |
if ocf_is_true "$OCF_RESKEY_force_kill"; then |
| 2428 |
ocf_run podman kill "$CONTAINER" |
| 2429 |
rc=$? |
| 2430 |
else |
| 2431 |
ocf_log info "waiting $timeout second[s] before killing container" |
| 2432 |
ocf_run podman stop -t="$timeout" "$CONTAINER" |
| 2433 |
rc=$? |
| 2434 |
# on stop, systemd will automatically delete any transient |
| 2435 |
# drop-in conf that has been created earlier |
| 2436 |
fi |
| 2437 |
|
| 2438 |
if [ $rc -ne 0 ]; then |
| 2439 |
# If the stop failed, it could be because the controlling conmon |
| 2440 |
# process died unexpectedly. If so, a generic error code is returned |
| 2441 |
# but the associated container exit code is -1. If that's the case, |
| 2442 |
# assume there's no failure and continue with the rm as usual. |
| 2443 |
if [ $rc -eq 125 ] && \ |
| 2444 |
podman inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$CONTAINER" | grep -Eq '^(exited|stopped):-1$'; then |
| 2445 |
ocf_log err "Container ${CONTAINER} had an unexpected stop outcome. Trying to remove it anyway." |
| 2446 |
else |
| 2447 |
ocf_exit_reason "Failed to stop container, ${CONTAINER}, based on image, ${OCF_RESKEY_image}." |
| 2448 |
return $OCF_ERR_GENERIC |
| 2449 |
fi |
| 2450 |
fi |
| 2451 |
|
| 2452 |
return $OCF_SUCCESS |
| 2453 |
} |
| 2454 |
|
| 2455 |
image_exists() |
| 2456 |
{ |
| 2457 |
if [ "$OCF_RESKEY_image" = "$OCF_RESKEY_image_default" ]; then |
| 2458 |
# the actual container image was not defined yet. Nor by |
| 2459 |
# the user via OCF_RESKEY, nor by reading the Pod manifest |
| 2460 |
return 0 |
| 2461 |
fi |
| 2462 |
if podman image exists "${OCF_RESKEY_image}"; then |
| 2463 |
# image found |
| 2464 |
return 0 |
| 2465 |
fi |
| 2466 |
|
| 2467 |
if ocf_is_true "$OCF_RESKEY_allow_pull"; then |
| 2468 |
REQUIRE_IMAGE_PULL=1 |
| 2469 |
ocf_log notice "Image (${OCF_RESKEY_image}) does not exist locally but will be pulled during start" |
| 2470 |
return 0 |
| 2471 |
fi |
| 2472 |
# image not found. |
| 2473 |
return 1 |
| 2474 |
} |
| 2475 |
|
| 2476 |
podman_validate() |
| 2477 |
{ |
| 2478 |
check_binary curl |
| 2479 |
check_binary crictl |
| 2480 |
check_binary oc |
| 2481 |
check_binary podman |
| 2482 |
check_binary jq |
| 2483 |
check_binary tar |
| 2484 |
|
| 2485 |
if [ -z "$OCF_RESKEY_node_ip_map" ]; then |
| 2486 |
ocf_exit_reason "'node_ip_map' option is required" |
| 2487 |
exit $OCF_ERR_CONFIGURED |
| 2488 |
fi |
| 2489 |
|
| 2490 |
if [ -z "$OCF_RESKEY_pod_manifest" ]; then |
| 2491 |
ocf_exit_reason "'pod_manifest' option is required" |
| 2492 |
exit $OCF_ERR_CONFIGURED |
| 2493 |
fi |
| 2494 |
|
| 2495 |
if [ -z "$OCF_RESKEY_image" ]; then |
| 2496 |
ocf_exit_reason "'image' option is required" |
| 2497 |
exit $OCF_ERR_CONFIGURED |
| 2498 |
fi |
| 2499 |
|
| 2500 |
if ! image_exists; then |
| 2501 |
ocf_exit_reason "base image, ${OCF_RESKEY_image}, could not be found." |
| 2502 |
exit $OCF_ERR_CONFIGURED |
| 2503 |
fi |
| 2504 |
|
| 2505 |
if [ "$OCF_RESKEY_oom" -lt -1000 ] || [ "$OCF_RESKEY_oom" -gt 1000 ]; then |
| 2506 |
ocf_exit_reason "'oom' value ${OCF_RESKEY_oom} is out of range [-1000:1000]" |
| 2507 |
exit $OCF_ERR_CONFIGURED |
| 2508 |
fi |
| 2509 |
|
| 2510 |
if ! echo "validation test" > "$ETCD_CERTS_HASH_FILE" \ |
| 2511 |
|| ! cat "$ETCD_CERTS_HASH_FILE" >/dev/null 2>&1 \ |
| 2512 |
|| ! rm "$ETCD_CERTS_HASH_FILE"; then |
| 2513 |
ocf_exit_reason "cannot read/write to certificate hash file $ETCD_CERTS_HASH_FILE" |
| 2514 |
exit $OCF_ERR_GENERIC |
| 2515 |
fi |
| 2516 |
|
| 2517 |
return $OCF_SUCCESS |
| 2518 |
} |
| 2519 |
|
| 2520 |
podman_notify() |
| 2521 |
{ |
| 2522 |
ocf_log info "notify: type=${OCF_RESKEY_CRM_meta_notify_type}, operation=${OCF_RESKEY_CRM_meta_notify_operation}, nodes { active=[${OCF_RESKEY_CRM_meta_notify_active_uname}], start=[${OCF_RESKEY_CRM_meta_notify_start_uname}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_uname}] }, resources { active=[${OCF_RESKEY_CRM_meta_notify_active_resource}], start =[${OCF_RESKEY_CRM_meta_notify_start_resource}], stop=[${OCF_RESKEY_CRM_meta_notify_stop_resource}] }" |
| 2523 |
} |
| 2524 |
|
| 2525 |
# TODO : |
| 2526 |
# When a user starts plural clones in a node in globally-unique, a user cannot appoint plural name parameters. |
| 2527 |
# When a user appoints reuse, the resource agent cannot connect plural clones with a container. |
| 2528 |
|
| 2529 |
if ocf_is_true "$OCF_RESKEY_CRM_meta_globally_unique"; then |
| 2530 |
if [ -n "$OCF_RESKEY_name" ]; then |
| 2531 |
if [ -n "$OCF_RESKEY_CRM_meta_clone_node_max" ] && [ "$OCF_RESKEY_CRM_meta_clone_node_max" -ne 1 ] |
| 2532 |
then |
| 2533 |
ocf_exit_reason "Cannot make plural clones from the same name parameter." |
| 2534 |
exit $OCF_ERR_CONFIGURED |
| 2535 |
fi |
| 2536 |
if [ -n "$OCF_RESKEY_CRM_meta_master_node_max" ] && [ "$OCF_RESKEY_CRM_meta_master_node_max" -ne 1 ] |
| 2537 |
then |
| 2538 |
ocf_exit_reason "Cannot make plural master from the same name parameter." |
| 2539 |
exit $OCF_ERR_CONFIGURED |
| 2540 |
fi |
| 2541 |
fi |
| 2542 |
: ${OCF_RESKEY_name=$(echo ${OCF_RESOURCE_INSTANCE} | tr ':' '-')} |
| 2543 |
else |
| 2544 |
: ${OCF_RESKEY_name=${OCF_RESOURCE_INSTANCE}} |
| 2545 |
fi |
| 2546 |
|
| 2547 |
CONTAINER=$OCF_RESKEY_name |
| 2548 |
POD_MANIFEST_COPY="${OCF_RESKEY_config_location}/pod.yaml" |
| 2549 |
ETCD_CONFIGURATION_FILE="${OCF_RESKEY_config_location}/config.yaml" |
| 2550 |
ETCD_BACKUP_FILE="${OCF_RESKEY_backup_location}/config-previous.tar.gz" |
| 2551 |
ETCD_MEMBER_DIR="/var/lib/etcd/member" |
| 2552 |
ETCD_REVISION_JSON="/var/lib/etcd/revision.json" |
| 2553 |
ETCD_REVISION_BUMP_PERCENTAGE=0.2 |
| 2554 |
ETCD_BUMP_REV_DEFAULT=1000000000 |
| 2555 |
ETCD_CERTS_HASH_FILE="${OCF_RESKEY_config_location}/certs.hash" |
| 2556 |
# State file location: Uses HA_RSCTMP to ensure automatic cleanup on reboot. |
| 2557 |
# This is intentional - reboots are controlled stops, not failures requiring detection. |
| 2558 |
CONTAINER_HEARTBEAT_FILE=${HA_RSCTMP}/podman-container-last-running |
| 2559 |
DELAY_SECOND_NODE_LEAVE_SEC=10 |
| 2560 |
# Shorter etcdctl command-timeout for monitor-path calls to prevent |
| 2561 |
# consuming the 25s monitor budget. Non-monitor callers use the default 5s. |
| 2562 |
MONITOR_ETCDCTL_TIMEOUT="3s" |
| 2563 |
|
| 2564 |
# Note: we currently monitor podman containers by with the "podman exec" |
| 2565 |
# command, so make sure that invocation is always valid by enforcing the |
| 2566 |
# exec command to be non-empty |
| 2567 |
: ${OCF_RESKEY_monitor_cmd:=/bin/true} |
| 2568 |
|
| 2569 |
# When OCF_RESKEY_drop_in_dependency is not populated, we |
| 2570 |
# look at another file-based way of enabling the option. |
| 2571 |
# Otherwise, consider it disabled. |
| 2572 |
if [ -z "$OCF_RESKEY_drop_in_dependency" ]; then |
| 2573 |
if [ -f "/etc/sysconfig/podman_drop_in" ] || \ |
| 2574 |
[ -f "/etc/default/podman_drop_in" ]; then |
| 2575 |
OCF_RESKEY_drop_in_dependency=yes |
| 2576 |
fi |
| 2577 |
fi |
| 2578 |
|
| 2579 |
|
| 2580 |
case $__OCF_ACTION in |
| 2581 |
meta-data) meta_data |
| 2582 |
exit $OCF_SUCCESS;; |
| 2583 |
usage|help) podman_usage |
| 2584 |
exit $OCF_SUCCESS |
| 2585 |
;; |
| 2586 |
esac |
| 2587 |
|
| 2588 |
NODENAME=$(ocf_local_nodename) |
| 2589 |
JOIN_AS_LEARNER=false |
| 2590 |
|
| 2591 |
case $__OCF_ACTION in |
| 2592 |
start) |
| 2593 |
podman_validate || exit $? |
| 2594 |
podman_start;; |
| 2595 |
stop) podman_stop;; |
| 2596 |
monitor) podman_monitor;; |
| 2597 |
notify) podman_notify;; |
| 2598 |
validate-all) podman_validate;; |
| 2599 |
*) podman_usage |
| 2600 |
exit $OCF_ERR_UNIMPLEMENTED |
| 2601 |
;; |
| 2602 |
esac |
| 2603 |
rc=$? |
| 2604 |
ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" |
| 2605 |
exit $rc |