| 1 |
|
| 2 |
# Copyright (c) 2024-2025, PostgreSQL Global Development Group |
| 3 |
|
| 4 |
use strict; |
| 5 |
use warnings FATAL => 'all'; |
| 6 |
use PostgreSQL::Test::Cluster; |
| 7 |
use PostgreSQL::Test::Utils; |
| 8 |
use Time::HiRes qw(usleep); |
| 9 |
use Test::More; |
| 10 |
|
| 11 |
################################################## |
| 12 |
# Test race condition when a restart point is running during a promotion, |
| 13 |
# checking that WAL segments are correctly removed in the restart point |
| 14 |
# while the promotion finishes. |
| 15 |
# |
| 16 |
# This test relies on an injection point that causes the checkpointer to |
| 17 |
# wait in the middle of a restart point on a standby. The checkpointer |
| 18 |
# is awaken to finish its restart point only once the promotion of the |
| 19 |
# standby is completed, and the node should be able to restart properly. |
| 20 |
################################################## |
| 21 |
|
| 22 |
if ($ENV{enable_injection_points} ne 'yes') |
| 23 |
{ |
| 24 |
plan skip_all => 'Injection points not supported by this build'; |
| 25 |
} |
| 26 |
|
| 27 |
# Initialize primary node. log_checkpoints is required as the checkpoint |
| 28 |
# activity is monitored based on the contents of the logs. |
| 29 |
my $node_primary = PostgreSQL::Test::Cluster->new('master'); |
| 30 |
$node_primary->init(allows_streaming => 1); |
| 31 |
$node_primary->append_conf( |
| 32 |
'postgresql.conf', q[ |
| 33 |
log_checkpoints = on |
| 34 |
restart_after_crash = on |
| 35 |
]); |
| 36 |
$node_primary->start; |
| 37 |
|
| 38 |
# Check if the extension injection_points is available, as it may be |
| 39 |
# possible that this script is run with installcheck, where the module |
| 40 |
# would not be installed by default. |
| 41 |
if (!$node_primary->check_extension('injection_points')) |
| 42 |
{ |
| 43 |
plan skip_all => 'Extension injection_points not installed'; |
| 44 |
} |
| 45 |
|
| 46 |
my $backup_name = 'my_backup'; |
| 47 |
$node_primary->backup($backup_name); |
| 48 |
|
| 49 |
# Setup a standby. |
| 50 |
my $node_standby = PostgreSQL::Test::Cluster->new('standby1'); |
| 51 |
$node_standby->init_from_backup($node_primary, $backup_name, |
| 52 |
has_streaming => 1); |
| 53 |
$node_standby->start; |
| 54 |
|
| 55 |
# Dummy table for the upcoming tests. |
| 56 |
$node_primary->safe_psql('postgres', 'checkpoint'); |
| 57 |
$node_primary->safe_psql('postgres', 'CREATE TABLE prim_tab (a int);'); |
| 58 |
|
| 59 |
# Register an injection point on the standby so as the follow-up |
| 60 |
# restart point will wait on it. |
| 61 |
$node_primary->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); |
| 62 |
# Wait until the extension has been created on the standby |
| 63 |
$node_primary->wait_for_replay_catchup($node_standby); |
| 64 |
|
| 65 |
# Note that from this point the checkpointer will wait in the middle of |
| 66 |
# a restart point on the standby. |
| 67 |
$node_standby->safe_psql('postgres', |
| 68 |
"SELECT injection_points_attach('create-restart-point', 'wait');"); |
| 69 |
|
| 70 |
# Execute a restart point on the standby, that we will now be waiting on. |
| 71 |
# This needs to be in the background. |
| 72 |
my $logstart = -s $node_standby->logfile; |
| 73 |
my $psql_session = |
| 74 |
$node_standby->background_psql('postgres', on_error_stop => 0); |
| 75 |
$psql_session->query_until( |
| 76 |
qr/starting_checkpoint/, q( |
| 77 |
\echo starting_checkpoint |
| 78 |
CHECKPOINT; |
| 79 |
)); |
| 80 |
|
| 81 |
# Switch one WAL segment to make the previous restart point remove the |
| 82 |
# segment once the restart point completes. |
| 83 |
$node_primary->safe_psql('postgres', 'INSERT INTO prim_tab VALUES (1);'); |
| 84 |
$node_primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); |
| 85 |
$node_primary->wait_for_replay_catchup($node_standby); |
| 86 |
|
| 87 |
# Wait until the checkpointer is in the middle of the restart point |
| 88 |
# processing. |
| 89 |
$node_standby->wait_for_event('checkpointer', 'create-restart-point'); |
| 90 |
|
| 91 |
# Check the logs that the restart point has started on standby. This is |
| 92 |
# optional, but let's be sure. |
| 93 |
ok( $node_standby->log_contains( |
| 94 |
"restartpoint starting: immediate wait", $logstart), |
| 95 |
"restartpoint has started"); |
| 96 |
|
| 97 |
# Trigger promotion during the restart point. |
| 98 |
$node_primary->stop; |
| 99 |
$node_standby->promote; |
| 100 |
|
| 101 |
# Update the start position before waking up the checkpointer! |
| 102 |
$logstart = -s $node_standby->logfile; |
| 103 |
|
| 104 |
# Now wake up the checkpointer. |
| 105 |
$node_standby->safe_psql('postgres', |
| 106 |
"SELECT injection_points_wakeup('create-restart-point');"); |
| 107 |
|
| 108 |
# Wait until the previous restart point completes on the newly-promoted |
| 109 |
# standby, checking the logs for that. |
| 110 |
my $checkpoint_complete = 0; |
| 111 |
foreach my $i (0 .. 10 * $PostgreSQL::Test::Utils::timeout_default) |
| 112 |
{ |
| 113 |
if ($node_standby->log_contains("restartpoint complete", $logstart)) |
| 114 |
{ |
| 115 |
$checkpoint_complete = 1; |
| 116 |
last; |
| 117 |
} |
| 118 |
usleep(100_000); |
| 119 |
} |
| 120 |
is($checkpoint_complete, 1, 'restart point has completed'); |
| 121 |
|
| 122 |
# Kill with SIGKILL, forcing all the backends to restart. |
| 123 |
my $psql_timeout = IPC::Run::timer(3600); |
| 124 |
my ($killme_stdin, $killme_stdout, $killme_stderr) = ('', '', ''); |
| 125 |
my $killme = IPC::Run::start( |
| 126 |
[ |
| 127 |
'psql', '--no-psqlrc', '--no-align', '--tuples-only', '--quiet', |
| 128 |
'--set' => 'ON_ERROR_STOP=1', |
| 129 |
'--file' => '-', |
| 130 |
'--dbname' => $node_standby->connstr('postgres') |
| 131 |
], |
| 132 |
'<' => \$killme_stdin, |
| 133 |
'>' => \$killme_stdout, |
| 134 |
'2>' => \$killme_stderr, |
| 135 |
$psql_timeout); |
| 136 |
$killme_stdin .= q[ |
| 137 |
SELECT pg_backend_pid(); |
| 138 |
]; |
| 139 |
$killme->pump until $killme_stdout =~ /[[:digit:]]+[\r\n]$/; |
| 140 |
my $pid = $killme_stdout; |
| 141 |
chomp($pid); |
| 142 |
$killme_stdout = ''; |
| 143 |
$killme_stderr = ''; |
| 144 |
|
| 145 |
my $ret = PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'KILL', $pid); |
| 146 |
is($ret, 0, 'killed process with KILL'); |
| 147 |
|
| 148 |
# Wait until the server restarts, finish consuming output. |
| 149 |
$killme_stdin .= q[ |
| 150 |
SELECT 1; |
| 151 |
]; |
| 152 |
ok( pump_until( |
| 153 |
$killme, |
| 154 |
$psql_timeout, |
| 155 |
\$killme_stderr, |
| 156 |
qr/server closed the connection unexpectedly|connection to server was lost|could not send data to server/m |
| 157 |
), |
| 158 |
"psql query died successfully after SIGKILL"); |
| 159 |
$killme->finish; |
| 160 |
|
| 161 |
# Wait till server finishes restarting. |
| 162 |
$node_standby->poll_query_until('postgres', undef, ''); |
| 163 |
|
| 164 |
# After recovery, the server should be able to start. |
| 165 |
my $stdout; |
| 166 |
my $stderr; |
| 167 |
($ret, $stdout, $stderr) = $node_standby->psql('postgres', 'select 1'); |
| 168 |
is($ret, 0, "psql connect success"); |
| 169 |
is($stdout, 1, "psql select 1"); |
| 170 |
|
| 171 |
done_testing(); |