| 1 |
|
| 2 |
# Copyright (c) 2021-2025, PostgreSQL Global Development Group |
| 3 |
|
| 4 |
# |
| 5 |
# Tests restarts of postgres due to crashes of a subprocess. |
| 6 |
# |
| 7 |
# Two longer-running psql subprocesses are used: One to kill a |
| 8 |
# backend, triggering a crash-restart cycle, one to detect when |
| 9 |
# postmaster noticed the backend died. The second backend is |
| 10 |
# necessary because it's otherwise hard to determine if postmaster is |
| 11 |
# still accepting new sessions (because it hasn't noticed that the |
| 12 |
# backend died), or because it's already restarted. |
| 13 |
# |
| 14 |
use strict; |
| 15 |
use warnings FATAL => 'all'; |
| 16 |
use PostgreSQL::Test::Cluster; |
| 17 |
use PostgreSQL::Test::Utils; |
| 18 |
use Test::More; |
| 19 |
|
| 20 |
my $psql_timeout = IPC::Run::timer($PostgreSQL::Test::Utils::timeout_default); |
| 21 |
|
| 22 |
my $node = PostgreSQL::Test::Cluster->new('primary'); |
| 23 |
$node->init(allows_streaming => 1); |
| 24 |
$node->start(); |
| 25 |
|
| 26 |
# by default PostgreSQL::Test::Cluster doesn't restart after a crash |
| 27 |
$node->safe_psql( |
| 28 |
'postgres', |
| 29 |
q[ALTER SYSTEM SET restart_after_crash = 1; |
| 30 |
ALTER SYSTEM SET log_connections = receipt; |
| 31 |
SELECT pg_reload_conf();]); |
| 32 |
|
| 33 |
# Run psql, keeping session alive, so we have an alive backend to kill. |
| 34 |
my ($killme_stdin, $killme_stdout, $killme_stderr) = ('', '', ''); |
| 35 |
my $killme = IPC::Run::start( |
| 36 |
[ |
| 37 |
'psql', '--no-psqlrc', '--quiet', '--no-align', '--tuples-only', |
| 38 |
'--set' => 'ON_ERROR_STOP=1', |
| 39 |
'--file' => '-', |
| 40 |
'--dbname' => $node->connstr('postgres') |
| 41 |
], |
| 42 |
'<' => \$killme_stdin, |
| 43 |
'>' => \$killme_stdout, |
| 44 |
'2>' => \$killme_stderr, |
| 45 |
$psql_timeout); |
| 46 |
|
| 47 |
# Need a second psql to check if crash-restart happened. |
| 48 |
my ($monitor_stdin, $monitor_stdout, $monitor_stderr) = ('', '', ''); |
| 49 |
my $monitor = IPC::Run::start( |
| 50 |
[ |
| 51 |
'psql', '--no-psqlrc', '--quiet', '--no-align', '--tuples-only', |
| 52 |
'--set' => 'ON_ERROR_STOP=1', |
| 53 |
'--file', '-', '--dbname' => $node->connstr('postgres') |
| 54 |
], |
| 55 |
'<' => \$monitor_stdin, |
| 56 |
'>' => \$monitor_stdout, |
| 57 |
'2>' => \$monitor_stderr, |
| 58 |
$psql_timeout); |
| 59 |
|
| 60 |
#create table, insert row that should survive |
| 61 |
$killme_stdin .= q[ |
| 62 |
CREATE TABLE alive(status text); |
| 63 |
INSERT INTO alive VALUES($$committed-before-sigquit$$); |
| 64 |
SELECT pg_backend_pid(); |
| 65 |
]; |
| 66 |
ok( pump_until( |
| 67 |
$killme, $psql_timeout, \$killme_stdout, qr/[[:digit:]]+[\r\n]$/m), |
| 68 |
'acquired pid for SIGQUIT'); |
| 69 |
my $pid = $killme_stdout; |
| 70 |
chomp($pid); |
| 71 |
$killme_stdout = ''; |
| 72 |
$killme_stderr = ''; |
| 73 |
|
| 74 |
#insert a row that should *not* survive, due to in-progress xact |
| 75 |
$killme_stdin .= q[ |
| 76 |
BEGIN; |
| 77 |
INSERT INTO alive VALUES($$in-progress-before-sigquit$$) RETURNING status; |
| 78 |
]; |
| 79 |
ok( pump_until( |
| 80 |
$killme, $psql_timeout, |
| 81 |
\$killme_stdout, qr/in-progress-before-sigquit/m), |
| 82 |
'inserted in-progress-before-sigquit'); |
| 83 |
$killme_stdout = ''; |
| 84 |
$killme_stderr = ''; |
| 85 |
|
| 86 |
|
| 87 |
# Start longrunning query in second session; its failure will signal that |
| 88 |
# crash-restart has occurred. The initial wait for the trivial select is to |
| 89 |
# be sure that psql successfully connected to backend. |
| 90 |
$monitor_stdin .= q[ |
| 91 |
SELECT $$psql-connected$$; |
| 92 |
SELECT pg_sleep(3600); |
| 93 |
]; |
| 94 |
ok( pump_until( |
| 95 |
$monitor, $psql_timeout, \$monitor_stdout, qr/psql-connected/m), |
| 96 |
'monitor connected'); |
| 97 |
$monitor_stdout = ''; |
| 98 |
$monitor_stderr = ''; |
| 99 |
|
| 100 |
# kill once with QUIT - we expect psql to exit, while emitting error message first |
| 101 |
my $ret = PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'QUIT', $pid); |
| 102 |
|
| 103 |
# Exactly process should have been alive to be killed |
| 104 |
is($ret, 0, "killed process with SIGQUIT"); |
| 105 |
|
| 106 |
# Check that psql sees the killed backend as having been terminated |
| 107 |
$killme_stdin .= q[ |
| 108 |
SELECT 1; |
| 109 |
]; |
| 110 |
ok( pump_until( |
| 111 |
$killme, |
| 112 |
$psql_timeout, |
| 113 |
\$killme_stderr, |
| 114 |
qr/WARNING: terminating connection because of unexpected SIGQUIT signal|server closed the connection unexpectedly|connection to server was lost|could not send data to server/m |
| 115 |
), |
| 116 |
"psql query died successfully after SIGQUIT"); |
| 117 |
$killme_stderr = ''; |
| 118 |
$killme_stdout = ''; |
| 119 |
$killme->finish; |
| 120 |
|
| 121 |
# Wait till server restarts - we should get the WARNING here, but |
| 122 |
# sometimes the server is unable to send that, if interrupted while |
| 123 |
# sending. |
| 124 |
ok( pump_until( |
| 125 |
$monitor, |
| 126 |
$psql_timeout, |
| 127 |
\$monitor_stderr, |
| 128 |
qr/WARNING: terminating connection because of crash of another server process|server closed the connection unexpectedly|connection to server was lost|could not send data to server/m |
| 129 |
), |
| 130 |
"psql monitor died successfully after SIGQUIT"); |
| 131 |
$monitor->finish; |
| 132 |
|
| 133 |
# Wait till server restarts |
| 134 |
is($node->poll_query_until('postgres', undef, ''), |
| 135 |
"1", "reconnected after SIGQUIT"); |
| 136 |
|
| 137 |
|
| 138 |
# restart psql processes, now that the crash cycle finished |
| 139 |
($killme_stdin, $killme_stdout, $killme_stderr) = ('', '', ''); |
| 140 |
$killme->run(); |
| 141 |
($monitor_stdin, $monitor_stdout, $monitor_stderr) = ('', '', ''); |
| 142 |
$monitor->run(); |
| 143 |
|
| 144 |
|
| 145 |
# Acquire pid of new backend |
| 146 |
$killme_stdin .= q[ |
| 147 |
SELECT pg_backend_pid(); |
| 148 |
]; |
| 149 |
ok( pump_until( |
| 150 |
$killme, $psql_timeout, \$killme_stdout, qr/[[:digit:]]+[\r\n]$/m), |
| 151 |
"acquired pid for SIGKILL"); |
| 152 |
$pid = $killme_stdout; |
| 153 |
chomp($pid); |
| 154 |
$killme_stdout = ''; |
| 155 |
$killme_stderr = ''; |
| 156 |
|
| 157 |
# Insert test rows |
| 158 |
$killme_stdin .= q[ |
| 159 |
INSERT INTO alive VALUES($$committed-before-sigkill$$) RETURNING status; |
| 160 |
BEGIN; |
| 161 |
INSERT INTO alive VALUES($$in-progress-before-sigkill$$) RETURNING status; |
| 162 |
]; |
| 163 |
ok( pump_until( |
| 164 |
$killme, $psql_timeout, |
| 165 |
\$killme_stdout, qr/in-progress-before-sigkill/m), |
| 166 |
'inserted in-progress-before-sigkill'); |
| 167 |
$killme_stdout = ''; |
| 168 |
$killme_stderr = ''; |
| 169 |
|
| 170 |
# Re-start longrunning query in second session; its failure will signal that |
| 171 |
# crash-restart has occurred. The initial wait for the trivial select is to |
| 172 |
# be sure that psql successfully connected to backend. |
| 173 |
$monitor_stdin .= q[ |
| 174 |
SELECT $$psql-connected$$; |
| 175 |
SELECT pg_sleep(3600); |
| 176 |
]; |
| 177 |
ok( pump_until( |
| 178 |
$monitor, $psql_timeout, \$monitor_stdout, qr/psql-connected/m), |
| 179 |
'monitor connected'); |
| 180 |
$monitor_stdout = ''; |
| 181 |
$monitor_stderr = ''; |
| 182 |
|
| 183 |
|
| 184 |
# kill with SIGKILL this time - we expect the backend to exit, without |
| 185 |
# being able to emit an error message |
| 186 |
$ret = PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'KILL', $pid); |
| 187 |
is($ret, 0, "killed process with KILL"); |
| 188 |
|
| 189 |
# Check that psql sees the server as being terminated. No WARNING, |
| 190 |
# because signal handlers aren't being run on SIGKILL. |
| 191 |
$killme_stdin .= q[ |
| 192 |
SELECT 1; |
| 193 |
]; |
| 194 |
ok( pump_until( |
| 195 |
$killme, |
| 196 |
$psql_timeout, |
| 197 |
\$killme_stderr, |
| 198 |
qr/server closed the connection unexpectedly|connection to server was lost|could not send data to server/m |
| 199 |
), |
| 200 |
"psql query died successfully after SIGKILL"); |
| 201 |
$killme->finish; |
| 202 |
|
| 203 |
# Wait till server restarts - we should get the WARNING here, but |
| 204 |
# sometimes the server is unable to send that, if interrupted while |
| 205 |
# sending. |
| 206 |
ok( pump_until( |
| 207 |
$monitor, |
| 208 |
$psql_timeout, |
| 209 |
\$monitor_stderr, |
| 210 |
qr/WARNING: terminating connection because of crash of another server process|server closed the connection unexpectedly|connection to server was lost|could not send data to server/m |
| 211 |
), |
| 212 |
"psql monitor died successfully after SIGKILL"); |
| 213 |
$monitor->finish; |
| 214 |
|
| 215 |
# Wait till server restarts |
| 216 |
is($node->poll_query_until('postgres', undef, ''), |
| 217 |
"1", "reconnected after SIGKILL"); |
| 218 |
|
| 219 |
# Make sure the committed rows survived, in-progress ones not |
| 220 |
is( $node->safe_psql('postgres', 'SELECT * FROM alive'), |
| 221 |
"committed-before-sigquit\ncommitted-before-sigkill", |
| 222 |
'data survived'); |
| 223 |
|
| 224 |
is( $node->safe_psql( |
| 225 |
'postgres', |
| 226 |
'INSERT INTO alive VALUES($$before-orderly-restart$$) RETURNING status' |
| 227 |
), |
| 228 |
'before-orderly-restart', |
| 229 |
'can still write after crash restart'); |
| 230 |
|
| 231 |
# Just to be sure, check that an orderly restart now still works |
| 232 |
$node->restart(); |
| 233 |
|
| 234 |
is( $node->safe_psql('postgres', 'SELECT * FROM alive'), |
| 235 |
"committed-before-sigquit\ncommitted-before-sigkill\nbefore-orderly-restart", |
| 236 |
'data survived'); |
| 237 |
|
| 238 |
is( $node->safe_psql( |
| 239 |
'postgres', |
| 240 |
'INSERT INTO alive VALUES($$after-orderly-restart$$) RETURNING status' |
| 241 |
), |
| 242 |
'after-orderly-restart', |
| 243 |
'can still write after orderly restart'); |
| 244 |
|
| 245 |
$node->stop(); |
| 246 |
|
| 247 |
done_testing(); |