Debian Codegraph @

src/test/recovery/t/013_crash_restart.pl

raw
1
2 # Copyright (c) 2021-2025, PostgreSQL Global Development Group
3
4 #
5 # Tests restarts of postgres due to crashes of a subprocess.
6 #
7 # Two longer-running psql subprocesses are used: One to kill a
8 # backend, triggering a crash-restart cycle, one to detect when
9 # postmaster noticed the backend died. The second backend is
10 # necessary because it's otherwise hard to determine if postmaster is
11 # still accepting new sessions (because it hasn't noticed that the
12 # backend died), or because it's already restarted.
13 #
14 use strict;
15 use warnings FATAL => 'all';
16 use PostgreSQL::Test::Cluster;
17 use PostgreSQL::Test::Utils;
18 use Test::More;
19
20 my $psql_timeout = IPC::Run::timer($PostgreSQL::Test::Utils::timeout_default);
21
22 my $node = PostgreSQL::Test::Cluster->new('primary');
23 $node->init(allows_streaming => 1);
24 $node->start();
25
26 # by default PostgreSQL::Test::Cluster doesn't restart after a crash
27 $node->safe_psql(
28 'postgres',
29 q[ALTER SYSTEM SET restart_after_crash = 1;
30 ALTER SYSTEM SET log_connections = receipt;
31 SELECT pg_reload_conf();]);
32
33 # Run psql, keeping session alive, so we have an alive backend to kill.
34 my ($killme_stdin, $killme_stdout, $killme_stderr) = ('', '', '');
35 my $killme = IPC::Run::start(
36 [
37 'psql', '--no-psqlrc', '--quiet', '--no-align', '--tuples-only',
38 '--set' => 'ON_ERROR_STOP=1',
39 '--file' => '-',
40 '--dbname' => $node->connstr('postgres')
41 ],
42 '<' => \$killme_stdin,
43 '>' => \$killme_stdout,
44 '2>' => \$killme_stderr,
45 $psql_timeout);
46
47 # Need a second psql to check if crash-restart happened.
48 my ($monitor_stdin, $monitor_stdout, $monitor_stderr) = ('', '', '');
49 my $monitor = IPC::Run::start(
50 [
51 'psql', '--no-psqlrc', '--quiet', '--no-align', '--tuples-only',
52 '--set' => 'ON_ERROR_STOP=1',
53 '--file', '-', '--dbname' => $node->connstr('postgres')
54 ],
55 '<' => \$monitor_stdin,
56 '>' => \$monitor_stdout,
57 '2>' => \$monitor_stderr,
58 $psql_timeout);
59
60 #create table, insert row that should survive
61 $killme_stdin .= q[
62 CREATE TABLE alive(status text);
63 INSERT INTO alive VALUES($$committed-before-sigquit$$);
64 SELECT pg_backend_pid();
65 ];
66 ok( pump_until(
67 $killme, $psql_timeout, \$killme_stdout, qr/[[:digit:]]+[\r\n]$/m),
68 'acquired pid for SIGQUIT');
69 my $pid = $killme_stdout;
70 chomp($pid);
71 $killme_stdout = '';
72 $killme_stderr = '';
73
74 #insert a row that should *not* survive, due to in-progress xact
75 $killme_stdin .= q[
76 BEGIN;
77 INSERT INTO alive VALUES($$in-progress-before-sigquit$$) RETURNING status;
78 ];
79 ok( pump_until(
80 $killme, $psql_timeout,
81 \$killme_stdout, qr/in-progress-before-sigquit/m),
82 'inserted in-progress-before-sigquit');
83 $killme_stdout = '';
84 $killme_stderr = '';
85
86
87 # Start longrunning query in second session; its failure will signal that
88 # crash-restart has occurred. The initial wait for the trivial select is to
89 # be sure that psql successfully connected to backend.
90 $monitor_stdin .= q[
91 SELECT $$psql-connected$$;
92 SELECT pg_sleep(3600);
93 ];
94 ok( pump_until(
95 $monitor, $psql_timeout, \$monitor_stdout, qr/psql-connected/m),
96 'monitor connected');
97 $monitor_stdout = '';
98 $monitor_stderr = '';
99
100 # kill once with QUIT - we expect psql to exit, while emitting error message first
101 my $ret = PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'QUIT', $pid);
102
103 # Exactly process should have been alive to be killed
104 is($ret, 0, "killed process with SIGQUIT");
105
106 # Check that psql sees the killed backend as having been terminated
107 $killme_stdin .= q[
108 SELECT 1;
109 ];
110 ok( pump_until(
111 $killme,
112 $psql_timeout,
113 \$killme_stderr,
114 qr/WARNING: terminating connection because of unexpected SIGQUIT signal|server closed the connection unexpectedly|connection to server was lost|could not send data to server/m
115 ),
116 "psql query died successfully after SIGQUIT");
117 $killme_stderr = '';
118 $killme_stdout = '';
119 $killme->finish;
120
121 # Wait till server restarts - we should get the WARNING here, but
122 # sometimes the server is unable to send that, if interrupted while
123 # sending.
124 ok( pump_until(
125 $monitor,
126 $psql_timeout,
127 \$monitor_stderr,
128 qr/WARNING: terminating connection because of crash of another server process|server closed the connection unexpectedly|connection to server was lost|could not send data to server/m
129 ),
130 "psql monitor died successfully after SIGQUIT");
131 $monitor->finish;
132
133 # Wait till server restarts
134 is($node->poll_query_until('postgres', undef, ''),
135 "1", "reconnected after SIGQUIT");
136
137
138 # restart psql processes, now that the crash cycle finished
139 ($killme_stdin, $killme_stdout, $killme_stderr) = ('', '', '');
140 $killme->run();
141 ($monitor_stdin, $monitor_stdout, $monitor_stderr) = ('', '', '');
142 $monitor->run();
143
144
145 # Acquire pid of new backend
146 $killme_stdin .= q[
147 SELECT pg_backend_pid();
148 ];
149 ok( pump_until(
150 $killme, $psql_timeout, \$killme_stdout, qr/[[:digit:]]+[\r\n]$/m),
151 "acquired pid for SIGKILL");
152 $pid = $killme_stdout;
153 chomp($pid);
154 $killme_stdout = '';
155 $killme_stderr = '';
156
157 # Insert test rows
158 $killme_stdin .= q[
159 INSERT INTO alive VALUES($$committed-before-sigkill$$) RETURNING status;
160 BEGIN;
161 INSERT INTO alive VALUES($$in-progress-before-sigkill$$) RETURNING status;
162 ];
163 ok( pump_until(
164 $killme, $psql_timeout,
165 \$killme_stdout, qr/in-progress-before-sigkill/m),
166 'inserted in-progress-before-sigkill');
167 $killme_stdout = '';
168 $killme_stderr = '';
169
170 # Re-start longrunning query in second session; its failure will signal that
171 # crash-restart has occurred. The initial wait for the trivial select is to
172 # be sure that psql successfully connected to backend.
173 $monitor_stdin .= q[
174 SELECT $$psql-connected$$;
175 SELECT pg_sleep(3600);
176 ];
177 ok( pump_until(
178 $monitor, $psql_timeout, \$monitor_stdout, qr/psql-connected/m),
179 'monitor connected');
180 $monitor_stdout = '';
181 $monitor_stderr = '';
182
183
184 # kill with SIGKILL this time - we expect the backend to exit, without
185 # being able to emit an error message
186 $ret = PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'KILL', $pid);
187 is($ret, 0, "killed process with KILL");
188
189 # Check that psql sees the server as being terminated. No WARNING,
190 # because signal handlers aren't being run on SIGKILL.
191 $killme_stdin .= q[
192 SELECT 1;
193 ];
194 ok( pump_until(
195 $killme,
196 $psql_timeout,
197 \$killme_stderr,
198 qr/server closed the connection unexpectedly|connection to server was lost|could not send data to server/m
199 ),
200 "psql query died successfully after SIGKILL");
201 $killme->finish;
202
203 # Wait till server restarts - we should get the WARNING here, but
204 # sometimes the server is unable to send that, if interrupted while
205 # sending.
206 ok( pump_until(
207 $monitor,
208 $psql_timeout,
209 \$monitor_stderr,
210 qr/WARNING: terminating connection because of crash of another server process|server closed the connection unexpectedly|connection to server was lost|could not send data to server/m
211 ),
212 "psql monitor died successfully after SIGKILL");
213 $monitor->finish;
214
215 # Wait till server restarts
216 is($node->poll_query_until('postgres', undef, ''),
217 "1", "reconnected after SIGKILL");
218
219 # Make sure the committed rows survived, in-progress ones not
220 is( $node->safe_psql('postgres', 'SELECT * FROM alive'),
221 "committed-before-sigquit\ncommitted-before-sigkill",
222 'data survived');
223
224 is( $node->safe_psql(
225 'postgres',
226 'INSERT INTO alive VALUES($$before-orderly-restart$$) RETURNING status'
227 ),
228 'before-orderly-restart',
229 'can still write after crash restart');
230
231 # Just to be sure, check that an orderly restart now still works
232 $node->restart();
233
234 is( $node->safe_psql('postgres', 'SELECT * FROM alive'),
235 "committed-before-sigquit\ncommitted-before-sigkill\nbefore-orderly-restart",
236 'data survived');
237
238 is( $node->safe_psql(
239 'postgres',
240 'INSERT INTO alive VALUES($$after-orderly-restart$$) RETURNING status'
241 ),
242 'after-orderly-restart',
243 'can still write after orderly restart');
244
245 $node->stop();
246
247 done_testing();