Debian Codegraph @

src/test/recovery/t/039_end_of_wal.pl

raw
1 # Copyright (c) 2023-2025, PostgreSQL Global Development Group
2 #
3 # Test detecting end-of-WAL conditions. This test suite generates
4 # fake defective page and record headers to trigger various failure
5 # scenarios.
6
7 use strict;
8 use warnings FATAL => 'all';
9 use PostgreSQL::Test::Cluster;
10 use PostgreSQL::Test::Utils;
11 use Test::More;
12 use Fcntl qw(SEEK_SET);
13
14 use integer; # causes / operator to use integer math
15
16 # Is this a big-endian system ("network" byte order)? We can't use 'Q' in
17 # pack() calls because it's not available in some perl builds, so we need to
18 # break 64 bit LSN values into two 'I' values. Fortunately we don't need to
19 # deal with high values, so we can just write 0 for the high order 32 bits, but
20 # we need to know the endianness to do that.
21 my $BIG_ENDIAN = pack("L", 0x12345678) eq pack("N", 0x12345678);
22
23 # Fields retrieved from code headers.
24 my @scan_result = scan_server_header('access/xlog_internal.h',
25 '#define\s+XLOG_PAGE_MAGIC\s+(\w+)');
26 my $XLP_PAGE_MAGIC = hex($scan_result[0]);
27 @scan_result = scan_server_header('access/xlog_internal.h',
28 '#define\s+XLP_FIRST_IS_CONTRECORD\s+(\w+)');
29 my $XLP_FIRST_IS_CONTRECORD = hex($scan_result[0]);
30
31 # Values queried from the server
32 my $WAL_SEGMENT_SIZE;
33 my $WAL_BLOCK_SIZE;
34 my $TLI;
35
36 # Get GUC value, converted to an int.
37 sub get_int_setting
38 {
39 my $node = shift;
40 my $name = shift;
41 return int(
42 $node->safe_psql(
43 'postgres',
44 "SELECT setting FROM pg_settings WHERE name = '$name'"));
45 }
46
47 sub start_of_page
48 {
49 my $lsn = shift;
50 return $lsn & ~($WAL_BLOCK_SIZE - 1);
51 }
52
53 sub start_of_next_page
54 {
55 my $lsn = shift;
56 return start_of_page($lsn) + $WAL_BLOCK_SIZE;
57 }
58
59 # Build a fake WAL record header based on the data given by the caller.
60 # This needs to follow the format of the C structure XLogRecord. To
61 # be inserted with write_wal().
62 sub build_record_header
63 {
64 my $xl_tot_len = shift;
65 my $xl_xid = shift || 0;
66 my $xl_prev = shift || 0;
67 my $xl_info = shift || 0;
68 my $xl_rmid = shift || 0;
69 my $xl_crc = shift || 0;
70
71 # This needs to follow the structure XLogRecord:
72 # I for xl_tot_len
73 # I for xl_xid
74 # II for xl_prev
75 # C for xl_info
76 # C for xl_rmid
77 # BB for two bytes of padding
78 # I for xl_crc
79 return pack("IIIICCBBI",
80 $xl_tot_len, $xl_xid,
81 $BIG_ENDIAN ? 0 : $xl_prev,
82 $BIG_ENDIAN ? $xl_prev : 0,
83 $xl_info, $xl_rmid, 0, 0, $xl_crc);
84 }
85
86 # Build a fake WAL page header, based on the data given by the caller
87 # This needs to follow the format of the C structure XLogPageHeaderData.
88 # To be inserted with write_wal().
89 sub build_page_header
90 {
91 my $xlp_magic = shift;
92 my $xlp_info = shift || 0;
93 my $xlp_tli = shift || 0;
94 my $xlp_pageaddr = shift || 0;
95 my $xlp_rem_len = shift || 0;
96
97 # This needs to follow the structure XLogPageHeaderData:
98 # S for xlp_magic
99 # S for xlp_info
100 # I for xlp_tli
101 # II for xlp_pageaddr
102 # I for xlp_rem_len
103 return pack("SSIIII",
104 $xlp_magic, $xlp_info, $xlp_tli,
105 $BIG_ENDIAN ? 0 : $xlp_pageaddr,
106 $BIG_ENDIAN ? $xlp_pageaddr : 0, $xlp_rem_len);
107 }
108
109 # Setup a new node. The configuration chosen here minimizes the number
110 # of arbitrary records that could get generated in a cluster. Enlarging
111 # checkpoint_timeout avoids noise with checkpoint activity. wal_level
112 # set to "minimal" avoids random standby snapshot records. Autovacuum
113 # could also trigger randomly, generating random WAL activity of its own.
114 my $node = PostgreSQL::Test::Cluster->new("node");
115 $node->init;
116 $node->append_conf(
117 'postgresql.conf',
118 q[wal_level = minimal
119 autovacuum = off
120 checkpoint_timeout = '30min'
121 ]);
122 $node->start;
123 $node->safe_psql('postgres', "CREATE TABLE t AS SELECT 42");
124
125 $WAL_SEGMENT_SIZE = get_int_setting($node, 'wal_segment_size');
126 $WAL_BLOCK_SIZE = get_int_setting($node, 'wal_block_size');
127 $TLI = $node->safe_psql('postgres',
128 "SELECT timeline_id FROM pg_control_checkpoint();");
129
130 # Initial LSN may vary across systems due to different catalog contents set up
131 # by initdb. Switch to a new WAL file so all systems start out in the same
132 # place. The first test depends on trailing zeroes on a page with a valid
133 # header.
134 $node->safe_psql('postgres', "SELECT pg_switch_wal();");
135
136 my $end_lsn;
137 my $prev_lsn;
138
139 ###########################################################################
140 note "Single-page end-of-WAL detection";
141 ###########################################################################
142
143 # xl_tot_len is 0 (a common case, we hit trailing zeroes).
144 $node->emit_wal(0);
145 $end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
146 $node->stop('immediate');
147 my $log_size = -s $node->logfile;
148 $node->start;
149 ok( $node->log_contains(
150 "invalid record length at .*: expected at least 24, got 0", $log_size
151 ),
152 "xl_tot_len zero");
153
154 # xl_tot_len is < 24 (presumably recycled garbage).
155 $node->emit_wal(0);
156 $end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
157 $node->stop('immediate');
158 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE, build_record_header(23));
159 $log_size = -s $node->logfile;
160 $node->start;
161 ok( $node->log_contains(
162 "invalid record length at .*: expected at least 24, got 23",
163 $log_size),
164 "xl_tot_len short");
165
166 # xl_tot_len in final position, not big enough to span into a new page but
167 # also not eligible for regular record header validation
168 $node->emit_wal(0);
169 $end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
170 $node->stop('immediate');
171 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE, build_record_header(1));
172 $log_size = -s $node->logfile;
173 $node->start;
174 ok( $node->log_contains(
175 "invalid record length at .*: expected at least 24, got 1", $log_size
176 ),
177 "xl_tot_len short at end-of-page");
178
179 # Need more pages, but xl_prev check fails first.
180 $node->emit_wal(0);
181 $end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
182 $node->stop('immediate');
183 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
184 build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
185 $log_size = -s $node->logfile;
186 $node->start;
187 ok( $node->log_contains(
188 "record with incorrect prev-link 0/DEADBEEF at .*", $log_size),
189 "xl_prev bad");
190
191 # xl_crc check fails.
192 $node->emit_wal(0);
193 $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
194 $end_lsn = $node->emit_wal(10);
195 $node->stop('immediate');
196 # Corrupt a byte in that record, breaking its CRC.
197 $node->write_wal($TLI, $end_lsn - 8, $WAL_SEGMENT_SIZE, '!');
198 $log_size = -s $node->logfile;
199 $node->start;
200 ok( $node->log_contains(
201 "incorrect resource manager data checksum in record at .*", $log_size
202 ),
203 "xl_crc bad");
204
205
206 ###########################################################################
207 note "Multi-page end-of-WAL detection, header is not split";
208 ###########################################################################
209
210 # This series of tests requires a valid xl_prev set in the record header
211 # written to WAL.
212
213 # Good xl_prev, we hit zero page next (zero magic).
214 $node->emit_wal(0);
215 $prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
216 $end_lsn = $node->emit_wal(0);
217 $node->stop('immediate');
218 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
219 build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
220 $log_size = -s $node->logfile;
221 $node->start;
222 ok($node->log_contains("invalid magic number 0000 .* LSN .*", $log_size),
223 "xlp_magic zero");
224
225 # Good xl_prev, we hit garbage page next (bad magic).
226 $node->emit_wal(0);
227 $prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
228 $end_lsn = $node->emit_wal(0);
229 $node->stop('immediate');
230 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
231 build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
232 $node->write_wal($TLI, start_of_next_page($end_lsn),
233 $WAL_SEGMENT_SIZE, build_page_header(0xcafe, 0, 1, 0));
234 $log_size = -s $node->logfile;
235 $node->start;
236 ok($node->log_contains("invalid magic number CAFE .* LSN .*", $log_size),
237 "xlp_magic bad");
238
239 # Good xl_prev, we hit typical recycled page (good xlp_magic, bad
240 # xlp_pageaddr).
241 $node->emit_wal(0);
242 $prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
243 $end_lsn = $node->emit_wal(0);
244 $node->stop('immediate');
245 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
246 build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
247 $node->write_wal($TLI, start_of_next_page($end_lsn),
248 $WAL_SEGMENT_SIZE, build_page_header($XLP_PAGE_MAGIC, 0, 1, 0xbaaaaaad));
249 $log_size = -s $node->logfile;
250 $node->start;
251 ok( $node->log_contains(
252 "unexpected pageaddr 0/BAAAAAAD in .*, LSN .*,", $log_size),
253 "xlp_pageaddr bad");
254
255 # Good xl_prev, xlp_magic, xlp_pageaddr, but bogus xlp_info.
256 $node->emit_wal(0);
257 $prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
258 $end_lsn = $node->emit_wal(0);
259 $node->stop('immediate');
260 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
261 build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
262 $node->write_wal(
263 $TLI,
264 start_of_next_page($end_lsn),
265 $WAL_SEGMENT_SIZE,
266 build_page_header(
267 $XLP_PAGE_MAGIC, 0x1234, 1, start_of_next_page($end_lsn)));
268 $log_size = -s $node->logfile;
269 $node->start;
270 ok($node->log_contains("invalid info bits 1234 in .*, LSN .*,", $log_size),
271 "xlp_info bad");
272
273 # Good xl_prev, xlp_magic, xlp_pageaddr, but xlp_info doesn't mention
274 # continuation record.
275 $node->emit_wal(0);
276 $prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
277 $end_lsn = $node->emit_wal(0);
278 $node->stop('immediate');
279 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
280 build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
281 $node->write_wal($TLI, start_of_next_page($end_lsn),
282 $WAL_SEGMENT_SIZE,
283 build_page_header($XLP_PAGE_MAGIC, 0, 1, start_of_next_page($end_lsn)));
284 $log_size = -s $node->logfile;
285 $node->start;
286 ok($node->log_contains("there is no contrecord flag at .*", $log_size),
287 "xlp_info lacks XLP_FIRST_IS_CONTRECORD");
288
289 # Good xl_prev, xlp_magic, xlp_pageaddr, xlp_info but xlp_rem_len doesn't add
290 # up.
291 $node->emit_wal(0);
292 $prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
293 $end_lsn = $node->emit_wal(0);
294 $node->stop('immediate');
295 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
296 build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
297 $node->write_wal(
298 $TLI,
299 start_of_next_page($end_lsn),
300 $WAL_SEGMENT_SIZE,
301 build_page_header(
302 $XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD,
303 1, start_of_next_page($end_lsn),
304 123456));
305 $log_size = -s $node->logfile;
306 $node->start;
307 ok( $node->log_contains(
308 "invalid contrecord length 123456 .* at .*", $log_size),
309 "xlp_rem_len bad");
310
311
312 ###########################################################################
313 note "Multi-page, but header is split, so page checks are done first";
314 ###########################################################################
315
316 # xl_prev is bad and xl_tot_len is too big, but we'll check xlp_magic first.
317 $node->emit_wal(0);
318 $end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
319 $node->stop('immediate');
320 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
321 build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
322 $log_size = -s $node->logfile;
323 $node->start;
324 ok($node->log_contains("invalid magic number 0000 .* LSN .*", $log_size),
325 "xlp_magic zero (split record header)");
326
327 # And we'll also check xlp_pageaddr before any header checks.
328 $node->emit_wal(0);
329 $end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
330 $node->stop('immediate');
331 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
332 build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
333 $node->write_wal(
334 $TLI,
335 start_of_next_page($end_lsn),
336 $WAL_SEGMENT_SIZE,
337 build_page_header(
338 $XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD, 1, 0xbaaaaaad));
339 $log_size = -s $node->logfile;
340 $node->start;
341 ok( $node->log_contains(
342 "unexpected pageaddr 0/BAAAAAAD in .*, LSN .*,", $log_size),
343 "xlp_pageaddr bad (split record header)");
344
345 # We'll also discover that xlp_rem_len doesn't add up before any
346 # header checks,
347 $node->emit_wal(0);
348 $end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
349 $node->stop('immediate');
350 $node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
351 build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
352 $node->write_wal(
353 $TLI,
354 start_of_next_page($end_lsn),
355 $WAL_SEGMENT_SIZE,
356 build_page_header(
357 $XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD,
358 1, start_of_next_page($end_lsn),
359 123456));
360 $log_size = -s $node->logfile;
361 $node->start;
362 ok( $node->log_contains(
363 "invalid contrecord length 123456 .* at .*", $log_size),
364 "xlp_rem_len bad (split record header)");
365
366 done_testing();