Debian Codegraph @

src/test/modules/test_aio/t/001_aio.pl

raw
1 # Copyright (c) 2025, PostgreSQL Global Development Group
2
3 use strict;
4 use warnings FATAL => 'all';
5
6 use PostgreSQL::Test::Cluster;
7 use PostgreSQL::Test::Utils;
8 use Test::More;
9
10
11 ###
12 # Test io_method=worker
13 ###
14 my $node_worker = create_node('worker');
15 $node_worker->start();
16
17 test_generic('worker', $node_worker);
18 SKIP:
19 {
20 skip 'Injection points not supported by this build', 1
21 unless $ENV{enable_injection_points} eq 'yes';
22 test_inject_worker('worker', $node_worker);
23 }
24
25 $node_worker->stop();
26
27
28 ###
29 # Test io_method=io_uring
30 ###
31
32 if (have_io_uring())
33 {
34 my $node_uring = create_node('io_uring');
35 $node_uring->start();
36 test_generic('io_uring', $node_uring);
37 $node_uring->stop();
38 }
39
40
41 ###
42 # Test io_method=sync
43 ###
44
45 my $node_sync = create_node('sync');
46
47 # just to have one test not use the default auto-tuning
48
49 $node_sync->append_conf(
50 'postgresql.conf', qq(
51 io_max_concurrency=4
52 ));
53
54 $node_sync->start();
55 test_generic('sync', $node_sync);
56 $node_sync->stop();
57
58 done_testing();
59
60
61 ###
62 # Test Helpers
63 ###
64
65 sub create_node
66 {
67 local $Test::Builder::Level = $Test::Builder::Level + 1;
68
69 my $io_method = shift;
70
71 my $node = PostgreSQL::Test::Cluster->new($io_method);
72
73 # Want to test initdb for each IO method, otherwise we could just reuse
74 # the cluster.
75 #
76 # Unfortunately Cluster::init() puts PG_TEST_INITDB_EXTRA_OPTS after the
77 # options specified by ->extra, if somebody puts -c io_method=xyz in
78 # PG_TEST_INITDB_EXTRA_OPTS it would break this test. Fix that up if we
79 # detect it.
80 local $ENV{PG_TEST_INITDB_EXTRA_OPTS} = $ENV{PG_TEST_INITDB_EXTRA_OPTS};
81 if (defined $ENV{PG_TEST_INITDB_EXTRA_OPTS}
82 && $ENV{PG_TEST_INITDB_EXTRA_OPTS} =~ m/io_method=/)
83 {
84 $ENV{PG_TEST_INITDB_EXTRA_OPTS} .= " -c io_method=$io_method";
85 }
86
87 $node->init(extra => [ '-c', "io_method=$io_method" ]);
88
89 $node->append_conf(
90 'postgresql.conf', qq(
91 shared_preload_libraries=test_aio
92 log_min_messages = 'DEBUG3'
93 log_statement=all
94 log_error_verbosity=default
95 restart_after_crash=false
96 temp_buffers=100
97 ));
98
99 # Even though we used -c io_method=... above, if TEMP_CONFIG sets
100 # io_method, it'd override the setting persisted at initdb time. While
101 # using (and later verifying) the setting from initdb provides some
102 # verification of having used the io_method during initdb, it's probably
103 # not worth the complication of only appending if the variable is set in
104 # in TEMP_CONFIG.
105 $node->append_conf(
106 'postgresql.conf', qq(
107 io_method=$io_method
108 ));
109
110 ok(1, "$io_method: initdb");
111
112 return $node;
113 }
114
115 sub have_io_uring
116 {
117 # To detect if io_uring is supported, we look at the error message for
118 # assigning an invalid value to an enum GUC, which lists all the valid
119 # options. We need to use -C to deal with running as administrator on
120 # windows, the superuser check is omitted if -C is used.
121 my ($stdout, $stderr) =
122 run_command [qw(postgres -C invalid -c io_method=invalid)];
123 die "can't determine supported io_method values"
124 unless $stderr =~ m/Available values: ([^\.]+)\./;
125 my $methods = $1;
126 note "supported io_method values are: $methods";
127
128 return ($methods =~ m/io_uring/) ? 1 : 0;
129 }
130
131 sub psql_like
132 {
133 local $Test::Builder::Level = $Test::Builder::Level + 1;
134 my $io_method = shift;
135 my $psql = shift;
136 my $name = shift;
137 my $sql = shift;
138 my $expected_stdout = shift;
139 my $expected_stderr = shift;
140 my ($cmdret, $output);
141
142 ($output, $cmdret) = $psql->query($sql);
143
144 like($output, $expected_stdout, "$io_method: $name: expected stdout");
145 like($psql->{stderr}, $expected_stderr,
146 "$io_method: $name: expected stderr");
147 $psql->{stderr} = '';
148
149 return $output;
150 }
151
152 sub query_wait_block
153 {
154 local $Test::Builder::Level = $Test::Builder::Level + 1;
155 my $io_method = shift;
156 my $node = shift;
157 my $psql = shift;
158 my $name = shift;
159 my $sql = shift;
160 my $waitfor = shift;
161
162 my $pid = $psql->query_safe('SELECT pg_backend_pid()');
163
164 $psql->{stdin} .= qq($sql;\n);
165 $psql->{run}->pump_nb();
166 ok(1, "$io_method: $name: issued sql");
167
168 $node->poll_query_until('postgres',
169 qq(SELECT wait_event FROM pg_stat_activity WHERE pid = $pid),
170 $waitfor);
171 ok(1, "$io_method: $name: observed $waitfor wait event");
172 }
173
174 # Returns count of checksum failures for the specified database or for shared
175 # relations, if $datname is undefined.
176 sub checksum_failures
177 {
178 local $Test::Builder::Level = $Test::Builder::Level + 1;
179 my $psql = shift;
180 my $datname = shift;
181 my $checksum_count;
182 my $checksum_last_failure;
183
184 if (defined $datname)
185 {
186 $checksum_count = $psql->query_safe(
187 qq(
188 SELECT checksum_failures FROM pg_stat_database WHERE datname = '$datname';
189 ));
190 $checksum_last_failure = $psql->query_safe(
191 qq(
192 SELECT checksum_last_failure FROM pg_stat_database WHERE datname = '$datname';
193 ));
194 }
195 else
196 {
197 $checksum_count = $psql->query_safe(
198 qq(
199 SELECT checksum_failures FROM pg_stat_database WHERE datname IS NULL;
200 ));
201 $checksum_last_failure = $psql->query_safe(
202 qq(
203 SELECT checksum_last_failure FROM pg_stat_database WHERE datname IS NULL;
204 ));
205 }
206
207 return $checksum_count, $checksum_last_failure;
208 }
209
210 ###
211 # Sub-tests
212 ###
213
214 # Sanity checks for the IO handle API
215 sub test_handle
216 {
217 my $io_method = shift;
218 my $node = shift;
219
220 my $psql = $node->background_psql('postgres', on_error_stop => 0);
221
222 # leak warning: implicit xact
223 psql_like(
224 $io_method,
225 $psql,
226 "handle_get() leak in implicit xact",
227 qq(SELECT handle_get()),
228 qr/^$/,
229 qr/leaked AIO handle/,
230 "$io_method: leaky handle_get() warns");
231
232 # leak warning: explicit xact
233 psql_like(
234 $io_method, $psql,
235 "handle_get() leak in explicit xact",
236 qq(BEGIN; SELECT handle_get(); COMMIT),
237 qr/^$/, qr/leaked AIO handle/);
238
239
240 # leak warning: explicit xact, rollback
241 psql_like(
242 $io_method,
243 $psql,
244 "handle_get() leak in explicit xact, rollback",
245 qq(BEGIN; SELECT handle_get(); ROLLBACK;),
246 qr/^$/,
247 qr/leaked AIO handle/);
248
249 # leak warning: subtrans
250 psql_like(
251 $io_method,
252 $psql,
253 "handle_get() leak in subxact",
254 qq(BEGIN; SAVEPOINT foo; SELECT handle_get(); COMMIT;),
255 qr/^$/,
256 qr/leaked AIO handle/);
257
258 # leak warning + error: released in different command (thus resowner)
259 psql_like(
260 $io_method,
261 $psql,
262 "handle_release() in different command",
263 qq(BEGIN; SELECT handle_get(); SELECT handle_release_last(); COMMIT;),
264 qr/^$/,
265 qr/leaked AIO handle.*release in unexpected state/ms);
266
267 # no leak, release in same command
268 psql_like(
269 $io_method,
270 $psql,
271 "handle_release() in same command",
272 qq(BEGIN; SELECT handle_get() UNION ALL SELECT handle_release_last(); COMMIT;),
273 qr/^$/,
274 qr/^$/);
275
276 # normal handle use
277 psql_like($io_method, $psql, "handle_get_release()",
278 qq(SELECT handle_get_release()),
279 qr/^$/, qr/^$/);
280
281 # should error out, API violation
282 psql_like(
283 $io_method,
284 $psql,
285 "handle_get_twice()",
286 qq(SELECT handle_get_twice()),
287 qr/^$/,
288 qr/ERROR: API violation: Only one IO can be handed out$/);
289
290 # recover after error in implicit xact
291 psql_like(
292 $io_method,
293 $psql,
294 "handle error recovery in implicit xact",
295 qq(SELECT handle_get_and_error(); SELECT 'ok', handle_get_release()),
296 qr/^|ok$/,
297 qr/ERROR.*as you command/);
298
299 # recover after error in implicit xact
300 psql_like(
301 $io_method,
302 $psql,
303 "handle error recovery in explicit xact",
304 qq(BEGIN; SELECT handle_get_and_error(); SELECT handle_get_release(), 'ok'; COMMIT;),
305 qr/^|ok$/,
306 qr/ERROR.*as you command/);
307
308 # recover after error in subtrans
309 psql_like(
310 $io_method,
311 $psql,
312 "handle error recovery in explicit subxact",
313 qq(BEGIN; SAVEPOINT foo; SELECT handle_get_and_error(); ROLLBACK TO SAVEPOINT foo; SELECT handle_get_release(); ROLLBACK;),
314 qr/^|ok$/,
315 qr/ERROR.*as you command/);
316
317 $psql->quit();
318 }
319
320 # Sanity checks for the batchmode API
321 sub test_batchmode
322 {
323 my $io_method = shift;
324 my $node = shift;
325
326 my $psql = $node->background_psql('postgres', on_error_stop => 0);
327
328 # In a build with RELCACHE_FORCE_RELEASE and CATCACHE_FORCE_RELEASE, just
329 # using SELECT batch_start() causes spurious test failures, because the
330 # lookup of the type information when printing the result tuple also
331 # starts a batch. The easiest way around is to not print a result tuple.
332 my $batch_start_sql = qq(SELECT WHERE batch_start() IS NULL);
333
334 # leak warning & recovery: implicit xact
335 psql_like(
336 $io_method,
337 $psql,
338 "batch_start() leak & cleanup in implicit xact",
339 $batch_start_sql,
340 qr/^$/,
341 qr/open AIO batch at end/,
342 "$io_method: leaky batch_start() warns");
343
344 # leak warning & recovery: explicit xact
345 psql_like(
346 $io_method,
347 $psql,
348 "batch_start() leak & cleanup in explicit xact",
349 qq(BEGIN; $batch_start_sql; COMMIT;),
350 qr/^$/,
351 qr/open AIO batch at end/,
352 "$io_method: leaky batch_start() warns");
353
354
355 # leak warning & recovery: explicit xact, rollback
356 #
357 # XXX: This doesn't fail right now, due to not getting a chance to do
358 # something at transaction command commit. That's not a correctness issue,
359 # it just means it's a bit harder to find buggy code.
360 #psql_like($io_method, $psql,
361 # "batch_start() leak & cleanup after abort",
362 # qq(BEGIN; $batch_start_sql; ROLLBACK;),
363 # qr/^$/,
364 # qr/open AIO batch at end/, "$io_method: leaky batch_start() warns");
365
366 # no warning, batch closed in same command
367 psql_like(
368 $io_method,
369 $psql,
370 "batch_start(), batch_end() works",
371 qq($batch_start_sql UNION ALL SELECT WHERE batch_end() IS NULL),
372 qr/^$/,
373 qr/^$/,
374 "$io_method: batch_start(), batch_end()");
375
376 $psql->quit();
377 }
378
379 # Test that simple cases of invalid pages are reported
380 sub test_io_error
381 {
382 my $io_method = shift;
383 my $node = shift;
384 my ($ret, $output);
385
386 my $psql = $node->background_psql('postgres', on_error_stop => 0);
387
388 $psql->query_safe(
389 qq(
390 CREATE TEMPORARY TABLE tmp_corr(data int not null);
391 INSERT INTO tmp_corr SELECT generate_series(1, 10000);
392 SELECT modify_rel_block('tmp_corr', 1, corrupt_header=>true);
393 ));
394
395 foreach my $tblname (qw(tbl_corr tmp_corr))
396 {
397 my $invalid_page_re =
398 $tblname eq 'tbl_corr'
399 ? qr/invalid page in block 1 of relation "base\/\d+\/\d+/
400 : qr/invalid page in block 1 of relation "base\/\d+\/t\d+_\d+/;
401
402 # verify the error is reported in custom C code
403 psql_like(
404 $io_method,
405 $psql,
406 "read_rel_block_ll() of $tblname page",
407 qq(SELECT read_rel_block_ll('$tblname', 1)),
408 qr/^$/,
409 $invalid_page_re);
410
411 # verify the error is reported for bufmgr reads, seq scan
412 psql_like(
413 $io_method, $psql,
414 "sequential scan of $tblname block fails",
415 qq(SELECT count(*) FROM $tblname),
416 qr/^$/, $invalid_page_re);
417
418 # verify the error is reported for bufmgr reads, tid scan
419 psql_like(
420 $io_method,
421 $psql,
422 "tid scan of $tblname block fails",
423 qq(SELECT count(*) FROM $tblname WHERE ctid = '(1, 1)'),
424 qr/^$/,
425 $invalid_page_re);
426 }
427
428 $psql->quit();
429 }
430
431 # Test interplay between StartBufferIO and TerminateBufferIO
432 sub test_startwait_io
433 {
434 my $io_method = shift;
435 my $node = shift;
436 my ($ret, $output);
437
438 my $psql_a = $node->background_psql('postgres', on_error_stop => 0);
439 my $psql_b = $node->background_psql('postgres', on_error_stop => 0);
440
441
442 ### Verify behavior for normal tables
443
444 # create a buffer we can play around with
445 my $buf_id = psql_like(
446 $io_method, $psql_a,
447 "creation of toy buffer succeeds",
448 qq(SELECT buffer_create_toy('tbl_ok', 1)),
449 qr/^\d+$/, qr/^$/);
450
451 # check that one backend can perform StartBufferIO
452 psql_like(
453 $io_method,
454 $psql_a,
455 "first StartBufferIO",
456 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>false);),
457 qr/^t$/,
458 qr/^$/);
459
460 # but not twice on the same buffer (non-waiting)
461 psql_like(
462 $io_method,
463 $psql_a,
464 "second StartBufferIO fails, same session",
465 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>true);),
466 qr/^f$/,
467 qr/^$/);
468 psql_like(
469 $io_method,
470 $psql_b,
471 "second StartBufferIO fails, other session",
472 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>true);),
473 qr/^f$/,
474 qr/^$/);
475
476 # start io in a different session, will block
477 query_wait_block(
478 $io_method,
479 $node,
480 $psql_b,
481 "blocking start buffer io",
482 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>false);),
483 "BufferIo");
484
485 # Terminate the IO, without marking it as success, this should trigger the
486 # waiting session to be able to start the io
487 psql_like(
488 $io_method,
489 $psql_a,
490 "blocking start buffer io, terminating io, not valid",
491 qq(SELECT buffer_call_terminate_io($buf_id, for_input=>true, succeed=>false, io_error=>false, release_aio=>false)),
492 qr/^$/,
493 qr/^$/);
494
495
496 # Because the IO was terminated, but not marked as valid, second session should get the right to start io
497 pump_until($psql_b->{run}, $psql_b->{timeout}, \$psql_b->{stdout}, qr/t/);
498 ok(1, "$io_method: blocking start buffer io, can start io");
499
500 # terminate the IO again
501 $psql_b->query_safe(
502 qq(SELECT buffer_call_terminate_io($buf_id, for_input=>true, succeed=>false, io_error=>false, release_aio=>false);)
503 );
504
505
506 # same as the above scenario, but mark IO as having succeeded
507 psql_like(
508 $io_method,
509 $psql_a,
510 "blocking buffer io w/ success: first start buffer io",
511 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>false);),
512 qr/^t$/,
513 qr/^$/);
514
515 # start io in a different session, will block
516 query_wait_block(
517 $io_method,
518 $node,
519 $psql_b,
520 "blocking start buffer io",
521 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>false);),
522 "BufferIo");
523
524 # Terminate the IO, marking it as success
525 psql_like(
526 $io_method,
527 $psql_a,
528 "blocking start buffer io, terminating io, valid",
529 qq(SELECT buffer_call_terminate_io($buf_id, for_input=>true, succeed=>true, io_error=>false, release_aio=>false)),
530 qr/^$/,
531 qr/^$/);
532
533 # Because the IO was terminated, and marked as valid, second session should complete but not need io
534 pump_until($psql_b->{run}, $psql_b->{timeout}, \$psql_b->{stdout}, qr/f/);
535 ok(1, "$io_method: blocking start buffer io, no need to start io");
536
537 # buffer is valid now, make it invalid again
538 $psql_a->query_safe(qq(SELECT buffer_create_toy('tbl_ok', 1);));
539
540
541 ### Verify behavior for temporary tables
542
543 # Can't unfortunately share the code with the normal table case, there are
544 # too many behavioral differences.
545
546 # create a buffer we can play around with
547 $psql_a->query_safe(
548 qq(
549 CREATE TEMPORARY TABLE tmp_ok(data int not null);
550 INSERT INTO tmp_ok SELECT generate_series(1, 10000);
551 ));
552 $buf_id = $psql_a->query_safe(qq(SELECT buffer_create_toy('tmp_ok', 3);));
553
554 # check that one backend can perform StartLocalBufferIO
555 psql_like(
556 $io_method,
557 $psql_a,
558 "first StartLocalBufferIO",
559 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>true);),
560 qr/^t$/,
561 qr/^$/);
562
563 # Because local buffers don't use IO_IN_PROGRESS, a second StartLocalBufferIO
564 # succeeds as well. This test mostly serves as a documentation of that
565 # fact. If we had actually started IO, it'd be different.
566 psql_like(
567 $io_method,
568 $psql_a,
569 "second StartLocalBufferIO succeeds, same session",
570 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>true);),
571 qr/^t$/,
572 qr/^$/);
573
574 # Terminate the IO again, without marking it as a success
575 $psql_a->query_safe(
576 qq(SELECT buffer_call_terminate_io($buf_id, for_input=>true, succeed=>false, io_error=>false, release_aio=>false);)
577 );
578 psql_like(
579 $io_method,
580 $psql_a,
581 "StartLocalBufferIO after not marking valid succeeds, same session",
582 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>true);),
583 qr/^t$/,
584 qr/^$/);
585
586 # Terminate the IO again, marking it as a success
587 $psql_a->query_safe(
588 qq(SELECT buffer_call_terminate_io($buf_id, for_input=>true, succeed=>true, io_error=>false, release_aio=>false);)
589 );
590
591 # Now another StartLocalBufferIO should fail, this time because the buffer
592 # is already valid.
593 psql_like(
594 $io_method,
595 $psql_a,
596 "StartLocalBufferIO after marking valid fails",
597 qq(SELECT buffer_call_start_io($buf_id, for_input=>true, nowait=>false);),
598 qr/^f$/,
599 qr/^$/);
600
601 $psql_a->quit();
602 $psql_b->quit();
603 }
604
605 # Test that if the backend issuing a read doesn't wait for the IO's
606 # completion, another backend can complete the IO
607 sub test_complete_foreign
608 {
609 my $io_method = shift;
610 my $node = shift;
611 my ($ret, $output);
612
613 my $psql_a = $node->background_psql('postgres', on_error_stop => 0);
614 my $psql_b = $node->background_psql('postgres', on_error_stop => 0);
615
616 # Issue IO without waiting for completion, then sleep
617 $psql_a->query_safe(
618 qq(SELECT read_rel_block_ll('tbl_ok', 1, wait_complete=>false);));
619
620 # Check that another backend can read the relevant block
621 psql_like(
622 $io_method,
623 $psql_b,
624 "completing read started by sleeping backend",
625 qq(SELECT count(*) FROM tbl_ok WHERE ctid = '(1,1)' LIMIT 1),
626 qr/^1$/,
627 qr/^$/);
628
629 # Issue IO without waiting for completion, then exit.
630 $psql_a->query_safe(
631 qq(SELECT read_rel_block_ll('tbl_ok', 1, wait_complete=>false);));
632 $psql_a->reconnect_and_clear();
633
634 # Check that another backend can read the relevant block. This verifies
635 # that the exiting backend left the AIO in a sane state.
636 psql_like(
637 $io_method,
638 $psql_b,
639 "read buffer started by exited backend",
640 qq(SELECT count(*) FROM tbl_ok WHERE ctid = '(1,1)' LIMIT 1),
641 qr/^1$/,
642 qr/^$/);
643
644 # Read a tbl_corr block, then sleep. The other session will retry the IO
645 # and also fail. The easiest thing to verify that seems to be to check
646 # that both are in the log.
647 my $log_location = -s $node->logfile;
648 $psql_a->query_safe(
649 qq(SELECT read_rel_block_ll('tbl_corr', 1, wait_complete=>false);));
650
651 psql_like(
652 $io_method,
653 $psql_b,
654 "completing read of tbl_corr block started by other backend",
655 qq(SELECT count(*) FROM tbl_corr WHERE ctid = '(1,1)' LIMIT 1),
656 qr/^$/,
657 qr/invalid page in block/);
658
659 # The log message issued for the read_rel_block_ll() should be logged as a LOG
660 $node->wait_for_log(qr/LOG[^\n]+invalid page in/, $log_location);
661 ok(1,
662 "$io_method: completing read of tbl_corr block started by other backend: LOG message for background read"
663 );
664
665 # But for the SELECT, it should be an ERROR
666 $log_location =
667 $node->wait_for_log(qr/ERROR[^\n]+invalid page in/, $log_location);
668 ok(1,
669 "$io_method: completing read of tbl_corr block started by other backend: ERROR message for foreground read"
670 );
671
672 $psql_a->quit();
673 $psql_b->quit();
674 }
675
676 # Test that we deal correctly with FDs being closed while IO is in progress
677 sub test_close_fd
678 {
679 my $io_method = shift;
680 my $node = shift;
681 my ($ret, $output);
682
683 my $psql = $node->background_psql('postgres', on_error_stop => 0);
684
685 psql_like(
686 $io_method,
687 $psql,
688 "close all FDs after read, waiting for results",
689 qq(
690 SELECT read_rel_block_ll('tbl_ok', 1,
691 wait_complete=>true,
692 batchmode_enter=>true,
693 smgrreleaseall=>true,
694 batchmode_exit=>true
695 );),
696 qr/^$/,
697 qr/^$/);
698
699 psql_like(
700 $io_method,
701 $psql,
702 "close all FDs after read, no waiting",
703 qq(
704 SELECT read_rel_block_ll('tbl_ok', 1,
705 wait_complete=>false,
706 batchmode_enter=>true,
707 smgrreleaseall=>true,
708 batchmode_exit=>true
709 );),
710 qr/^$/,
711 qr/^$/);
712
713 # Check that another backend can read the relevant block
714 psql_like(
715 $io_method,
716 $psql,
717 "close all FDs after read, no waiting, query works",
718 qq(SELECT count(*) FROM tbl_ok WHERE ctid = '(1,1)' LIMIT 1),
719 qr/^1$/,
720 qr/^$/);
721
722 $psql->quit();
723 }
724
725 # Tests using injection points. Mostly to exercise hard IO errors that are
726 # hard to trigger without using injection points.
727 sub test_inject
728 {
729 my $io_method = shift;
730 my $node = shift;
731 my ($ret, $output);
732
733 my $psql = $node->background_psql('postgres', on_error_stop => 0);
734
735 # injected what we'd expect
736 $psql->query_safe(qq(SELECT inj_io_short_read_attach(8192);));
737 $psql->query_safe(qq(SELECT invalidate_rel_block('tbl_ok', 2);));
738 psql_like(
739 $io_method, $psql,
740 "injection point not triggering failure",
741 qq(SELECT count(*) FROM tbl_ok WHERE ctid = '(2, 1)'),
742 qr/^1$/, qr/^$/);
743
744 # injected a read shorter than a single block, expecting error
745 $psql->query_safe(qq(SELECT inj_io_short_read_attach(17);));
746 $psql->query_safe(qq(SELECT invalidate_rel_block('tbl_ok', 2);));
747 psql_like(
748 $io_method,
749 $psql,
750 "single block short read fails",
751 qq(SELECT count(*) FROM tbl_ok WHERE ctid = '(2, 1)'),
752 qr/^$/,
753 qr/ERROR:.*could not read blocks 2\.\.2 in file "base\/.*": read only 0 of 8192 bytes/
754 );
755
756 # shorten multi-block read to a single block, should retry
757 my $inval_query = qq(SELECT invalidate_rel_block('tbl_ok', 0);
758 SELECT invalidate_rel_block('tbl_ok', 1);
759 SELECT invalidate_rel_block('tbl_ok', 2);
760 SELECT invalidate_rel_block('tbl_ok', 3);
761 /* gap */
762 SELECT invalidate_rel_block('tbl_ok', 5);
763 SELECT invalidate_rel_block('tbl_ok', 6);
764 SELECT invalidate_rel_block('tbl_ok', 7);
765 SELECT invalidate_rel_block('tbl_ok', 8););
766
767 $psql->query_safe($inval_query);
768 $psql->query_safe(qq(SELECT inj_io_short_read_attach(8192);));
769 psql_like(
770 $io_method, $psql,
771 "multi block short read (1 block) is retried",
772 qq(SELECT count(*) FROM tbl_ok),
773 qr/^10000$/, qr/^$/);
774
775 # shorten multi-block read to two blocks, should retry
776 $psql->query_safe($inval_query);
777 $psql->query_safe(qq(SELECT inj_io_short_read_attach(8192*2);));
778
779 psql_like(
780 $io_method, $psql,
781 "multi block short read (2 blocks) is retried",
782 qq(SELECT count(*) FROM tbl_ok),
783 qr/^10000$/, qr/^$/);
784
785 # verify that page verification errors are detected even as part of a
786 # shortened multi-block read (tbl_corr, block 1 is corrupted)
787 $psql->query_safe(
788 qq(
789 SELECT invalidate_rel_block('tbl_corr', 0);
790 SELECT invalidate_rel_block('tbl_corr', 1);
791 SELECT invalidate_rel_block('tbl_corr', 2);
792 SELECT inj_io_short_read_attach(8192);
793 ));
794
795 psql_like(
796 $io_method,
797 $psql,
798 "shortened multi-block read detects invalid page",
799 qq(SELECT count(*) FROM tbl_corr WHERE ctid < '(2, 1)'),
800 qr/^$/,
801 qr/ERROR:.*invalid page in block 1 of relation "base\/.*/);
802
803 # trigger a hard error, should error out
804 $psql->query_safe(
805 qq(
806 SELECT inj_io_short_read_attach(-errno_from_string('EIO'));
807 SELECT invalidate_rel_block('tbl_ok', 2);
808 ));
809
810 psql_like(
811 $io_method,
812 $psql,
813 "first hard IO error is reported",
814 qq(SELECT count(*) FROM tbl_ok),
815 qr/^$/,
816 qr!ERROR:.*could not read blocks 2\.\.2 in file "base/.*": (?:I/O|Input/output) error!
817 );
818
819 psql_like(
820 $io_method,
821 $psql,
822 "second hard IO error is reported",
823 qq(SELECT count(*) FROM tbl_ok),
824 qr/^$/,
825 qr!ERROR:.*could not read blocks 2\.\.2 in file "base/.*": (?:I/O|Input/output) error!
826 );
827
828 $psql->query_safe(qq(SELECT inj_io_short_read_detach()));
829
830 # now the IO should be ok.
831 psql_like(
832 $io_method, $psql,
833 "recovers after hard error",
834 qq(SELECT count(*) FROM tbl_ok),
835 qr/^10000$/, qr/^$/);
836
837 # trigger a different hard error, should error out
838 $psql->query_safe(
839 qq(
840 SELECT inj_io_short_read_attach(-errno_from_string('EROFS'));
841 SELECT invalidate_rel_block('tbl_ok', 2);
842 ));
843 psql_like(
844 $io_method,
845 $psql,
846 "different hard IO error is reported",
847 qq(SELECT count(*) FROM tbl_ok),
848 qr/^$/,
849 qr/ERROR:.*could not read blocks 2\.\.2 in file \"base\/.*\": Read-only file system/
850 );
851 $psql->query_safe(qq(SELECT inj_io_short_read_detach()));
852
853 $psql->quit();
854 }
855
856 # Tests using injection points, only for io_method=worker.
857 #
858 # io_method=worker has the special case of needing to reopen files. That can
859 # in theory fail, because the file could be gone. That's a hard path to test
860 # for real, so we use an injection point to trigger it.
861 sub test_inject_worker
862 {
863 my $io_method = shift;
864 my $node = shift;
865 my ($ret, $output);
866
867 my $psql = $node->background_psql('postgres', on_error_stop => 0);
868
869 # trigger a failure to reopen, should error out, but should recover
870 $psql->query_safe(
871 qq(
872 SELECT inj_io_reopen_attach();
873 SELECT invalidate_rel_block('tbl_ok', 1);
874 ));
875
876 psql_like(
877 $io_method,
878 $psql,
879 "failure to open: detected",
880 qq(SELECT count(*) FROM tbl_ok),
881 qr/^$/,
882 qr/ERROR:.*could not read blocks 1\.\.1 in file "base\/.*": No such file or directory/
883 );
884
885 $psql->query_safe(qq(SELECT inj_io_reopen_detach();));
886
887 # check that we indeed recover
888 psql_like(
889 $io_method, $psql,
890 "failure to open: recovers",
891 qq(SELECT count(*) FROM tbl_ok),
892 qr/^10000$/, qr/^$/);
893
894 $psql->quit();
895 }
896
897 # Verify that we handle a relation getting removed (due to a rollback or a
898 # DROP TABLE) while IO is ongoing for that table.
899 sub test_invalidate
900 {
901 my $io_method = shift;
902 my $node = shift;
903
904 my $psql = $node->background_psql('postgres', on_error_stop => 0);
905
906 foreach my $persistency (qw(normal unlogged temporary))
907 {
908 my $sql_persistency = $persistency eq 'normal' ? '' : $persistency;
909 my $tblname = $persistency . '_transactional';
910
911 my $create_sql = qq(
912 CREATE $sql_persistency TABLE $tblname (id int not null, data text not null) WITH (AUTOVACUUM_ENABLED = false);
913 INSERT INTO $tblname(id, data) SELECT generate_series(1, 10000) as id, repeat('a', 200);
914 );
915
916 # Verify that outstanding read IO does not cause problems with
917 # AbortTransaction -> smgrDoPendingDeletes -> smgrdounlinkall -> ...
918 # -> Invalidate[Local]Buffer.
919 $psql->query_safe("BEGIN; $create_sql;");
920 $psql->query_safe(
921 qq(
922 SELECT read_rel_block_ll('$tblname', 1, wait_complete=>false);
923 ));
924 psql_like(
925 $io_method,
926 $psql,
927 "rollback of newly created $persistency table with outstanding IO",
928 qq(ROLLBACK),
929 qr/^$/,
930 qr/^$/);
931
932 # Verify that outstanding read IO does not cause problems with
933 # CommitTransaction -> smgrDoPendingDeletes -> smgrdounlinkall -> ...
934 # -> Invalidate[Local]Buffer.
935 $psql->query_safe("BEGIN; $create_sql; COMMIT;");
936 $psql->query_safe(
937 qq(
938 BEGIN;
939 SELECT read_rel_block_ll('$tblname', 1, wait_complete=>false);
940 ));
941
942 psql_like(
943 $io_method, $psql,
944 "drop $persistency table with outstanding IO",
945 qq(DROP TABLE $tblname),
946 qr/^$/, qr/^$/);
947
948 psql_like($io_method, $psql,
949 "commit after drop $persistency table with outstanding IO",
950 qq(COMMIT), qr/^$/, qr/^$/);
951 }
952
953 $psql->quit();
954 }
955
956 # Test behavior related to ZERO_ON_ERROR and zero_damaged_pages
957 sub test_zero
958 {
959 my $io_method = shift;
960 my $node = shift;
961
962 my $psql_a = $node->background_psql('postgres', on_error_stop => 0);
963 my $psql_b = $node->background_psql('postgres', on_error_stop => 0);
964
965 foreach my $persistency (qw(normal temporary))
966 {
967 my $sql_persistency = $persistency eq 'normal' ? '' : $persistency;
968
969 $psql_a->query_safe(
970 qq(
971 CREATE $sql_persistency TABLE tbl_zero(id int) WITH (AUTOVACUUM_ENABLED = false);
972 INSERT INTO tbl_zero SELECT generate_series(1, 10000);
973 ));
974
975 $psql_a->query_safe(
976 qq(
977 SELECT modify_rel_block('tbl_zero', 0, corrupt_header=>true);
978 ));
979
980 # Check that page validity errors are detected
981 psql_like(
982 $io_method,
983 $psql_a,
984 "$persistency: test reading of invalid block 0",
985 qq(
986 SELECT read_rel_block_ll('tbl_zero', 0, zero_on_error=>false)),
987 qr/^$/,
988 qr/^psql:<stdin>:\d+: ERROR: invalid page in block 0 of relation "base\/.*\/.*$/
989 );
990
991 # Check that page validity errors are zeroed
992 psql_like(
993 $io_method,
994 $psql_a,
995 "$persistency: test zeroing of invalid block 0",
996 qq(
997 SELECT read_rel_block_ll('tbl_zero', 0, zero_on_error=>true)),
998 qr/^$/,
999 qr/^psql:<stdin>:\d+: WARNING: invalid page in block 0 of relation "base\/.*\/.*"; zeroing out page$/
1000 );
1001
1002 # And that once the corruption is fixed, we can read again
1003 $psql_a->query(
1004 qq(
1005 SELECT modify_rel_block('tbl_zero', 0, zero=>true);
1006 ));
1007 $psql_a->{stderr} = '';
1008
1009 psql_like(
1010 $io_method,
1011 $psql_a,
1012 "$persistency: test re-read of block 0",
1013 qq(
1014 SELECT read_rel_block_ll('tbl_zero', 0, zero_on_error=>false)),
1015 qr/^$/,
1016 qr/^$/);
1017
1018 # Check a page validity error in another block, to ensure we report
1019 # the correct block number
1020 $psql_a->query_safe(
1021 qq(
1022 SELECT modify_rel_block('tbl_zero', 3, corrupt_header=>true);
1023 ));
1024 psql_like(
1025 $io_method,
1026 $psql_a,
1027 "$persistency: test zeroing of invalid block 3",
1028 qq(SELECT read_rel_block_ll('tbl_zero', 3, zero_on_error=>true);),
1029 qr/^$/,
1030 qr/^psql:<stdin>:\d+: WARNING: invalid page in block 3 of relation "base\/.*\/.*"; zeroing out page$/
1031 );
1032
1033
1034 # Check one read reporting multiple invalid blocks
1035 $psql_a->query_safe(
1036 qq(
1037 SELECT modify_rel_block('tbl_zero', 2, corrupt_header=>true);
1038 SELECT modify_rel_block('tbl_zero', 3, corrupt_header=>true);
1039 ));
1040 # First test error
1041 psql_like(
1042 $io_method,
1043 $psql_a,
1044 "$persistency: test reading of invalid block 2,3 in larger read",
1045 qq(SELECT read_rel_block_ll('tbl_zero', 1, nblocks=>4, zero_on_error=>false)),
1046 qr/^$/,
1047 qr/^psql:<stdin>:\d+: ERROR: 2 invalid pages among blocks 1..4 of relation "base\/.*\/.*\nDETAIL: Block 2 held the first invalid page\.\nHINT:[^\n]+$/
1048 );
1049
1050 # Then test zeroing via ZERO_ON_ERROR flag
1051 psql_like(
1052 $io_method,
1053 $psql_a,
1054 "$persistency: test zeroing of invalid block 2,3 in larger read, ZERO_ON_ERROR",
1055 qq(SELECT read_rel_block_ll('tbl_zero', 1, nblocks=>4, zero_on_error=>true)),
1056 qr/^$/,
1057 qr/^psql:<stdin>:\d+: WARNING: zeroing out 2 invalid pages among blocks 1..4 of relation "base\/.*\/.*\nDETAIL: Block 2 held the first zeroed page\.\nHINT:[^\n]+$/
1058 );
1059
1060 # Then test zeroing via zero_damaged_pages
1061 psql_like(
1062 $io_method,
1063 $psql_a,
1064 "$persistency: test zeroing of invalid block 2,3 in larger read, zero_damaged_pages",
1065 qq(
1066 BEGIN;
1067 SET LOCAL zero_damaged_pages = true;
1068 SELECT read_rel_block_ll('tbl_zero', 1, nblocks=>4, zero_on_error=>false)
1069 COMMIT;
1070 ),
1071 qr/^$/,
1072 qr/^psql:<stdin>:\d+: WARNING: zeroing out 2 invalid pages among blocks 1..4 of relation "base\/.*\/.*\nDETAIL: Block 2 held the first zeroed page\.\nHINT:[^\n]+$/
1073 );
1074
1075 $psql_a->query_safe(qq(COMMIT));
1076
1077
1078 # Verify that bufmgr.c IO detects page validity errors
1079 $psql_a->query(
1080 qq(
1081 SELECT invalidate_rel_block('tbl_zero', g.i)
1082 FROM generate_series(0, 15) g(i);
1083 SELECT modify_rel_block('tbl_zero', 3, zero=>true);
1084 ));
1085 $psql_a->{stderr} = '';
1086
1087 psql_like(
1088 $io_method,
1089 $psql_a,
1090 "$persistency: verify reading zero_damaged_pages=off",
1091 qq(
1092 SELECT count(*) FROM tbl_zero),
1093 qr/^$/,
1094 qr/^psql:<stdin>:\d+: ERROR: invalid page in block 2 of relation "base\/.*\/.*$/
1095 );
1096
1097 # Verify that bufmgr.c IO zeroes out pages with page validity errors
1098 psql_like(
1099 $io_method,
1100 $psql_a,
1101 "$persistency: verify zero_damaged_pages=on",
1102 qq(
1103 BEGIN;
1104 SET LOCAL zero_damaged_pages = true;
1105 SELECT count(*) FROM tbl_zero;
1106 COMMIT;
1107 ),
1108 qr/^\d+$/,
1109 qr/^psql:<stdin>:\d+: WARNING: invalid page in block 2 of relation "base\/.*\/.*$/
1110 );
1111
1112 # Check that warnings/errors about page validity in an IO started by
1113 # session A that session B might complete aren't logged visibly to
1114 # session B.
1115 #
1116 # This will only ever trigger for io_method's like io_uring, that can
1117 # complete IO's in a client backend. But it doesn't seem worth
1118 # restricting to that.
1119 #
1120 # This requires cross-session access to the same relation, hence the
1121 # restriction to non-temporary table.
1122 if ($sql_persistency ne 'temporary')
1123 {
1124 # Create a corruption and then read the block without waiting for
1125 # completion.
1126 $psql_a->query(
1127 qq(
1128 SELECT modify_rel_block('tbl_zero', 1, corrupt_header=>true);
1129 SELECT read_rel_block_ll('tbl_zero', 1, wait_complete=>false, zero_on_error=>true)
1130 ));
1131
1132 psql_like(
1133 $io_method,
1134 $psql_b,
1135 "$persistency: test completing read by other session doesn't generate warning",
1136 qq(SELECT count(*) > 0 FROM tbl_zero;),
1137 qr/^t$/,
1138 qr/^$/);
1139 }
1140
1141 # Clean up
1142 $psql_a->query_safe(
1143 qq(
1144 DROP TABLE tbl_zero;
1145 ));
1146 }
1147
1148 $psql_a->{stderr} = '';
1149
1150 $psql_a->quit();
1151 $psql_b->quit();
1152 }
1153
1154 # Test that we detect checksum failures and report them
1155 sub test_checksum
1156 {
1157 my $io_method = shift;
1158 my $node = shift;
1159
1160 my $psql_a = $node->background_psql('postgres', on_error_stop => 0);
1161
1162 $psql_a->query_safe(
1163 qq(
1164 CREATE TABLE tbl_normal(id int) WITH (AUTOVACUUM_ENABLED = false);
1165 INSERT INTO tbl_normal SELECT generate_series(1, 5000);
1166 SELECT modify_rel_block('tbl_normal', 3, corrupt_checksum=>true);
1167
1168 CREATE TEMPORARY TABLE tbl_temp(id int) WITH (AUTOVACUUM_ENABLED = false);
1169 INSERT INTO tbl_temp SELECT generate_series(1, 5000);
1170 SELECT modify_rel_block('tbl_temp', 3, corrupt_checksum=>true);
1171 SELECT modify_rel_block('tbl_temp', 4, corrupt_checksum=>true);
1172 ));
1173
1174 # To be able to test checksum failures on shared rels we need a shared rel
1175 # with invalid pages - which is a bit scary. pg_shseclabel seems like a
1176 # good bet, as it's not accessed in a default configuration.
1177 $psql_a->query_safe(
1178 qq(
1179 SELECT grow_rel('pg_shseclabel', 4);
1180 SELECT modify_rel_block('pg_shseclabel', 2, corrupt_checksum=>true);
1181 SELECT modify_rel_block('pg_shseclabel', 3, corrupt_checksum=>true);
1182 ));
1183
1184
1185 # Check that page validity errors are detected, checksums stats increase, normal rel
1186 my ($cs_count_before, $cs_ts_before) =
1187 checksum_failures($psql_a, 'postgres');
1188 psql_like(
1189 $io_method,
1190 $psql_a,
1191 "normal rel: test reading of invalid block 3",
1192 qq(
1193 SELECT read_rel_block_ll('tbl_normal', 3, nblocks=>1, zero_on_error=>false);),
1194 qr/^$/,
1195 qr/^psql:<stdin>:\d+: ERROR: invalid page in block 3 of relation "base\/\d+\/\d+"$/
1196 );
1197
1198 my ($cs_count_after, $cs_ts_after) =
1199 checksum_failures($psql_a, 'postgres');
1200
1201 cmp_ok($cs_count_before + 1,
1202 '<=', $cs_count_after,
1203 "$io_method: normal rel: checksum count increased");
1204 cmp_ok($cs_ts_after, 'ne', '',
1205 "$io_method: normal rel: checksum timestamp is not null");
1206
1207
1208 # Check that page validity errors are detected, checksums stats increase, temp rel
1209 ($cs_count_after, $cs_ts_after) = checksum_failures($psql_a, 'postgres');
1210 psql_like(
1211 $io_method,
1212 $psql_a,
1213 "temp rel: test reading of invalid block 4, valid block 5",
1214 qq(
1215 SELECT read_rel_block_ll('tbl_temp', 4, nblocks=>2, zero_on_error=>false);),
1216 qr/^$/,
1217 qr/^psql:<stdin>:\d+: ERROR: invalid page in block 4 of relation "base\/\d+\/t\d+_\d+"$/
1218 );
1219
1220 ($cs_count_after, $cs_ts_after) = checksum_failures($psql_a, 'postgres');
1221
1222 cmp_ok($cs_count_before + 1,
1223 '<=', $cs_count_after,
1224 "$io_method: temp rel: checksum count increased");
1225 cmp_ok($cs_ts_after, 'ne', '',
1226 "$io_method: temp rel: checksum timestamp is not null");
1227
1228
1229 # Check that page validity errors are detected, checksums stats increase, shared rel
1230 ($cs_count_before, $cs_ts_after) = checksum_failures($psql_a);
1231 psql_like(
1232 $io_method,
1233 $psql_a,
1234 "shared rel: reading of invalid blocks 2+3",
1235 qq(
1236 SELECT read_rel_block_ll('pg_shseclabel', 2, nblocks=>2, zero_on_error=>false);),
1237 qr/^$/,
1238 qr/^psql:<stdin>:\d+: ERROR: 2 invalid pages among blocks 2..3 of relation "global\/\d+"\nDETAIL: Block 2 held the first invalid page\.\nHINT:[^\n]+$/
1239 );
1240
1241 ($cs_count_after, $cs_ts_after) = checksum_failures($psql_a);
1242
1243 cmp_ok($cs_count_before + 1,
1244 '<=', $cs_count_after,
1245 "$io_method: shared rel: checksum count increased");
1246 cmp_ok($cs_ts_after, 'ne', '',
1247 "$io_method: shared rel: checksum timestamp is not null");
1248
1249
1250 # and restore sanity
1251 $psql_a->query(
1252 qq(
1253 SELECT modify_rel_block('pg_shseclabel', 1, zero=>true);
1254 DROP TABLE tbl_normal;
1255 ));
1256 $psql_a->{stderr} = '';
1257
1258 $psql_a->quit();
1259 }
1260
1261 # Verify checksum handling when creating database from a database with an
1262 # invalid block. This also serves as a minimal check that cross-database IO is
1263 # handled reasonably.
1264 sub test_checksum_createdb
1265 {
1266 my $io_method = shift;
1267 my $node = shift;
1268
1269 my $psql = $node->background_psql('postgres', on_error_stop => 0);
1270
1271 $node->safe_psql('postgres',
1272 'CREATE DATABASE regression_createdb_source');
1273
1274 $node->safe_psql(
1275 'regression_createdb_source', qq(
1276 CREATE EXTENSION test_aio;
1277 CREATE TABLE tbl_cs_fail(data int not null) WITH (AUTOVACUUM_ENABLED = false);
1278 INSERT INTO tbl_cs_fail SELECT generate_series(1, 1000);
1279 SELECT modify_rel_block('tbl_cs_fail', 1, corrupt_checksum=>true);
1280 ));
1281
1282 my $createdb_sql = qq(
1283 CREATE DATABASE regression_createdb_target
1284 TEMPLATE regression_createdb_source
1285 STRATEGY wal_log;
1286 );
1287
1288 # Verify that CREATE DATABASE of an invalid database fails and is
1289 # accounted for accurately.
1290 #
1291 # Note: On windows additional WARNING messages might be printed, due to
1292 # "some useless files may be left behind" warnings. While we probably
1293 # should prevent those from occurring, they're independent of AIO, so we
1294 # shouldn't fail because of them here.
1295 my ($cs_count_before, $cs_ts_before) =
1296 checksum_failures($psql, 'regression_createdb_source');
1297 psql_like(
1298 $io_method,
1299 $psql,
1300 "create database w/ wal strategy, invalid source",
1301 $createdb_sql,
1302 qr/^$/,
1303 qr/psql:<stdin>:\d+: ERROR: invalid page in block 1 of relation "base\/\d+\/\d+"$/
1304 );
1305 my ($cs_count_after, $cs_ts_after) =
1306 checksum_failures($psql, 'regression_createdb_source');
1307 cmp_ok($cs_count_before + 1, '<=', $cs_count_after,
1308 "$io_method: create database w/ wal strategy, invalid source: checksum count increased"
1309 );
1310
1311 # Verify that CREATE DATABASE of the fixed database succeeds.
1312 $node->safe_psql(
1313 'regression_createdb_source', qq(
1314 SELECT modify_rel_block('tbl_cs_fail', 1, zero=>true);
1315 ));
1316 psql_like($io_method, $psql,
1317 "create database w/ wal strategy, valid source",
1318 $createdb_sql, qr/^$/, qr/^$/);
1319
1320 $psql->quit();
1321 }
1322
1323 # Test that we detect checksum failures and report them
1324 #
1325 # In several places we make sure that the server log actually contains
1326 # individual information for each block involved in the IO.
1327 sub test_ignore_checksum
1328 {
1329 my $io_method = shift;
1330 my $node = shift;
1331
1332 my $psql = $node->background_psql('postgres', on_error_stop => 0);
1333
1334 # Test setup
1335 $psql->query_safe(
1336 qq(
1337 CREATE TABLE tbl_cs_fail(id int) WITH (AUTOVACUUM_ENABLED = false);
1338 INSERT INTO tbl_cs_fail SELECT generate_series(1, 10000);
1339 ));
1340
1341 my $count_sql = "SELECT count(*) FROM tbl_cs_fail";
1342 my $invalidate_sql = qq(
1343 SELECT invalidate_rel_block('tbl_cs_fail', g.i)
1344 FROM generate_series(0, 6) g(i);
1345 );
1346
1347 my $expect = $psql->query_safe($count_sql);
1348
1349
1350 # Very basic tests for ignore_checksum_failure=off / on
1351
1352 $psql->query_safe(
1353 qq(
1354 SELECT modify_rel_block('tbl_cs_fail', 1, corrupt_checksum=>true);
1355 SELECT modify_rel_block('tbl_cs_fail', 5, corrupt_checksum=>true);
1356 SELECT modify_rel_block('tbl_cs_fail', 6, corrupt_checksum=>true);
1357 ));
1358
1359 $psql->query_safe($invalidate_sql);
1360 psql_like(
1361 $io_method,
1362 $psql,
1363 "reading block w/ wrong checksum with ignore_checksum_failure=off fails",
1364 $count_sql,
1365 qr/^$/,
1366 qr/ERROR: invalid page in block/);
1367
1368 $psql->query_safe("SET ignore_checksum_failure=on");
1369
1370 $psql->query_safe($invalidate_sql);
1371 psql_like(
1372 $io_method,
1373 $psql,
1374 "reading block w/ wrong checksum with ignore_checksum_failure=off succeeds",
1375 $count_sql,
1376 qr/^$expect$/,
1377 qr/WARNING: ignoring (checksum failure|\d checksum failures)/);
1378
1379
1380 # Verify that ignore_checksum_failure=off works in multi-block reads
1381
1382 $psql->query_safe(
1383 qq(
1384 SELECT modify_rel_block('tbl_cs_fail', 2, zero=>true);
1385 SELECT modify_rel_block('tbl_cs_fail', 3, corrupt_checksum=>true);
1386 SELECT modify_rel_block('tbl_cs_fail', 4, corrupt_header=>true);
1387 ));
1388
1389 my $log_location = -s $node->logfile;
1390 psql_like(
1391 $io_method,
1392 $psql,
1393 "test reading of checksum failed block 3, with ignore",
1394 qq(
1395 SELECT read_rel_block_ll('tbl_cs_fail', 3, nblocks=>1, zero_on_error=>false);),
1396 qr/^$/,
1397 qr/^psql:<stdin>:\d+: WARNING: ignoring checksum failure in block 3/
1398 );
1399
1400 # Check that the log contains a LOG message about the failure
1401 $log_location =
1402 $node->wait_for_log(qr/LOG: ignoring checksum failure/, $log_location);
1403
1404 # check that we error
1405 psql_like(
1406 $io_method,
1407 $psql,
1408 "test reading of valid block 2, checksum failed 3, invalid 4, zero=false with ignore",
1409 qq(
1410 SELECT read_rel_block_ll('tbl_cs_fail', 2, nblocks=>3, zero_on_error=>false);),
1411 qr/^$/,
1412 qr/^psql:<stdin>:\d+: ERROR: invalid page in block 4 of relation "base\/\d+\/\d+"$/
1413 );
1414
1415 # Test multi-block read with different problems in different blocks
1416 $psql->query(
1417 qq(
1418 SELECT modify_rel_block('tbl_cs_fail', 1, zero=>true);
1419 SELECT modify_rel_block('tbl_cs_fail', 2, corrupt_checksum=>true);
1420 SELECT modify_rel_block('tbl_cs_fail', 3, corrupt_checksum=>true, corrupt_header=>true);
1421 SELECT modify_rel_block('tbl_cs_fail', 4, corrupt_header=>true);
1422 SELECT modify_rel_block('tbl_cs_fail', 5, corrupt_header=>true);
1423 ));
1424 $psql->{stderr} = '';
1425
1426 $log_location = -s $node->logfile;
1427 psql_like(
1428 $io_method,
1429 $psql,
1430 "test reading of valid block 1, checksum failed 2, 3, invalid 3-5, zero=true",
1431 qq(
1432 SELECT read_rel_block_ll('tbl_cs_fail', 1, nblocks=>5, zero_on_error=>true);),
1433 qr/^$/,
1434 qr/^psql:<stdin>:\d+: WARNING: zeroing 3 page\(s\) and ignoring 2 checksum failure\(s\) among blocks 1..5 of relation "/
1435 );
1436
1437
1438 # Unfortunately have to scan the whole log since determining $log_location
1439 # above in each of the tests, as wait_for_log() returns the size of the
1440 # file.
1441
1442 $node->wait_for_log(qr/LOG: ignoring checksum failure in block 2/,
1443 $log_location);
1444 ok(1, "$io_method: found information about checksum failure in block 2");
1445
1446 $node->wait_for_log(
1447 qr/LOG: invalid page in block 3 of relation "base.*"; zeroing out page/,
1448 $log_location);
1449 ok(1, "$io_method: found information about invalid page in block 3");
1450
1451 $node->wait_for_log(
1452 qr/LOG: invalid page in block 4 of relation "base.*"; zeroing out page/,
1453 $log_location);
1454 ok(1, "$io_method: found information about checksum failure in block 4");
1455
1456 $node->wait_for_log(
1457 qr/LOG: invalid page in block 5 of relation "base.*"; zeroing out page/,
1458 $log_location);
1459 ok(1, "$io_method: found information about checksum failure in block 5");
1460
1461
1462 # Reading a page with both an invalid header and an invalid checksum
1463 $psql->query(
1464 qq(
1465 SELECT modify_rel_block('tbl_cs_fail', 3, corrupt_checksum=>true, corrupt_header=>true);
1466 ));
1467 $psql->{stderr} = '';
1468
1469 psql_like(
1470 $io_method,
1471 $psql,
1472 "test reading of block with both invalid header and invalid checksum, zero=false",
1473 qq(
1474 SELECT read_rel_block_ll('tbl_cs_fail', 3, nblocks=>1, zero_on_error=>false);),
1475 qr/^$/,
1476 qr/^psql:<stdin>:\d+: ERROR: invalid page in block 3 of relation "/);
1477
1478 psql_like(
1479 $io_method,
1480 $psql,
1481 "test reading of block 3 with both invalid header and invalid checksum, zero=true",
1482 qq(
1483 SELECT read_rel_block_ll('tbl_cs_fail', 3, nblocks=>1, zero_on_error=>true);),
1484 qr/^$/,
1485 qr/^psql:<stdin>:\d+: WARNING: invalid page in block 3 of relation "base\/.*"; zeroing out page/
1486 );
1487
1488
1489 $psql->quit();
1490 }
1491
1492
1493 # Run all tests that are supported for all io_methods
1494 sub test_generic
1495 {
1496 my $io_method = shift;
1497 my $node = shift;
1498
1499 is($node->safe_psql('postgres', 'SHOW io_method'),
1500 $io_method, "$io_method: io_method set correctly");
1501
1502 $node->safe_psql(
1503 'postgres', qq(
1504 CREATE EXTENSION test_aio;
1505 CREATE TABLE tbl_corr(data int not null) WITH (AUTOVACUUM_ENABLED = false);
1506 CREATE TABLE tbl_ok(data int not null) WITH (AUTOVACUUM_ENABLED = false);
1507
1508 INSERT INTO tbl_corr SELECT generate_series(1, 10000);
1509 INSERT INTO tbl_ok SELECT generate_series(1, 10000);
1510 SELECT grow_rel('tbl_corr', 16);
1511 SELECT grow_rel('tbl_ok', 16);
1512
1513 SELECT modify_rel_block('tbl_corr', 1, corrupt_header=>true);
1514 CHECKPOINT;
1515 ));
1516
1517 test_handle($io_method, $node);
1518 test_io_error($io_method, $node);
1519 test_batchmode($io_method, $node);
1520 test_startwait_io($io_method, $node);
1521 test_complete_foreign($io_method, $node);
1522 test_close_fd($io_method, $node);
1523 test_invalidate($io_method, $node);
1524 test_zero($io_method, $node);
1525 test_checksum($io_method, $node);
1526 test_ignore_checksum($io_method, $node);
1527 test_checksum_createdb($io_method, $node);
1528
1529 SKIP:
1530 {
1531 skip 'Injection points not supported by this build', 1
1532 unless $ENV{enable_injection_points} eq 'yes';
1533 test_inject($io_method, $node);
1534 }
1535 }