Debian Codegraph @

src/test/recovery/t/029_stats_restart.pl

raw
1 # Copyright (c) 2021-2025, PostgreSQL Global Development Group
2
3 # Tests statistics handling around restarts, including handling of crashes and
4 # invalid stats files, as well as restoring stats after "normal" restarts.
5
6 use strict;
7 use warnings FATAL => 'all';
8 use PostgreSQL::Test::Cluster;
9 use PostgreSQL::Test::Utils;
10 use Test::More;
11 use File::Copy;
12
13 my $node = PostgreSQL::Test::Cluster->new('primary');
14 $node->init(allows_streaming => 1);
15 $node->append_conf('postgresql.conf', "track_functions = 'all'");
16 $node->start;
17
18 my $connect_db = 'postgres';
19 my $db_under_test = 'test';
20
21 my $sect = "startup";
22
23 # Check some WAL statistics after a fresh startup. The startup process
24 # should have done WAL reads, and initialization some WAL writes.
25 my $standalone_io_stats = io_stats('init', 'wal', 'standalone backend');
26 my $startup_io_stats = io_stats('normal', 'wal', 'startup');
27 cmp_ok(
28 '0', '<',
29 $standalone_io_stats->{writes},
30 "$sect: increased standalone backend IO writes");
31 cmp_ok(
32 '0', '<',
33 $startup_io_stats->{reads},
34 "$sect: increased startup IO reads");
35
36 # create test objects
37 $node->safe_psql($connect_db, "CREATE DATABASE $db_under_test");
38 $node->safe_psql($db_under_test,
39 "CREATE TABLE tab_stats_crash_discard_test1 AS SELECT generate_series(1,100) AS a"
40 );
41 $node->safe_psql($db_under_test,
42 "CREATE FUNCTION func_stats_crash_discard1() RETURNS VOID AS 'select 2;' LANGUAGE SQL IMMUTABLE"
43 );
44
45 # collect object oids
46 my $dboid = $node->safe_psql($db_under_test,
47 "SELECT oid FROM pg_database WHERE datname = '$db_under_test'");
48 my $funcoid = $node->safe_psql($db_under_test,
49 "SELECT 'func_stats_crash_discard1()'::regprocedure::oid");
50 my $tableoid = $node->safe_psql($db_under_test,
51 "SELECT 'tab_stats_crash_discard_test1'::regclass::oid");
52
53 # generate stats and flush them
54 trigger_funcrel_stat();
55
56 # verify stats objects exist
57 $sect = "initial";
58 is(have_stats('database', $dboid, 0), 't', "$sect: db stats do exist");
59 is(have_stats('function', $dboid, $funcoid),
60 't', "$sect: function stats do exist");
61 is(have_stats('relation', $dboid, $tableoid),
62 't', "$sect: relation stats do exist");
63
64 # regular shutdown
65 $node->stop();
66
67 # backup stats files
68 my $statsfile = $PostgreSQL::Test::Utils::tmp_check . '/' . "discard_stats1";
69 ok(!-f "$statsfile", "backup statsfile cannot already exist");
70
71 my $datadir = $node->data_dir();
72 my $og_stats = "$datadir/pg_stat/pgstat.stat";
73 ok(-f "$og_stats", "origin stats file must exist");
74 copy($og_stats, $statsfile) or die "Copy failed: $!";
75
76
77 ## test discarding of stats file after crash etc
78
79 $node->start;
80
81 $sect = "copy";
82 is(have_stats('database', $dboid, 0), 't', "$sect: db stats do exist");
83 is(have_stats('function', $dboid, $funcoid),
84 't', "$sect: function stats do exist");
85 is(have_stats('relation', $dboid, $tableoid),
86 't', "$sect: relation stats do exist");
87
88 $node->stop('immediate');
89
90 ok(!-f "$og_stats", "no stats file should exist after immediate shutdown");
91
92 # copy the old stats back to test we discard stats after crash restart
93 copy($statsfile, $og_stats) or die "Copy failed: $!";
94
95 $node->start;
96
97 # stats should have been discarded
98 $sect = "post immediate";
99 is(have_stats('database', $dboid, 0), 'f', "$sect: db stats do not exist");
100 is(have_stats('function', $dboid, $funcoid),
101 'f', "$sect: function stats do exist");
102 is(have_stats('relation', $dboid, $tableoid),
103 'f', "$sect: relation stats do not exist");
104
105 # get rid of backup statsfile
106 unlink $statsfile or die "cannot unlink $statsfile $!";
107
108
109 # generate new stats and flush them
110 trigger_funcrel_stat();
111
112 $sect = "post immediate, new";
113 is(have_stats('database', $dboid, 0), 't', "$sect: db stats do exist");
114 is(have_stats('function', $dboid, $funcoid),
115 't', "$sect: function stats do exist");
116 is(have_stats('relation', $dboid, $tableoid),
117 't', "$sect: relation stats do exist");
118
119 # regular shutdown
120 $node->stop();
121
122
123 ## check an invalid stats file is handled
124
125 overwrite_file($og_stats, "ZZZZZZZZZZZZZ");
126
127 # normal startup and no issues despite invalid stats file
128 $node->start;
129
130 # no stats present due to invalid stats file
131 $sect = "invalid_overwrite";
132 is(have_stats('database', $dboid, 0), 'f', "$sect: db stats do not exist");
133 is(have_stats('function', $dboid, $funcoid),
134 'f', "$sect: function stats do not exist");
135 is(have_stats('relation', $dboid, $tableoid),
136 'f', "$sect: relation stats do not exist");
137
138
139 ## check invalid stats file starting with valid contents, but followed by
140 ## invalid content is handled.
141
142 trigger_funcrel_stat();
143 $node->stop;
144 append_file($og_stats, "XYZ");
145 $node->start;
146
147 $sect = "invalid_append";
148 is(have_stats('database', $dboid, 0), 'f', "$sect: db stats do not exist");
149 is(have_stats('function', $dboid, $funcoid),
150 'f', "$sect: function stats do not exist");
151 is(have_stats('relation', $dboid, $tableoid),
152 'f', "$sect: relation stats do not exist");
153
154
155 ## checks related to stats persistency around restarts and resets
156
157 # Ensure enough checkpoints to protect against races for test after reset,
158 # even on very slow machines.
159 $node->safe_psql($connect_db, "CHECKPOINT; CHECKPOINT;");
160
161
162 ## check checkpoint and wal stats are incremented due to restart
163
164 my $ckpt_start = checkpoint_stats();
165 my $wal_start = wal_stats();
166 $node->restart;
167
168 $sect = "post restart";
169 my $ckpt_restart = checkpoint_stats();
170 my $wal_restart = wal_stats();
171
172 cmp_ok(
173 $ckpt_start->{count}, '<',
174 $ckpt_restart->{count},
175 "$sect: increased checkpoint count");
176 cmp_ok(
177 $wal_start->{records}, '<',
178 $wal_restart->{records},
179 "$sect: increased wal record count");
180 cmp_ok($wal_start->{bytes}, '<', $wal_restart->{bytes},
181 "$sect: increased wal bytes");
182 is( $ckpt_start->{reset},
183 $ckpt_restart->{reset},
184 "$sect: checkpoint stats_reset equal");
185 is($wal_start->{reset}, $wal_restart->{reset},
186 "$sect: wal stats_reset equal");
187
188
189 ## Check that checkpoint stats are reset, WAL stats aren't affected
190
191 $node->safe_psql($connect_db, "SELECT pg_stat_reset_shared('checkpointer')");
192
193 $sect = "post ckpt reset";
194 my $ckpt_reset = checkpoint_stats();
195 my $wal_ckpt_reset = wal_stats();
196
197 cmp_ok($ckpt_restart->{count},
198 '>', $ckpt_reset->{count}, "$sect: checkpoint count smaller");
199 cmp_ok($ckpt_start->{reset}, 'lt', $ckpt_reset->{reset},
200 "$sect: stats_reset newer");
201
202 cmp_ok(
203 $wal_restart->{records},
204 '<=',
205 $wal_ckpt_reset->{records},
206 "$sect: wal record count not affected by reset");
207 is( $wal_start->{reset},
208 $wal_ckpt_reset->{reset},
209 "$sect: wal stats_reset equal");
210
211
212 ## check that checkpoint stats stay reset after restart
213
214 $node->restart;
215
216 $sect = "post ckpt reset & restart";
217 my $ckpt_restart_reset = checkpoint_stats();
218 my $wal_restart2 = wal_stats();
219
220 # made sure above there's enough checkpoints that this will be stable even on slow machines
221 cmp_ok(
222 $ckpt_restart_reset->{count},
223 '<',
224 $ckpt_restart->{count},
225 "$sect: checkpoint still reset");
226 is($ckpt_restart_reset->{reset},
227 $ckpt_reset->{reset}, "$sect: stats_reset same");
228
229 cmp_ok(
230 $wal_ckpt_reset->{records},
231 '<',
232 $wal_restart2->{records},
233 "$sect: increased wal record count");
234 cmp_ok(
235 $wal_ckpt_reset->{bytes},
236 '<',
237 $wal_restart2->{bytes},
238 "$sect: increased wal bytes");
239 is( $wal_start->{reset},
240 $wal_restart2->{reset},
241 "$sect: wal stats_reset equal");
242
243
244 ## check WAL stats stay reset
245
246 $node->safe_psql($connect_db, "SELECT pg_stat_reset_shared('wal')");
247
248 $sect = "post wal reset";
249 my $wal_reset = wal_stats();
250
251 cmp_ok(
252 $wal_reset->{records}, '<',
253 $wal_restart2->{records},
254 "$sect: smaller record count");
255 cmp_ok(
256 $wal_reset->{bytes}, '<',
257 $wal_restart2->{bytes},
258 "$sect: smaller bytes");
259 cmp_ok(
260 $wal_reset->{reset}, 'gt',
261 $wal_restart2->{reset},
262 "$sect: newer stats_reset");
263
264 $node->restart;
265
266 $sect = "post wal reset & restart";
267 my $wal_reset_restart = wal_stats();
268
269 # enough WAL generated during prior tests and initdb to make this not racy
270 cmp_ok(
271 $wal_reset_restart->{records},
272 '<',
273 $wal_restart2->{records},
274 "$sect: smaller record count");
275 cmp_ok(
276 $wal_reset->{bytes}, '<',
277 $wal_restart2->{bytes},
278 "$sect: smaller bytes");
279 cmp_ok(
280 $wal_reset->{reset}, 'gt',
281 $wal_restart2->{reset},
282 "$sect: newer stats_reset");
283
284 $node->stop('immediate');
285 $node->start;
286
287 $sect = "post immediate restart";
288 my $wal_restart_immediate = wal_stats();
289
290 cmp_ok(
291 $wal_reset_restart->{reset},
292 'lt',
293 $wal_restart_immediate->{reset},
294 "$sect: reset timestamp is new");
295
296 $node->stop;
297 done_testing();
298
299 sub trigger_funcrel_stat
300 {
301 $node->safe_psql(
302 $db_under_test, q[
303 SELECT * FROM tab_stats_crash_discard_test1;
304 SELECT func_stats_crash_discard1();
305 SELECT pg_stat_force_next_flush();]);
306 }
307
308 sub have_stats
309 {
310 my ($kind, $dboid, $objid) = @_;
311
312 return $node->safe_psql($connect_db,
313 "SELECT pg_stat_have_stats('$kind', $dboid, $objid)");
314 }
315
316 sub overwrite_file
317 {
318 my ($filename, $str) = @_;
319 open my $fh, ">", $filename
320 or die "could not overwrite \"$filename\": $!";
321 print $fh $str;
322 close $fh;
323 return;
324 }
325
326 sub append_file
327 {
328 my ($filename, $str) = @_;
329 open my $fh, ">>", $filename
330 or die "could not append to \"$filename\": $!";
331 print $fh $str;
332 close $fh;
333 return;
334 }
335
336 sub checkpoint_stats
337 {
338 my %results;
339
340 $results{count} = $node->safe_psql($connect_db,
341 "SELECT num_timed + num_requested FROM pg_stat_checkpointer");
342 $results{reset} = $node->safe_psql($connect_db,
343 "SELECT stats_reset FROM pg_stat_checkpointer");
344
345 return \%results;
346 }
347
348 sub wal_stats
349 {
350 my %results;
351 $results{records} =
352 $node->safe_psql($connect_db, "SELECT wal_records FROM pg_stat_wal");
353 $results{bytes} =
354 $node->safe_psql($connect_db, "SELECT wal_bytes FROM pg_stat_wal");
355 $results{reset} =
356 $node->safe_psql($connect_db, "SELECT stats_reset FROM pg_stat_wal");
357
358 return \%results;
359 }
360
361 sub io_stats
362 {
363 my ($context, $object, $backend_type) = @_;
364 my %results;
365
366 $results{writes} = $node->safe_psql(
367 $connect_db, qq{SELECT writes FROM pg_stat_io
368 WHERE context = '$context' AND object = '$object' AND
369 backend_type = '$backend_type'});
370 $results{reads} = $node->safe_psql(
371 $connect_db, qq{SELECT reads FROM pg_stat_io
372 WHERE context = '$context' AND object = '$object' AND
373 backend_type = '$backend_type'});
374
375 return \%results;
376 }