Debian Codegraph @

src/backend/backup/basebackup.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * basebackup.c
4 * code for taking a base backup and streaming it to a standby
5 *
6 * Portions Copyright (c) 2010-2025, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/backup/basebackup.c
10 *
11 *-------------------------------------------------------------------------
12 */
13 #include "postgres.h"
14
15 #include <sys/stat.h>
16 #include <unistd.h>
17 #include <time.h>
18
19 #include "access/xlog_internal.h"
20 #include "access/xlogbackup.h"
21 #include "backup/backup_manifest.h"
22 #include "backup/basebackup.h"
23 #include "backup/basebackup_incremental.h"
24 #include "backup/basebackup_sink.h"
25 #include "backup/basebackup_target.h"
26 #include "catalog/pg_tablespace_d.h"
27 #include "commands/defrem.h"
28 #include "common/compression.h"
29 #include "common/file_perm.h"
30 #include "common/file_utils.h"
31 #include "lib/stringinfo.h"
32 #include "miscadmin.h"
33 #include "nodes/pg_list.h"
34 #include "pgstat.h"
35 #include "pgtar.h"
36 #include "postmaster/syslogger.h"
37 #include "postmaster/walsummarizer.h"
38 #include "replication/slot.h"
39 #include "replication/walsender.h"
40 #include "replication/walsender_private.h"
41 #include "storage/bufpage.h"
42 #include "storage/checksum.h"
43 #include "storage/dsm_impl.h"
44 #include "storage/ipc.h"
45 #include "storage/reinit.h"
46 #include "utils/builtins.h"
47 #include "utils/guc.h"
48 #include "utils/ps_status.h"
49 #include "utils/relcache.h"
50 #include "utils/resowner.h"
51
52 /*
53 * How much data do we want to send in one CopyData message? Note that
54 * this may also result in reading the underlying files in chunks of this
55 * size.
56 *
57 * NB: The buffer size is required to be a multiple of the system block
58 * size, so use that value instead if it's bigger than our preference.
59 */
60 #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
61
62 typedef struct
63 {
64 const char *label;
65 bool progress;
66 bool fastcheckpoint;
67 bool nowait;
68 bool includewal;
69 bool incremental;
70 uint32 maxrate;
71 bool sendtblspcmapfile;
72 bool send_to_client;
73 bool use_copytblspc;
74 BaseBackupTargetHandle *target_handle;
75 backup_manifest_option manifest;
76 pg_compress_algorithm compression;
77 pg_compress_specification compression_specification;
78 pg_checksum_type manifest_checksum_type;
79 } basebackup_options;
80
81 static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
82 struct backup_manifest_info *manifest,
83 IncrementalBackupInfo *ib);
84 static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
85 List *tablespaces, bool sendtblspclinks,
86 backup_manifest_info *manifest, Oid spcoid,
87 IncrementalBackupInfo *ib);
88 static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
89 struct stat *statbuf, bool missing_ok,
90 Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
91 unsigned segno,
92 backup_manifest_info *manifest,
93 unsigned num_incremental_blocks,
94 BlockNumber *incremental_blocks,
95 unsigned truncation_block_length);
96 static off_t read_file_data_into_buffer(bbsink *sink,
97 const char *readfilename, int fd,
98 off_t offset, size_t length,
99 BlockNumber blkno,
100 bool verify_checksum,
101 int *checksum_failures);
102 static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
103 size_t *bytes_done, void *data, size_t length);
104 static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
105 BlockNumber blkno,
106 uint16 *expected_checksum);
107 static void sendFileWithContent(bbsink *sink, const char *filename,
108 const char *content, int len,
109 backup_manifest_info *manifest);
110 static int64 _tarWriteHeader(bbsink *sink, const char *filename,
111 const char *linktarget, struct stat *statbuf,
112 bool sizeonly);
113 static void _tarWritePadding(bbsink *sink, int len);
114 static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
115 static void perform_base_backup(basebackup_options *opt, bbsink *sink,
116 IncrementalBackupInfo *ib);
117 static void parse_basebackup_options(List *options, basebackup_options *opt);
118 static int compareWalFileNames(const ListCell *a, const ListCell *b);
119 static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
120 const char *filename, bool partial_read_ok);
121
122 /* Was the backup currently in-progress initiated in recovery mode? */
123 static bool backup_started_in_recovery = false;
124
125 /* Total number of checksum failures during base backup. */
126 static long long int total_checksum_failures;
127
128 /* Do not verify checksums. */
129 static bool noverify_checksums = false;
130
131 /*
132 * Definition of one element part of an exclusion list, used for paths part
133 * of checksum validation or base backups. "name" is the name of the file
134 * or path to check for exclusion. If "match_prefix" is true, any items
135 * matching the name as prefix are excluded.
136 */
137 struct exclude_list_item
138 {
139 const char *name;
140 bool match_prefix;
141 };
142
143 /*
144 * The contents of these directories are removed or recreated during server
145 * start so they are not included in backups. The directories themselves are
146 * kept and included as empty to preserve access permissions.
147 *
148 * Note: this list should be kept in sync with the filter lists in pg_rewind's
149 * filemap.c.
150 */
151 static const char *const excludeDirContents[] =
152 {
153 /*
154 * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
155 * because extensions like pg_stat_statements store data there.
156 */
157 PG_STAT_TMP_DIR,
158
159 /*
160 * It is generally not useful to backup the contents of this directory
161 * even if the intention is to restore to another primary. See backup.sgml
162 * for a more detailed description.
163 */
164 PG_REPLSLOT_DIR,
165
166 /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
167 PG_DYNSHMEM_DIR,
168
169 /* Contents removed on startup, see AsyncShmemInit(). */
170 "pg_notify",
171
172 /*
173 * Old contents are loaded for possible debugging but are not required for
174 * normal operation, see SerialInit().
175 */
176 "pg_serial",
177
178 /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
179 "pg_snapshots",
180
181 /* Contents zeroed on startup, see StartupSUBTRANS(). */
182 "pg_subtrans",
183
184 /* end of list */
185 NULL
186 };
187
188 /*
189 * List of files excluded from backups.
190 */
191 static const struct exclude_list_item excludeFiles[] =
192 {
193 /* Skip auto conf temporary file. */
194 {PG_AUTOCONF_FILENAME ".tmp", false},
195
196 /* Skip current log file temporary file */
197 {LOG_METAINFO_DATAFILE_TMP, false},
198
199 /*
200 * Skip relation cache because it is rebuilt on startup. This includes
201 * temporary files.
202 */
203 {RELCACHE_INIT_FILENAME, true},
204
205 /*
206 * backup_label and tablespace_map should not exist in a running cluster
207 * capable of doing an online backup, but exclude them just in case.
208 */
209 {BACKUP_LABEL_FILE, false},
210 {TABLESPACE_MAP, false},
211
212 /*
213 * If there's a backup_manifest, it belongs to a backup that was used to
214 * start this server. It is *not* correct for this backup. Our
215 * backup_manifest is injected into the backup separately if users want
216 * it.
217 */
218 {"backup_manifest", false},
219
220 {"postmaster.pid", false},
221 {"postmaster.opts", false},
222
223 /* end of list */
224 {NULL, false}
225 };
226
227 /*
228 * Actually do a base backup for the specified tablespaces.
229 *
230 * This is split out mainly to avoid complaints about "variable might be
231 * clobbered by longjmp" from stupider versions of gcc.
232 */
233 static void
234 perform_base_backup(basebackup_options *opt, bbsink *sink,
235 IncrementalBackupInfo *ib)
236 {
237 bbsink_state state;
238 XLogRecPtr endptr;
239 TimeLineID endtli;
240 backup_manifest_info manifest;
241 BackupState *backup_state;
242 StringInfo tablespace_map;
243
244 /* Initial backup state, insofar as we know it now. */
245 state.tablespaces = NIL;
246 state.tablespace_num = 0;
247 state.bytes_done = 0;
248 state.bytes_total = 0;
249 state.bytes_total_is_valid = false;
250
251 /* we're going to use a BufFile, so we need a ResourceOwner */
252 Assert(AuxProcessResourceOwner != NULL);
253 Assert(CurrentResourceOwner == AuxProcessResourceOwner ||
254 CurrentResourceOwner == NULL);
255 CurrentResourceOwner = AuxProcessResourceOwner;
256
257 backup_started_in_recovery = RecoveryInProgress();
258
259 InitializeBackupManifest(&manifest, opt->manifest,
260 opt->manifest_checksum_type);
261
262 total_checksum_failures = 0;
263
264 /* Allocate backup related variables. */
265 backup_state = (BackupState *) palloc0(sizeof(BackupState));
266 tablespace_map = makeStringInfo();
267
268 basebackup_progress_wait_checkpoint();
269 do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
270 backup_state, tablespace_map);
271
272 state.startptr = backup_state->startpoint;
273 state.starttli = backup_state->starttli;
274
275 /*
276 * Once do_pg_backup_start has been called, ensure that any failure causes
277 * us to abort the backup so we don't "leak" a backup counter. For this
278 * reason, *all* functionality between do_pg_backup_start() and the end of
279 * do_pg_backup_stop() should be inside the error cleanup block!
280 */
281
282 PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
283 {
284 ListCell *lc;
285 tablespaceinfo *newti;
286
287 /* If this is an incremental backup, execute preparatory steps. */
288 if (ib != NULL)
289 PrepareForIncrementalBackup(ib, backup_state);
290
291 /* Add a node for the base directory at the end */
292 newti = palloc0(sizeof(tablespaceinfo));
293 newti->size = -1;
294 state.tablespaces = lappend(state.tablespaces, newti);
295
296 /*
297 * Calculate the total backup size by summing up the size of each
298 * tablespace
299 */
300 if (opt->progress)
301 {
302 basebackup_progress_estimate_backup_size();
303
304 foreach(lc, state.tablespaces)
305 {
306 tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
307
308 if (tmp->path == NULL)
309 tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
310 true, NULL, InvalidOid, NULL);
311 else
312 tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
313 NULL, NULL);
314 state.bytes_total += tmp->size;
315 }
316 state.bytes_total_is_valid = true;
317 }
318
319 /* notify basebackup sink about start of backup */
320 bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
321
322 /* Send off our tablespaces one by one */
323 foreach(lc, state.tablespaces)
324 {
325 tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
326
327 if (ti->path == NULL)
328 {
329 struct stat statbuf;
330 bool sendtblspclinks = true;
331 char *backup_label;
332
333 bbsink_begin_archive(sink, "base.tar");
334
335 /* In the main tar, include the backup_label first... */
336 backup_label = build_backup_content(backup_state, false);
337 sendFileWithContent(sink, BACKUP_LABEL_FILE,
338 backup_label, -1, &manifest);
339 pfree(backup_label);
340
341 /* Then the tablespace_map file, if required... */
342 if (opt->sendtblspcmapfile)
343 {
344 sendFileWithContent(sink, TABLESPACE_MAP,
345 tablespace_map->data, -1, &manifest);
346 sendtblspclinks = false;
347 }
348
349 /* Then the bulk of the files... */
350 sendDir(sink, ".", 1, false, state.tablespaces,
351 sendtblspclinks, &manifest, InvalidOid, ib);
352
353 /* ... and pg_control after everything else. */
354 if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
355 ereport(ERROR,
356 (errcode_for_file_access(),
357 errmsg("could not stat file \"%s\": %m",
358 XLOG_CONTROL_FILE)));
359 sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
360 false, InvalidOid, InvalidOid,
361 InvalidRelFileNumber, 0, &manifest, 0, NULL, 0);
362 }
363 else
364 {
365 char *archive_name = psprintf("%u.tar", ti->oid);
366
367 bbsink_begin_archive(sink, archive_name);
368
369 sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
370 }
371
372 /*
373 * If we're including WAL, and this is the main data directory we
374 * don't treat this as the end of the tablespace. Instead, we will
375 * include the xlog files below and stop afterwards. This is safe
376 * since the main data directory is always sent *last*.
377 */
378 if (opt->includewal && ti->path == NULL)
379 {
380 Assert(lnext(state.tablespaces, lc) == NULL);
381 }
382 else
383 {
384 /* Properly terminate the tarfile. */
385 StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
386 "BLCKSZ too small for 2 tar blocks");
387 memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
388 bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
389
390 /* OK, that's the end of the archive. */
391 bbsink_end_archive(sink);
392 }
393 }
394
395 basebackup_progress_wait_wal_archive(&state);
396 do_pg_backup_stop(backup_state, !opt->nowait);
397
398 endptr = backup_state->stoppoint;
399 endtli = backup_state->stoptli;
400
401 /* Deallocate backup-related variables. */
402 destroyStringInfo(tablespace_map);
403 pfree(backup_state);
404 }
405 PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
406
407
408 if (opt->includewal)
409 {
410 /*
411 * We've left the last tar file "open", so we can now append the
412 * required WAL files to it.
413 */
414 char pathbuf[MAXPGPATH];
415 XLogSegNo segno;
416 XLogSegNo startsegno;
417 XLogSegNo endsegno;
418 struct stat statbuf;
419 List *historyFileList = NIL;
420 List *walFileList = NIL;
421 char firstoff[MAXFNAMELEN];
422 char lastoff[MAXFNAMELEN];
423 DIR *dir;
424 struct dirent *de;
425 ListCell *lc;
426 TimeLineID tli;
427
428 basebackup_progress_transfer_wal();
429
430 /*
431 * I'd rather not worry about timelines here, so scan pg_wal and
432 * include all WAL files in the range between 'startptr' and 'endptr',
433 * regardless of the timeline the file is stamped with. If there are
434 * some spurious WAL files belonging to timelines that don't belong in
435 * this server's history, they will be included too. Normally there
436 * shouldn't be such files, but if there are, there's little harm in
437 * including them.
438 */
439 XLByteToSeg(state.startptr, startsegno, wal_segment_size);
440 XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
441 XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
442 XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
443
444 dir = AllocateDir("pg_wal");
445 while ((de = ReadDir(dir, "pg_wal")) != NULL)
446 {
447 /* Does it look like a WAL segment, and is it in the range? */
448 if (IsXLogFileName(de->d_name) &&
449 strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
450 strcmp(de->d_name + 8, lastoff + 8) <= 0)
451 {
452 walFileList = lappend(walFileList, pstrdup(de->d_name));
453 }
454 /* Does it look like a timeline history file? */
455 else if (IsTLHistoryFileName(de->d_name))
456 {
457 historyFileList = lappend(historyFileList, pstrdup(de->d_name));
458 }
459 }
460 FreeDir(dir);
461
462 /*
463 * Before we go any further, check that none of the WAL segments we
464 * need were removed.
465 */
466 CheckXLogRemoved(startsegno, state.starttli);
467
468 /*
469 * Sort the WAL filenames. We want to send the files in order from
470 * oldest to newest, to reduce the chance that a file is recycled
471 * before we get a chance to send it over.
472 */
473 list_sort(walFileList, compareWalFileNames);
474
475 /*
476 * There must be at least one xlog file in the pg_wal directory, since
477 * we are doing backup-including-xlog.
478 */
479 if (walFileList == NIL)
480 ereport(ERROR,
481 (errmsg("could not find any WAL files")));
482
483 /*
484 * Sanity check: the first and last segment should cover startptr and
485 * endptr, with no gaps in between.
486 */
487 XLogFromFileName((char *) linitial(walFileList),
488 &tli, &segno, wal_segment_size);
489 if (segno != startsegno)
490 {
491 char startfname[MAXFNAMELEN];
492
493 XLogFileName(startfname, state.starttli, startsegno,
494 wal_segment_size);
495 ereport(ERROR,
496 (errmsg("could not find WAL file \"%s\"", startfname)));
497 }
498 foreach(lc, walFileList)
499 {
500 char *walFileName = (char *) lfirst(lc);
501 XLogSegNo currsegno = segno;
502 XLogSegNo nextsegno = segno + 1;
503
504 XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
505 if (!(nextsegno == segno || currsegno == segno))
506 {
507 char nextfname[MAXFNAMELEN];
508
509 XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
510 ereport(ERROR,
511 (errmsg("could not find WAL file \"%s\"", nextfname)));
512 }
513 }
514 if (segno != endsegno)
515 {
516 char endfname[MAXFNAMELEN];
517
518 XLogFileName(endfname, endtli, endsegno, wal_segment_size);
519 ereport(ERROR,
520 (errmsg("could not find WAL file \"%s\"", endfname)));
521 }
522
523 /* Ok, we have everything we need. Send the WAL files. */
524 foreach(lc, walFileList)
525 {
526 char *walFileName = (char *) lfirst(lc);
527 int fd;
528 ssize_t cnt;
529 pgoff_t len = 0;
530
531 snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
532 XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
533
534 fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
535 if (fd < 0)
536 {
537 int save_errno = errno;
538
539 /*
540 * Most likely reason for this is that the file was already
541 * removed by a checkpoint, so check for that to get a better
542 * error message.
543 */
544 CheckXLogRemoved(segno, tli);
545
546 errno = save_errno;
547 ereport(ERROR,
548 (errcode_for_file_access(),
549 errmsg("could not open file \"%s\": %m", pathbuf)));
550 }
551
552 if (fstat(fd, &statbuf) != 0)
553 ereport(ERROR,
554 (errcode_for_file_access(),
555 errmsg("could not stat file \"%s\": %m",
556 pathbuf)));
557 if (statbuf.st_size != wal_segment_size)
558 {
559 CheckXLogRemoved(segno, tli);
560 ereport(ERROR,
561 (errcode_for_file_access(),
562 errmsg("unexpected WAL file size \"%s\"", walFileName)));
563 }
564
565 /* send the WAL file itself */
566 _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
567
568 while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
569 Min(sink->bbs_buffer_length,
570 wal_segment_size - len),
571 len, pathbuf, true)) > 0)
572 {
573 CheckXLogRemoved(segno, tli);
574 bbsink_archive_contents(sink, cnt);
575
576 len += cnt;
577
578 if (len == wal_segment_size)
579 break;
580 }
581
582 if (len != wal_segment_size)
583 {
584 CheckXLogRemoved(segno, tli);
585 ereport(ERROR,
586 (errcode_for_file_access(),
587 errmsg("unexpected WAL file size \"%s\"", walFileName)));
588 }
589
590 /*
591 * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
592 * for padding.
593 */
594 Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
595
596 CloseTransientFile(fd);
597
598 /*
599 * Mark file as archived, otherwise files can get archived again
600 * after promotion of a new node. This is in line with
601 * walreceiver.c always doing an XLogArchiveForceDone() after a
602 * complete segment.
603 */
604 StatusFilePath(pathbuf, walFileName, ".done");
605 sendFileWithContent(sink, pathbuf, "", -1, &manifest);
606 }
607
608 /*
609 * Send timeline history files too. Only the latest timeline history
610 * file is required for recovery, and even that only if there happens
611 * to be a timeline switch in the first WAL segment that contains the
612 * checkpoint record, or if we're taking a base backup from a standby
613 * server and the target timeline changes while the backup is taken.
614 * But they are small and highly useful for debugging purposes, so
615 * better include them all, always.
616 */
617 foreach(lc, historyFileList)
618 {
619 char *fname = lfirst(lc);
620
621 snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
622
623 if (lstat(pathbuf, &statbuf) != 0)
624 ereport(ERROR,
625 (errcode_for_file_access(),
626 errmsg("could not stat file \"%s\": %m", pathbuf)));
627
628 sendFile(sink, pathbuf, pathbuf, &statbuf, false,
629 InvalidOid, InvalidOid, InvalidRelFileNumber, 0,
630 &manifest, 0, NULL, 0);
631
632 /* unconditionally mark file as archived */
633 StatusFilePath(pathbuf, fname, ".done");
634 sendFileWithContent(sink, pathbuf, "", -1, &manifest);
635 }
636
637 /* Properly terminate the tar file. */
638 StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
639 "BLCKSZ too small for 2 tar blocks");
640 memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
641 bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
642
643 /* OK, that's the end of the archive. */
644 bbsink_end_archive(sink);
645 }
646
647 AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
648 endptr, endtli);
649
650 SendBackupManifest(&manifest, sink);
651
652 bbsink_end_backup(sink, endptr, endtli);
653
654 if (total_checksum_failures)
655 {
656 if (total_checksum_failures > 1)
657 ereport(WARNING,
658 (errmsg_plural("%lld total checksum verification failure",
659 "%lld total checksum verification failures",
660 total_checksum_failures,
661 total_checksum_failures)));
662
663 ereport(ERROR,
664 (errcode(ERRCODE_DATA_CORRUPTED),
665 errmsg("checksum verification failure during base backup")));
666 }
667
668 /*
669 * Make sure to free the manifest before the resource owners as manifests
670 * use cryptohash contexts that may depend on resource owners (like
671 * OpenSSL).
672 */
673 FreeBackupManifest(&manifest);
674
675 /* clean up the resource owner we created */
676 ReleaseAuxProcessResources(true);
677 }
678
679 /*
680 * list_sort comparison function, to compare log/seg portion of WAL segment
681 * filenames, ignoring the timeline portion.
682 */
683 static int
684 compareWalFileNames(const ListCell *a, const ListCell *b)
685 {
686 char *fna = (char *) lfirst(a);
687 char *fnb = (char *) lfirst(b);
688
689 return strcmp(fna + 8, fnb + 8);
690 }
691
692 /*
693 * Parse the base backup options passed down by the parser
694 */
695 static void
696 parse_basebackup_options(List *options, basebackup_options *opt)
697 {
698 ListCell *lopt;
699 bool o_label = false;
700 bool o_progress = false;
701 bool o_checkpoint = false;
702 bool o_nowait = false;
703 bool o_wal = false;
704 bool o_incremental = false;
705 bool o_maxrate = false;
706 bool o_tablespace_map = false;
707 bool o_noverify_checksums = false;
708 bool o_manifest = false;
709 bool o_manifest_checksums = false;
710 bool o_target = false;
711 bool o_target_detail = false;
712 char *target_str = NULL;
713 char *target_detail_str = NULL;
714 bool o_compression = false;
715 bool o_compression_detail = false;
716 char *compression_detail_str = NULL;
717
718 MemSet(opt, 0, sizeof(*opt));
719 opt->manifest = MANIFEST_OPTION_NO;
720 opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
721 opt->compression = PG_COMPRESSION_NONE;
722 opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
723
724 foreach(lopt, options)
725 {
726 DefElem *defel = (DefElem *) lfirst(lopt);
727
728 if (strcmp(defel->defname, "label") == 0)
729 {
730 if (o_label)
731 ereport(ERROR,
732 (errcode(ERRCODE_SYNTAX_ERROR),
733 errmsg("duplicate option \"%s\"", defel->defname)));
734 opt->label = defGetString(defel);
735 o_label = true;
736 }
737 else if (strcmp(defel->defname, "progress") == 0)
738 {
739 if (o_progress)
740 ereport(ERROR,
741 (errcode(ERRCODE_SYNTAX_ERROR),
742 errmsg("duplicate option \"%s\"", defel->defname)));
743 opt->progress = defGetBoolean(defel);
744 o_progress = true;
745 }
746 else if (strcmp(defel->defname, "checkpoint") == 0)
747 {
748 char *optval = defGetString(defel);
749
750 if (o_checkpoint)
751 ereport(ERROR,
752 (errcode(ERRCODE_SYNTAX_ERROR),
753 errmsg("duplicate option \"%s\"", defel->defname)));
754 if (pg_strcasecmp(optval, "fast") == 0)
755 opt->fastcheckpoint = true;
756 else if (pg_strcasecmp(optval, "spread") == 0)
757 opt->fastcheckpoint = false;
758 else
759 ereport(ERROR,
760 (errcode(ERRCODE_SYNTAX_ERROR),
761 errmsg("unrecognized checkpoint type: \"%s\"",
762 optval)));
763 o_checkpoint = true;
764 }
765 else if (strcmp(defel->defname, "wait") == 0)
766 {
767 if (o_nowait)
768 ereport(ERROR,
769 (errcode(ERRCODE_SYNTAX_ERROR),
770 errmsg("duplicate option \"%s\"", defel->defname)));
771 opt->nowait = !defGetBoolean(defel);
772 o_nowait = true;
773 }
774 else if (strcmp(defel->defname, "wal") == 0)
775 {
776 if (o_wal)
777 ereport(ERROR,
778 (errcode(ERRCODE_SYNTAX_ERROR),
779 errmsg("duplicate option \"%s\"", defel->defname)));
780 opt->includewal = defGetBoolean(defel);
781 o_wal = true;
782 }
783 else if (strcmp(defel->defname, "incremental") == 0)
784 {
785 if (o_incremental)
786 ereport(ERROR,
787 (errcode(ERRCODE_SYNTAX_ERROR),
788 errmsg("duplicate option \"%s\"", defel->defname)));
789 opt->incremental = defGetBoolean(defel);
790 if (opt->incremental && !summarize_wal)
791 ereport(ERROR,
792 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
793 errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
794 o_incremental = true;
795 }
796 else if (strcmp(defel->defname, "max_rate") == 0)
797 {
798 int64 maxrate;
799
800 if (o_maxrate)
801 ereport(ERROR,
802 (errcode(ERRCODE_SYNTAX_ERROR),
803 errmsg("duplicate option \"%s\"", defel->defname)));
804
805 maxrate = defGetInt64(defel);
806 if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
807 ereport(ERROR,
808 (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
809 errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
810 (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
811
812 opt->maxrate = (uint32) maxrate;
813 o_maxrate = true;
814 }
815 else if (strcmp(defel->defname, "tablespace_map") == 0)
816 {
817 if (o_tablespace_map)
818 ereport(ERROR,
819 (errcode(ERRCODE_SYNTAX_ERROR),
820 errmsg("duplicate option \"%s\"", defel->defname)));
821 opt->sendtblspcmapfile = defGetBoolean(defel);
822 o_tablespace_map = true;
823 }
824 else if (strcmp(defel->defname, "verify_checksums") == 0)
825 {
826 if (o_noverify_checksums)
827 ereport(ERROR,
828 (errcode(ERRCODE_SYNTAX_ERROR),
829 errmsg("duplicate option \"%s\"", defel->defname)));
830 noverify_checksums = !defGetBoolean(defel);
831 o_noverify_checksums = true;
832 }
833 else if (strcmp(defel->defname, "manifest") == 0)
834 {
835 char *optval = defGetString(defel);
836 bool manifest_bool;
837
838 if (o_manifest)
839 ereport(ERROR,
840 (errcode(ERRCODE_SYNTAX_ERROR),
841 errmsg("duplicate option \"%s\"", defel->defname)));
842 if (parse_bool(optval, &manifest_bool))
843 {
844 if (manifest_bool)
845 opt->manifest = MANIFEST_OPTION_YES;
846 else
847 opt->manifest = MANIFEST_OPTION_NO;
848 }
849 else if (pg_strcasecmp(optval, "force-encode") == 0)
850 opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
851 else
852 ereport(ERROR,
853 (errcode(ERRCODE_SYNTAX_ERROR),
854 errmsg("unrecognized manifest option: \"%s\"",
855 optval)));
856 o_manifest = true;
857 }
858 else if (strcmp(defel->defname, "manifest_checksums") == 0)
859 {
860 char *optval = defGetString(defel);
861
862 if (o_manifest_checksums)
863 ereport(ERROR,
864 (errcode(ERRCODE_SYNTAX_ERROR),
865 errmsg("duplicate option \"%s\"", defel->defname)));
866 if (!pg_checksum_parse_type(optval,
867 &opt->manifest_checksum_type))
868 ereport(ERROR,
869 (errcode(ERRCODE_SYNTAX_ERROR),
870 errmsg("unrecognized checksum algorithm: \"%s\"",
871 optval)));
872 o_manifest_checksums = true;
873 }
874 else if (strcmp(defel->defname, "target") == 0)
875 {
876 if (o_target)
877 ereport(ERROR,
878 (errcode(ERRCODE_SYNTAX_ERROR),
879 errmsg("duplicate option \"%s\"", defel->defname)));
880 target_str = defGetString(defel);
881 o_target = true;
882 }
883 else if (strcmp(defel->defname, "target_detail") == 0)
884 {
885 char *optval = defGetString(defel);
886
887 if (o_target_detail)
888 ereport(ERROR,
889 (errcode(ERRCODE_SYNTAX_ERROR),
890 errmsg("duplicate option \"%s\"", defel->defname)));
891 target_detail_str = optval;
892 o_target_detail = true;
893 }
894 else if (strcmp(defel->defname, "compression") == 0)
895 {
896 char *optval = defGetString(defel);
897
898 if (o_compression)
899 ereport(ERROR,
900 (errcode(ERRCODE_SYNTAX_ERROR),
901 errmsg("duplicate option \"%s\"", defel->defname)));
902 if (!parse_compress_algorithm(optval, &opt->compression))
903 ereport(ERROR,
904 (errcode(ERRCODE_SYNTAX_ERROR),
905 errmsg("unrecognized compression algorithm: \"%s\"",
906 optval)));
907 o_compression = true;
908 }
909 else if (strcmp(defel->defname, "compression_detail") == 0)
910 {
911 if (o_compression_detail)
912 ereport(ERROR,
913 (errcode(ERRCODE_SYNTAX_ERROR),
914 errmsg("duplicate option \"%s\"", defel->defname)));
915 compression_detail_str = defGetString(defel);
916 o_compression_detail = true;
917 }
918 else
919 ereport(ERROR,
920 (errcode(ERRCODE_SYNTAX_ERROR),
921 errmsg("unrecognized base backup option: \"%s\"",
922 defel->defname)));
923 }
924
925 if (opt->label == NULL)
926 opt->label = "base backup";
927 if (opt->manifest == MANIFEST_OPTION_NO)
928 {
929 if (o_manifest_checksums)
930 ereport(ERROR,
931 (errcode(ERRCODE_SYNTAX_ERROR),
932 errmsg("manifest checksums require a backup manifest")));
933 opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
934 }
935
936 if (target_str == NULL)
937 {
938 if (target_detail_str != NULL)
939 ereport(ERROR,
940 (errcode(ERRCODE_SYNTAX_ERROR),
941 errmsg("target detail cannot be used without target")));
942 opt->use_copytblspc = true;
943 opt->send_to_client = true;
944 }
945 else if (strcmp(target_str, "client") == 0)
946 {
947 if (target_detail_str != NULL)
948 ereport(ERROR,
949 (errcode(ERRCODE_SYNTAX_ERROR),
950 errmsg("target \"%s\" does not accept a target detail",
951 target_str)));
952 opt->send_to_client = true;
953 }
954 else
955 opt->target_handle =
956 BaseBackupGetTargetHandle(target_str, target_detail_str);
957
958 if (o_compression_detail && !o_compression)
959 ereport(ERROR,
960 (errcode(ERRCODE_SYNTAX_ERROR),
961 errmsg("compression detail cannot be specified unless compression is enabled")));
962
963 if (o_compression)
964 {
965 char *error_detail;
966
967 parse_compress_specification(opt->compression, compression_detail_str,
968 &opt->compression_specification);
969 error_detail =
970 validate_compress_specification(&opt->compression_specification);
971 if (error_detail != NULL)
972 ereport(ERROR,
973 errcode(ERRCODE_SYNTAX_ERROR),
974 errmsg("invalid compression specification: %s",
975 error_detail));
976 }
977 }
978
979
980 /*
981 * SendBaseBackup() - send a complete base backup.
982 *
983 * The function will put the system into backup mode like pg_backup_start()
984 * does, so that the backup is consistent even though we read directly from
985 * the filesystem, bypassing the buffer cache.
986 */
987 void
988 SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
989 {
990 basebackup_options opt;
991 bbsink *sink;
992 SessionBackupState status = get_backup_status();
993
994 if (status == SESSION_BACKUP_RUNNING)
995 ereport(ERROR,
996 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
997 errmsg("a backup is already in progress in this session")));
998
999 parse_basebackup_options(cmd->options, &opt);
1000
1001 WalSndSetState(WALSNDSTATE_BACKUP);
1002
1003 if (update_process_title)
1004 {
1005 char activitymsg[50];
1006
1007 snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1008 opt.label);
1009 set_ps_display(activitymsg);
1010 }
1011
1012 /*
1013 * If we're asked to perform an incremental backup and the user has not
1014 * supplied a manifest, that's an ERROR.
1015 *
1016 * If we're asked to perform a full backup and the user did supply a
1017 * manifest, just ignore it.
1018 */
1019 if (!opt.incremental)
1020 ib = NULL;
1021 else if (ib == NULL)
1022 ereport(ERROR,
1023 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1024 errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1025
1026 /*
1027 * If the target is specifically 'client' then set up to stream the backup
1028 * to the client; otherwise, it's being sent someplace else and should not
1029 * be sent to the client. BaseBackupGetSink has the job of setting up a
1030 * sink to send the backup data wherever it needs to go.
1031 */
1032 sink = bbsink_copystream_new(opt.send_to_client);
1033 if (opt.target_handle != NULL)
1034 sink = BaseBackupGetSink(opt.target_handle, sink);
1035
1036 /* Set up network throttling, if client requested it */
1037 if (opt.maxrate > 0)
1038 sink = bbsink_throttle_new(sink, opt.maxrate);
1039
1040 /* Set up server-side compression, if client requested it */
1041 if (opt.compression == PG_COMPRESSION_GZIP)
1042 sink = bbsink_gzip_new(sink, &opt.compression_specification);
1043 else if (opt.compression == PG_COMPRESSION_LZ4)
1044 sink = bbsink_lz4_new(sink, &opt.compression_specification);
1045 else if (opt.compression == PG_COMPRESSION_ZSTD)
1046 sink = bbsink_zstd_new(sink, &opt.compression_specification);
1047
1048 /* Set up progress reporting. */
1049 sink = bbsink_progress_new(sink, opt.progress);
1050
1051 /*
1052 * Perform the base backup, but make sure we clean up the bbsink even if
1053 * an error occurs.
1054 */
1055 PG_TRY();
1056 {
1057 perform_base_backup(&opt, sink, ib);
1058 }
1059 PG_FINALLY();
1060 {
1061 bbsink_cleanup(sink);
1062 }
1063 PG_END_TRY();
1064 }
1065
1066 /*
1067 * Inject a file with given name and content in the output tar stream.
1068 *
1069 * "len" can optionally be set to an arbitrary length of data sent. If set
1070 * to -1, the content sent is treated as a string with strlen() as length.
1071 */
1072 static void
1073 sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1074 int len, backup_manifest_info *manifest)
1075 {
1076 struct stat statbuf;
1077 int bytes_done = 0;
1078 pg_checksum_context checksum_ctx;
1079
1080 if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1081 elog(ERROR, "could not initialize checksum of file \"%s\"",
1082 filename);
1083
1084 if (len < 0)
1085 len = strlen(content);
1086
1087 /*
1088 * Construct a stat struct for the file we're injecting in the tar.
1089 */
1090
1091 /* Windows doesn't have the concept of uid and gid */
1092 #ifdef WIN32
1093 statbuf.st_uid = 0;
1094 statbuf.st_gid = 0;
1095 #else
1096 statbuf.st_uid = geteuid();
1097 statbuf.st_gid = getegid();
1098 #endif
1099 statbuf.st_mtime = time(NULL);
1100 statbuf.st_mode = pg_file_create_mode;
1101 statbuf.st_size = len;
1102
1103 _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1104
1105 if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1106 elog(ERROR, "could not update checksum of file \"%s\"",
1107 filename);
1108
1109 while (bytes_done < len)
1110 {
1111 size_t remaining = len - bytes_done;
1112 size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1113
1114 memcpy(sink->bbs_buffer, content, nbytes);
1115 bbsink_archive_contents(sink, nbytes);
1116 bytes_done += nbytes;
1117 content += nbytes;
1118 }
1119
1120 _tarWritePadding(sink, len);
1121
1122 AddFileToBackupManifest(manifest, InvalidOid, filename, len,
1123 (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1124 }
1125
1126 /*
1127 * Include the tablespace directory pointed to by 'path' in the output tar
1128 * stream. If 'sizeonly' is true, we just calculate a total length and return
1129 * it, without actually sending anything.
1130 *
1131 * Only used to send auxiliary tablespaces, not PGDATA.
1132 */
1133 static int64
1134 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1135 backup_manifest_info *manifest, IncrementalBackupInfo *ib)
1136 {
1137 int64 size;
1138 char pathbuf[MAXPGPATH];
1139 struct stat statbuf;
1140
1141 /*
1142 * 'path' points to the tablespace location, but we only want to include
1143 * the version directory in it that belongs to us.
1144 */
1145 snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1146 TABLESPACE_VERSION_DIRECTORY);
1147
1148 /*
1149 * Store a directory entry in the tar file so we get the permissions
1150 * right.
1151 */
1152 if (lstat(pathbuf, &statbuf) != 0)
1153 {
1154 if (errno != ENOENT)
1155 ereport(ERROR,
1156 (errcode_for_file_access(),
1157 errmsg("could not stat file or directory \"%s\": %m",
1158 pathbuf)));
1159
1160 /* If the tablespace went away while scanning, it's no error. */
1161 return 0;
1162 }
1163
1164 size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1165 sizeonly);
1166
1167 /* Send all the files in the tablespace version directory */
1168 size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1169 spcoid, ib);
1170
1171 return size;
1172 }
1173
1174 /*
1175 * Include all files from the given directory in the output tar stream. If
1176 * 'sizeonly' is true, we just calculate a total length and return it, without
1177 * actually sending anything.
1178 *
1179 * Omit any directory in the tablespaces list, to avoid backing up
1180 * tablespaces twice when they were created inside PGDATA.
1181 *
1182 * If sendtblspclinks is true, we need to include symlink
1183 * information in the tar file. If not, we can skip that
1184 * as it will be sent separately in the tablespace_map file.
1185 */
1186 static int64
1187 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1188 List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1189 Oid spcoid, IncrementalBackupInfo *ib)
1190 {
1191 DIR *dir;
1192 struct dirent *de;
1193 char pathbuf[MAXPGPATH * 2];
1194 struct stat statbuf;
1195 int64 size = 0;
1196 const char *lastDir; /* Split last dir from parent path. */
1197 bool isRelationDir = false; /* Does directory contain relations? */
1198 bool isGlobalDir = false;
1199 Oid dboid = InvalidOid;
1200 BlockNumber *relative_block_numbers = NULL;
1201
1202 /*
1203 * Since this array is relatively large, avoid putting it on the stack.
1204 * But we don't need it at all if this is not an incremental backup.
1205 */
1206 if (ib != NULL)
1207 relative_block_numbers = palloc(sizeof(BlockNumber) * RELSEG_SIZE);
1208
1209 /*
1210 * Determine if the current path is a database directory that can contain
1211 * relations.
1212 *
1213 * Start by finding the location of the delimiter between the parent path
1214 * and the current path.
1215 */
1216 lastDir = last_dir_separator(path);
1217
1218 /* Does this path look like a database path (i.e. all digits)? */
1219 if (lastDir != NULL &&
1220 strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1221 {
1222 /* Part of path that contains the parent directory. */
1223 int parentPathLen = lastDir - path;
1224
1225 /*
1226 * Mark path as a database directory if the parent path is either
1227 * $PGDATA/base or a tablespace version path.
1228 */
1229 if (strncmp(path, "./base", parentPathLen) == 0 ||
1230 (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1231 strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1232 TABLESPACE_VERSION_DIRECTORY,
1233 sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1234 {
1235 isRelationDir = true;
1236 dboid = atooid(lastDir + 1);
1237 }
1238 }
1239 else if (strcmp(path, "./global") == 0)
1240 {
1241 isRelationDir = true;
1242 isGlobalDir = true;
1243 }
1244
1245 dir = AllocateDir(path);
1246 while ((de = ReadDir(dir, path)) != NULL)
1247 {
1248 int excludeIdx;
1249 bool excludeFound;
1250 RelFileNumber relfilenumber = InvalidRelFileNumber;
1251 ForkNumber relForkNum = InvalidForkNumber;
1252 unsigned segno = 0;
1253 bool isRelationFile = false;
1254
1255 /* Skip special stuff */
1256 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1257 continue;
1258
1259 /* Skip temporary files */
1260 if (strncmp(de->d_name,
1261 PG_TEMP_FILE_PREFIX,
1262 strlen(PG_TEMP_FILE_PREFIX)) == 0)
1263 continue;
1264
1265 /* Skip macOS system files */
1266 if (strcmp(de->d_name, ".DS_Store") == 0)
1267 continue;
1268
1269 /*
1270 * Check if the postmaster has signaled us to exit, and abort with an
1271 * error in that case. The error handler further up will call
1272 * do_pg_abort_backup() for us. Also check that if the backup was
1273 * started while still in recovery, the server wasn't promoted.
1274 * do_pg_backup_stop() will check that too, but it's better to stop
1275 * the backup early than continue to the end and fail there.
1276 */
1277 CHECK_FOR_INTERRUPTS();
1278 if (RecoveryInProgress() != backup_started_in_recovery)
1279 ereport(ERROR,
1280 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1281 errmsg("the standby was promoted during online backup"),
1282 errhint("This means that the backup being taken is corrupt "
1283 "and should not be used. "
1284 "Try taking another online backup.")));
1285
1286 /* Scan for files that should be excluded */
1287 excludeFound = false;
1288 for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1289 {
1290 int cmplen = strlen(excludeFiles[excludeIdx].name);
1291
1292 if (!excludeFiles[excludeIdx].match_prefix)
1293 cmplen++;
1294 if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1295 {
1296 elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1297 excludeFound = true;
1298 break;
1299 }
1300 }
1301
1302 if (excludeFound)
1303 continue;
1304
1305 /*
1306 * If there could be non-temporary relation files in this directory,
1307 * try to parse the filename.
1308 */
1309 if (isRelationDir)
1310 isRelationFile =
1311 parse_filename_for_nontemp_relation(de->d_name,
1312 &relfilenumber,
1313 &relForkNum, &segno);
1314
1315 /* Exclude all forks for unlogged tables except the init fork */
1316 if (isRelationFile && relForkNum != INIT_FORKNUM)
1317 {
1318 char initForkFile[MAXPGPATH];
1319
1320 /*
1321 * If any other type of fork, check if there is an init fork with
1322 * the same RelFileNumber. If so, the file can be excluded.
1323 */
1324 snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1325 path, relfilenumber);
1326
1327 if (lstat(initForkFile, &statbuf) == 0)
1328 {
1329 elog(DEBUG2,
1330 "unlogged relation file \"%s\" excluded from backup",
1331 de->d_name);
1332
1333 continue;
1334 }
1335 }
1336
1337 /* Exclude temporary relations */
1338 if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1339 {
1340 elog(DEBUG2,
1341 "temporary relation file \"%s\" excluded from backup",
1342 de->d_name);
1343
1344 continue;
1345 }
1346
1347 snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1348
1349 /* Skip pg_control here to back up it last */
1350 if (strcmp(pathbuf, "./" XLOG_CONTROL_FILE) == 0)
1351 continue;
1352
1353 if (lstat(pathbuf, &statbuf) != 0)
1354 {
1355 if (errno != ENOENT)
1356 ereport(ERROR,
1357 (errcode_for_file_access(),
1358 errmsg("could not stat file or directory \"%s\": %m",
1359 pathbuf)));
1360
1361 /* If the file went away while scanning, it's not an error. */
1362 continue;
1363 }
1364
1365 /* Scan for directories whose contents should be excluded */
1366 excludeFound = false;
1367 for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1368 {
1369 if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1370 {
1371 elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1372 convert_link_to_directory(pathbuf, &statbuf);
1373 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1374 &statbuf, sizeonly);
1375 excludeFound = true;
1376 break;
1377 }
1378 }
1379
1380 if (excludeFound)
1381 continue;
1382
1383 /*
1384 * We can skip pg_wal, the WAL segments need to be fetched from the
1385 * WAL archive anyway. But include it as an empty directory anyway, so
1386 * we get permissions right.
1387 */
1388 if (strcmp(pathbuf, "./pg_wal") == 0)
1389 {
1390 /* If pg_wal is a symlink, write it as a directory anyway */
1391 convert_link_to_directory(pathbuf, &statbuf);
1392 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1393 &statbuf, sizeonly);
1394
1395 /*
1396 * Also send archive_status and summaries directories (by
1397 * hackishly reusing statbuf from above ...).
1398 */
1399 size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1400 &statbuf, sizeonly);
1401 size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1402 &statbuf, sizeonly);
1403
1404 continue; /* don't recurse into pg_wal */
1405 }
1406
1407 /* Allow symbolic links in pg_tblspc only */
1408 if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
1409 {
1410 char linkpath[MAXPGPATH];
1411 int rllen;
1412
1413 rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1414 if (rllen < 0)
1415 ereport(ERROR,
1416 (errcode_for_file_access(),
1417 errmsg("could not read symbolic link \"%s\": %m",
1418 pathbuf)));
1419 if (rllen >= sizeof(linkpath))
1420 ereport(ERROR,
1421 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1422 errmsg("symbolic link \"%s\" target is too long",
1423 pathbuf)));
1424 linkpath[rllen] = '\0';
1425
1426 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1427 &statbuf, sizeonly);
1428 }
1429 else if (S_ISDIR(statbuf.st_mode))
1430 {
1431 bool skip_this_dir = false;
1432 ListCell *lc;
1433
1434 /*
1435 * Store a directory entry in the tar file so we can get the
1436 * permissions right.
1437 */
1438 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1439 sizeonly);
1440
1441 /*
1442 * Call ourselves recursively for a directory, unless it happens
1443 * to be a separate tablespace located within PGDATA.
1444 */
1445 foreach(lc, tablespaces)
1446 {
1447 tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1448
1449 /*
1450 * ti->rpath is the tablespace relative path within PGDATA, or
1451 * NULL if the tablespace has been properly located somewhere
1452 * else.
1453 *
1454 * Skip past the leading "./" in pathbuf when comparing.
1455 */
1456 if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1457 {
1458 skip_this_dir = true;
1459 break;
1460 }
1461 }
1462
1463 /*
1464 * skip sending directories inside pg_tblspc, if not required.
1465 */
1466 if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1467 skip_this_dir = true;
1468
1469 if (!skip_this_dir)
1470 size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1471 sendtblspclinks, manifest, spcoid, ib);
1472 }
1473 else if (S_ISREG(statbuf.st_mode))
1474 {
1475 bool sent = false;
1476 unsigned num_blocks_required = 0;
1477 unsigned truncation_block_length = 0;
1478 char tarfilenamebuf[MAXPGPATH * 2];
1479 char *tarfilename = pathbuf + basepathlen + 1;
1480 FileBackupMethod method = BACK_UP_FILE_FULLY;
1481
1482 if (ib != NULL && isRelationFile)
1483 {
1484 Oid relspcoid;
1485 char *lookup_path;
1486
1487 if (OidIsValid(spcoid))
1488 {
1489 relspcoid = spcoid;
1490 lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid,
1491 tarfilename);
1492 }
1493 else
1494 {
1495 if (isGlobalDir)
1496 relspcoid = GLOBALTABLESPACE_OID;
1497 else
1498 relspcoid = DEFAULTTABLESPACE_OID;
1499 lookup_path = pstrdup(tarfilename);
1500 }
1501
1502 method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1503 relfilenumber, relForkNum,
1504 segno, statbuf.st_size,
1505 &num_blocks_required,
1506 relative_block_numbers,
1507 &truncation_block_length);
1508 if (method == BACK_UP_FILE_INCREMENTALLY)
1509 {
1510 statbuf.st_size =
1511 GetIncrementalFileSize(num_blocks_required);
1512 snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1513 "%s/INCREMENTAL.%s",
1514 path + basepathlen + 1,
1515 de->d_name);
1516 tarfilename = tarfilenamebuf;
1517 }
1518
1519 pfree(lookup_path);
1520 }
1521
1522 if (!sizeonly)
1523 sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1524 true, dboid, spcoid,
1525 relfilenumber, segno, manifest,
1526 num_blocks_required,
1527 method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1528 truncation_block_length);
1529
1530 if (sent || sizeonly)
1531 {
1532 /* Add size. */
1533 size += statbuf.st_size;
1534
1535 /* Pad to a multiple of the tar block size. */
1536 size += tarPaddingBytesRequired(statbuf.st_size);
1537
1538 /* Size of the header for the file. */
1539 size += TAR_BLOCK_SIZE;
1540 }
1541 }
1542 else
1543 ereport(WARNING,
1544 (errmsg("skipping special file \"%s\"", pathbuf)));
1545 }
1546
1547 if (relative_block_numbers != NULL)
1548 pfree(relative_block_numbers);
1549
1550 FreeDir(dir);
1551 return size;
1552 }
1553
1554 /*
1555 * Given the member, write the TAR header & send the file.
1556 *
1557 * If 'missing_ok' is true, will not throw an error if the file is not found.
1558 *
1559 * If dboid is anything other than InvalidOid then any checksum failures
1560 * detected will get reported to the cumulative stats system.
1561 *
1562 * If the file is to be sent incrementally, then num_incremental_blocks
1563 * should be the number of blocks to be sent, and incremental_blocks
1564 * an array of block numbers relative to the start of the current segment.
1565 * If the whole file is to be sent, then incremental_blocks should be NULL,
1566 * and num_incremental_blocks can have any value, as it will be ignored.
1567 *
1568 * Returns true if the file was successfully sent, false if 'missing_ok',
1569 * and the file did not exist.
1570 */
1571 static bool
1572 sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1573 struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1574 RelFileNumber relfilenumber, unsigned segno,
1575 backup_manifest_info *manifest, unsigned num_incremental_blocks,
1576 BlockNumber *incremental_blocks, unsigned truncation_block_length)
1577 {
1578 int fd;
1579 BlockNumber blkno = 0;
1580 int checksum_failures = 0;
1581 off_t cnt;
1582 pgoff_t bytes_done = 0;
1583 bool verify_checksum = false;
1584 pg_checksum_context checksum_ctx;
1585 int ibindex = 0;
1586
1587 if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1588 elog(ERROR, "could not initialize checksum of file \"%s\"",
1589 readfilename);
1590
1591 fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1592 if (fd < 0)
1593 {
1594 if (errno == ENOENT && missing_ok)
1595 return false;
1596 ereport(ERROR,
1597 (errcode_for_file_access(),
1598 errmsg("could not open file \"%s\": %m", readfilename)));
1599 }
1600
1601 _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1602
1603 /*
1604 * Checksums are verified in multiples of BLCKSZ, so the buffer length
1605 * should be a multiple of the block size as well.
1606 */
1607 Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1608
1609 /*
1610 * If we weren't told not to verify checksums, and if checksums are
1611 * enabled for this cluster, and if this is a relation file, then verify
1612 * the checksum.
1613 */
1614 if (!noverify_checksums && DataChecksumsEnabled() &&
1615 RelFileNumberIsValid(relfilenumber))
1616 verify_checksum = true;
1617
1618 /*
1619 * If we're sending an incremental file, write the file header.
1620 */
1621 if (incremental_blocks != NULL)
1622 {
1623 unsigned magic = INCREMENTAL_MAGIC;
1624 size_t header_bytes_done = 0;
1625 char padding[BLCKSZ];
1626 size_t paddinglen;
1627
1628 /* Emit header data. */
1629 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1630 &magic, sizeof(magic));
1631 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1632 &num_incremental_blocks, sizeof(num_incremental_blocks));
1633 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1634 &truncation_block_length, sizeof(truncation_block_length));
1635 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1636 incremental_blocks,
1637 sizeof(BlockNumber) * num_incremental_blocks);
1638
1639 /*
1640 * Add padding to align header to a multiple of BLCKSZ, but only if
1641 * the incremental file has some blocks, and the alignment is actually
1642 * needed (i.e. header is not already a multiple of BLCKSZ). If there
1643 * are no blocks we don't want to make the file unnecessarily large,
1644 * as that might make some filesystem optimizations impossible.
1645 */
1646 if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1647 {
1648 paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1649
1650 memset(padding, 0, paddinglen);
1651 bytes_done += paddinglen;
1652
1653 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1654 padding, paddinglen);
1655 }
1656
1657 /* Flush out any data still in the buffer so it's again empty. */
1658 if (header_bytes_done > 0)
1659 {
1660 bbsink_archive_contents(sink, header_bytes_done);
1661 if (pg_checksum_update(&checksum_ctx,
1662 (uint8 *) sink->bbs_buffer,
1663 header_bytes_done) < 0)
1664 elog(ERROR, "could not update checksum of base backup");
1665 }
1666
1667 /* Update our notion of file position. */
1668 bytes_done += sizeof(magic);
1669 bytes_done += sizeof(num_incremental_blocks);
1670 bytes_done += sizeof(truncation_block_length);
1671 bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1672 }
1673
1674 /*
1675 * Loop until we read the amount of data the caller told us to expect. The
1676 * file could be longer, if it was extended while we were sending it, but
1677 * for a base backup we can ignore such extended data. It will be restored
1678 * from WAL.
1679 */
1680 while (1)
1681 {
1682 /*
1683 * Determine whether we've read all the data that we need, and if not,
1684 * read some more.
1685 */
1686 if (incremental_blocks == NULL)
1687 {
1688 size_t remaining = statbuf->st_size - bytes_done;
1689
1690 /*
1691 * If we've read the required number of bytes, then it's time to
1692 * stop.
1693 */
1694 if (bytes_done >= statbuf->st_size)
1695 break;
1696
1697 /*
1698 * Read as many bytes as will fit in the buffer, or however many
1699 * are left to read, whichever is less.
1700 */
1701 cnt = read_file_data_into_buffer(sink, readfilename, fd,
1702 bytes_done, remaining,
1703 blkno + segno * RELSEG_SIZE,
1704 verify_checksum,
1705 &checksum_failures);
1706 }
1707 else
1708 {
1709 BlockNumber relative_blkno;
1710
1711 /*
1712 * If we've read all the blocks, then it's time to stop.
1713 */
1714 if (ibindex >= num_incremental_blocks)
1715 break;
1716
1717 /*
1718 * Read just one block, whichever one is the next that we're
1719 * supposed to include.
1720 */
1721 relative_blkno = incremental_blocks[ibindex++];
1722 cnt = read_file_data_into_buffer(sink, readfilename, fd,
1723 relative_blkno * BLCKSZ,
1724 BLCKSZ,
1725 relative_blkno + segno * RELSEG_SIZE,
1726 verify_checksum,
1727 &checksum_failures);
1728
1729 /*
1730 * If we get a partial read, that must mean that the relation is
1731 * being truncated. Ultimately, it should be truncated to a
1732 * multiple of BLCKSZ, since this path should only be reached for
1733 * relation files, but we might transiently observe an
1734 * intermediate value.
1735 *
1736 * It should be fine to treat this just as if the entire block had
1737 * been truncated away - i.e. fill this and all later blocks with
1738 * zeroes. WAL replay will fix things up.
1739 */
1740 if (cnt < BLCKSZ)
1741 break;
1742 }
1743
1744 /*
1745 * If the amount of data we were able to read was not a multiple of
1746 * BLCKSZ, we cannot verify checksums, which are block-level.
1747 */
1748 if (verify_checksum && (cnt % BLCKSZ != 0))
1749 {
1750 ereport(WARNING,
1751 (errmsg("could not verify checksum in file \"%s\", block "
1752 "%u: read buffer size %d and page size %d "
1753 "differ",
1754 readfilename, blkno, (int) cnt, BLCKSZ)));
1755 verify_checksum = false;
1756 }
1757
1758 /*
1759 * If we hit end-of-file, a concurrent truncation must have occurred.
1760 * That's not an error condition, because WAL replay will fix things
1761 * up.
1762 */
1763 if (cnt == 0)
1764 break;
1765
1766 /* Update block number and # of bytes done for next loop iteration. */
1767 blkno += cnt / BLCKSZ;
1768 bytes_done += cnt;
1769
1770 /*
1771 * Make sure incremental files with block data are properly aligned
1772 * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1773 */
1774 Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
1775 (bytes_done % BLCKSZ != 0)));
1776
1777 /* Archive the data we just read. */
1778 bbsink_archive_contents(sink, cnt);
1779
1780 /* Also feed it to the checksum machinery. */
1781 if (pg_checksum_update(&checksum_ctx,
1782 (uint8 *) sink->bbs_buffer, cnt) < 0)
1783 elog(ERROR, "could not update checksum of base backup");
1784 }
1785
1786 /* If the file was truncated while we were sending it, pad it with zeros */
1787 while (bytes_done < statbuf->st_size)
1788 {
1789 size_t remaining = statbuf->st_size - bytes_done;
1790 size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1791
1792 MemSet(sink->bbs_buffer, 0, nbytes);
1793 if (pg_checksum_update(&checksum_ctx,
1794 (uint8 *) sink->bbs_buffer,
1795 nbytes) < 0)
1796 elog(ERROR, "could not update checksum of base backup");
1797 bbsink_archive_contents(sink, nbytes);
1798 bytes_done += nbytes;
1799 }
1800
1801 /*
1802 * Pad to a block boundary, per tar format requirements. (This small piece
1803 * of data is probably not worth throttling, and is not checksummed
1804 * because it's not actually part of the file.)
1805 */
1806 _tarWritePadding(sink, bytes_done);
1807
1808 CloseTransientFile(fd);
1809
1810 if (checksum_failures > 1)
1811 {
1812 ereport(WARNING,
1813 (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1814 "file \"%s\" has a total of %d checksum verification failures",
1815 checksum_failures,
1816 readfilename, checksum_failures)));
1817
1818 pgstat_prepare_report_checksum_failure(dboid);
1819 pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1820 }
1821
1822 total_checksum_failures += checksum_failures;
1823
1824 AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1825 (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1826
1827 return true;
1828 }
1829
1830 /*
1831 * Read some more data from the file into the bbsink's buffer, verifying
1832 * checksums as required.
1833 *
1834 * 'offset' is the file offset from which we should begin to read, and
1835 * 'length' is the amount of data that should be read. The actual amount
1836 * of data read will be less than the requested amount if the bbsink's
1837 * buffer isn't big enough to hold it all, or if the underlying file has
1838 * been truncated. The return value is the number of bytes actually read.
1839 *
1840 * 'blkno' is the block number of the first page in the bbsink's buffer
1841 * relative to the start of the relation.
1842 *
1843 * 'verify_checksum' indicates whether we should try to verify checksums
1844 * for the blocks we read. If we do this, we'll update *checksum_failures
1845 * and issue warnings as appropriate.
1846 */
1847 static off_t
1848 read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1849 off_t offset, size_t length, BlockNumber blkno,
1850 bool verify_checksum, int *checksum_failures)
1851 {
1852 off_t cnt;
1853 int i;
1854 char *page;
1855
1856 /* Try to read some more data. */
1857 cnt = basebackup_read_file(fd, sink->bbs_buffer,
1858 Min(sink->bbs_buffer_length, length),
1859 offset, readfilename, true);
1860
1861 /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1862 if (!verify_checksum || (cnt % BLCKSZ) != 0)
1863 return cnt;
1864
1865 /* Verify checksum for each block. */
1866 for (i = 0; i < cnt / BLCKSZ; i++)
1867 {
1868 int reread_cnt;
1869 uint16 expected_checksum;
1870
1871 page = sink->bbs_buffer + BLCKSZ * i;
1872
1873 /* If the page is OK, go on to the next one. */
1874 if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1875 &expected_checksum))
1876 continue;
1877
1878 /*
1879 * Retry the block on the first failure. It's possible that we read
1880 * the first 4K page of the block just before postgres updated the
1881 * entire block so it ends up looking torn to us. If, before we retry
1882 * the read, the concurrent write of the block finishes, the page LSN
1883 * will be updated and we'll realize that we should ignore this block.
1884 *
1885 * There's no guarantee that this will actually happen, though: the
1886 * torn write could take an arbitrarily long time to complete.
1887 * Retrying multiple times wouldn't fix this problem, either, though
1888 * it would reduce the chances of it happening in practice. The only
1889 * real fix here seems to be to have some kind of interlock that
1890 * allows us to wait until we can be certain that no write to the
1891 * block is in progress. Since we don't have any such thing right now,
1892 * we just do this and hope for the best.
1893 */
1894 reread_cnt =
1895 basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1896 BLCKSZ, offset + BLCKSZ * i,
1897 readfilename, false);
1898 if (reread_cnt == 0)
1899 {
1900 /*
1901 * If we hit end-of-file, a concurrent truncation must have
1902 * occurred, so reduce cnt to reflect only the blocks already
1903 * processed and break out of this loop.
1904 */
1905 cnt = BLCKSZ * i;
1906 break;
1907 }
1908
1909 /* If the page now looks OK, go on to the next one. */
1910 if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1911 &expected_checksum))
1912 continue;
1913
1914 /* Handle checksum failure. */
1915 (*checksum_failures)++;
1916 if (*checksum_failures <= 5)
1917 ereport(WARNING,
1918 (errmsg("checksum verification failed in "
1919 "file \"%s\", block %u: calculated "
1920 "%X but expected %X",
1921 readfilename, blkno + i, expected_checksum,
1922 ((PageHeader) page)->pd_checksum)));
1923 if (*checksum_failures == 5)
1924 ereport(WARNING,
1925 (errmsg("further checksum verification "
1926 "failures in file \"%s\" will not "
1927 "be reported", readfilename)));
1928 }
1929
1930 return cnt;
1931 }
1932
1933 /*
1934 * Push data into a bbsink.
1935 *
1936 * It's better, when possible, to read data directly into the bbsink's buffer,
1937 * rather than using this function to copy it into the buffer; this function is
1938 * for cases where that approach is not practical.
1939 *
1940 * bytes_done should point to a count of the number of bytes that are
1941 * currently used in the bbsink's buffer. Upon return, the bytes identified by
1942 * data and length will have been copied into the bbsink's buffer, flushing
1943 * as required, and *bytes_done will have been updated accordingly. If the
1944 * buffer was flushed, the previous contents will also have been fed to
1945 * checksum_ctx.
1946 *
1947 * Note that after one or more calls to this function it is the caller's
1948 * responsibility to perform any required final flush.
1949 */
1950 static void
1951 push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
1952 size_t *bytes_done, void *data, size_t length)
1953 {
1954 while (length > 0)
1955 {
1956 size_t bytes_to_copy;
1957
1958 /*
1959 * We use < here rather than <= so that if the data exactly fills the
1960 * remaining buffer space, we trigger a flush now.
1961 */
1962 if (length < sink->bbs_buffer_length - *bytes_done)
1963 {
1964 /* Append remaining data to buffer. */
1965 memcpy(sink->bbs_buffer + *bytes_done, data, length);
1966 *bytes_done += length;
1967 return;
1968 }
1969
1970 /* Copy until buffer is full and flush it. */
1971 bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
1972 memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
1973 data = ((char *) data) + bytes_to_copy;
1974 length -= bytes_to_copy;
1975 bbsink_archive_contents(sink, sink->bbs_buffer_length);
1976 if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
1977 sink->bbs_buffer_length) < 0)
1978 elog(ERROR, "could not update checksum");
1979 *bytes_done = 0;
1980 }
1981 }
1982
1983 /*
1984 * Try to verify the checksum for the provided page, if it seems appropriate
1985 * to do so.
1986 *
1987 * Returns true if verification succeeds or if we decide not to check it,
1988 * and false if verification fails. When return false, it also sets
1989 * *expected_checksum to the computed value.
1990 */
1991 static bool
1992 verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno,
1993 uint16 *expected_checksum)
1994 {
1995 PageHeader phdr;
1996 uint16 checksum;
1997
1998 /*
1999 * Only check pages which have not been modified since the start of the
2000 * base backup. Otherwise, they might have been written only halfway and
2001 * the checksum would not be valid. However, replaying WAL would
2002 * reinstate the correct page in this case. We also skip completely new
2003 * pages, since they don't have a checksum yet.
2004 */
2005 if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2006 return true;
2007
2008 /* Perform the actual checksum calculation. */
2009 checksum = pg_checksum_page(page, blkno);
2010
2011 /* See whether it matches the value from the page. */
2012 phdr = (PageHeader) page;
2013 if (phdr->pd_checksum == checksum)
2014 return true;
2015 *expected_checksum = checksum;
2016 return false;
2017 }
2018
2019 static int64
2020 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2021 struct stat *statbuf, bool sizeonly)
2022 {
2023 enum tarError rc;
2024
2025 if (!sizeonly)
2026 {
2027 /*
2028 * As of this writing, the smallest supported block size is 1kB, which
2029 * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2030 * multiple of BLCKSZ, it should be safe to assume that the buffer is
2031 * large enough to fit an entire tar block. We double-check by means
2032 * of these assertions.
2033 */
2034 StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
2035 "BLCKSZ too small for tar block");
2036 Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2037
2038 rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2039 statbuf->st_size, statbuf->st_mode,
2040 statbuf->st_uid, statbuf->st_gid,
2041 statbuf->st_mtime);
2042
2043 switch (rc)
2044 {
2045 case TAR_OK:
2046 break;
2047 case TAR_NAME_TOO_LONG:
2048 ereport(ERROR,
2049 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2050 errmsg("file name too long for tar format: \"%s\"",
2051 filename)));
2052 break;
2053 case TAR_SYMLINK_TOO_LONG:
2054 ereport(ERROR,
2055 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2056 errmsg("symbolic link target too long for tar format: "
2057 "file name \"%s\", target \"%s\"",
2058 filename, linktarget)));
2059 break;
2060 default:
2061 elog(ERROR, "unrecognized tar error: %d", rc);
2062 }
2063
2064 bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
2065 }
2066
2067 return TAR_BLOCK_SIZE;
2068 }
2069
2070 /*
2071 * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2072 */
2073 static void
2074 _tarWritePadding(bbsink *sink, int len)
2075 {
2076 int pad = tarPaddingBytesRequired(len);
2077
2078 /*
2079 * As in _tarWriteHeader, it should be safe to assume that the buffer is
2080 * large enough that we don't need to do this in multiple chunks.
2081 */
2082 Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2083 Assert(pad <= TAR_BLOCK_SIZE);
2084
2085 if (pad > 0)
2086 {
2087 MemSet(sink->bbs_buffer, 0, pad);
2088 bbsink_archive_contents(sink, pad);
2089 }
2090 }
2091
2092 /*
2093 * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2094 * directory, so that it will be written that way.
2095 */
2096 static void
2097 convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2098 {
2099 /* If symlink, write it as a directory anyway */
2100 if (S_ISLNK(statbuf->st_mode))
2101 statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2102 }
2103
2104 /*
2105 * Read some data from a file, setting a wait event and reporting any error
2106 * encountered.
2107 *
2108 * If partial_read_ok is false, also report an error if the number of bytes
2109 * read is not equal to the number of bytes requested.
2110 *
2111 * Returns the number of bytes read.
2112 */
2113 static ssize_t
2114 basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2115 const char *filename, bool partial_read_ok)
2116 {
2117 ssize_t rc;
2118
2119 pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
2120 rc = pg_pread(fd, buf, nbytes, offset);
2121 pgstat_report_wait_end();
2122
2123 if (rc < 0)
2124 ereport(ERROR,
2125 (errcode_for_file_access(),
2126 errmsg("could not read file \"%s\": %m", filename)));
2127 if (!partial_read_ok && rc > 0 && rc != nbytes)
2128 ereport(ERROR,
2129 (errcode_for_file_access(),
2130 errmsg("could not read file \"%s\": read %zd of %zu",
2131 filename, rc, nbytes)));
2132
2133 return rc;
2134 }