| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* basebackup.c |
| 4 |
* code for taking a base backup and streaming it to a standby |
| 5 |
* |
| 6 |
* Portions Copyright (c) 2010-2025, PostgreSQL Global Development Group |
| 7 |
* |
| 8 |
* IDENTIFICATION |
| 9 |
* src/backend/backup/basebackup.c |
| 10 |
* |
| 11 |
*------------------------------------------------------------------------- |
| 12 |
*/ |
| 13 |
#include "postgres.h" |
| 14 |
|
| 15 |
#include <sys/stat.h> |
| 16 |
#include <unistd.h> |
| 17 |
#include <time.h> |
| 18 |
|
| 19 |
#include "access/xlog_internal.h" |
| 20 |
#include "access/xlogbackup.h" |
| 21 |
#include "backup/backup_manifest.h" |
| 22 |
#include "backup/basebackup.h" |
| 23 |
#include "backup/basebackup_incremental.h" |
| 24 |
#include "backup/basebackup_sink.h" |
| 25 |
#include "backup/basebackup_target.h" |
| 26 |
#include "catalog/pg_tablespace_d.h" |
| 27 |
#include "commands/defrem.h" |
| 28 |
#include "common/compression.h" |
| 29 |
#include "common/file_perm.h" |
| 30 |
#include "common/file_utils.h" |
| 31 |
#include "lib/stringinfo.h" |
| 32 |
#include "miscadmin.h" |
| 33 |
#include "nodes/pg_list.h" |
| 34 |
#include "pgstat.h" |
| 35 |
#include "pgtar.h" |
| 36 |
#include "postmaster/syslogger.h" |
| 37 |
#include "postmaster/walsummarizer.h" |
| 38 |
#include "replication/slot.h" |
| 39 |
#include "replication/walsender.h" |
| 40 |
#include "replication/walsender_private.h" |
| 41 |
#include "storage/bufpage.h" |
| 42 |
#include "storage/checksum.h" |
| 43 |
#include "storage/dsm_impl.h" |
| 44 |
#include "storage/ipc.h" |
| 45 |
#include "storage/reinit.h" |
| 46 |
#include "utils/builtins.h" |
| 47 |
#include "utils/guc.h" |
| 48 |
#include "utils/ps_status.h" |
| 49 |
#include "utils/relcache.h" |
| 50 |
#include "utils/resowner.h" |
| 51 |
|
| 52 |
/* |
| 53 |
* How much data do we want to send in one CopyData message? Note that |
| 54 |
* this may also result in reading the underlying files in chunks of this |
| 55 |
* size. |
| 56 |
* |
| 57 |
* NB: The buffer size is required to be a multiple of the system block |
| 58 |
* size, so use that value instead if it's bigger than our preference. |
| 59 |
*/ |
| 60 |
#define SINK_BUFFER_LENGTH Max(32768, BLCKSZ) |
| 61 |
|
| 62 |
typedef struct |
| 63 |
{ |
| 64 |
const char *label; |
| 65 |
bool progress; |
| 66 |
bool fastcheckpoint; |
| 67 |
bool nowait; |
| 68 |
bool includewal; |
| 69 |
bool incremental; |
| 70 |
uint32 maxrate; |
| 71 |
bool sendtblspcmapfile; |
| 72 |
bool send_to_client; |
| 73 |
bool use_copytblspc; |
| 74 |
BaseBackupTargetHandle *target_handle; |
| 75 |
backup_manifest_option manifest; |
| 76 |
pg_compress_algorithm compression; |
| 77 |
pg_compress_specification compression_specification; |
| 78 |
pg_checksum_type manifest_checksum_type; |
| 79 |
} basebackup_options; |
| 80 |
|
| 81 |
static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly, |
| 82 |
struct backup_manifest_info *manifest, |
| 83 |
IncrementalBackupInfo *ib); |
| 84 |
static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, |
| 85 |
List *tablespaces, bool sendtblspclinks, |
| 86 |
backup_manifest_info *manifest, Oid spcoid, |
| 87 |
IncrementalBackupInfo *ib); |
| 88 |
static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, |
| 89 |
struct stat *statbuf, bool missing_ok, |
| 90 |
Oid dboid, Oid spcoid, RelFileNumber relfilenumber, |
| 91 |
unsigned segno, |
| 92 |
backup_manifest_info *manifest, |
| 93 |
unsigned num_incremental_blocks, |
| 94 |
BlockNumber *incremental_blocks, |
| 95 |
unsigned truncation_block_length); |
| 96 |
static off_t read_file_data_into_buffer(bbsink *sink, |
| 97 |
const char *readfilename, int fd, |
| 98 |
off_t offset, size_t length, |
| 99 |
BlockNumber blkno, |
| 100 |
bool verify_checksum, |
| 101 |
int *checksum_failures); |
| 102 |
static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, |
| 103 |
size_t *bytes_done, void *data, size_t length); |
| 104 |
static bool verify_page_checksum(Page page, XLogRecPtr start_lsn, |
| 105 |
BlockNumber blkno, |
| 106 |
uint16 *expected_checksum); |
| 107 |
static void sendFileWithContent(bbsink *sink, const char *filename, |
| 108 |
const char *content, int len, |
| 109 |
backup_manifest_info *manifest); |
| 110 |
static int64 _tarWriteHeader(bbsink *sink, const char *filename, |
| 111 |
const char *linktarget, struct stat *statbuf, |
| 112 |
bool sizeonly); |
| 113 |
static void _tarWritePadding(bbsink *sink, int len); |
| 114 |
static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf); |
| 115 |
static void perform_base_backup(basebackup_options *opt, bbsink *sink, |
| 116 |
IncrementalBackupInfo *ib); |
| 117 |
static void parse_basebackup_options(List *options, basebackup_options *opt); |
| 118 |
static int compareWalFileNames(const ListCell *a, const ListCell *b); |
| 119 |
static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, |
| 120 |
const char *filename, bool partial_read_ok); |
| 121 |
|
| 122 |
/* Was the backup currently in-progress initiated in recovery mode? */ |
| 123 |
static bool backup_started_in_recovery = false; |
| 124 |
|
| 125 |
/* Total number of checksum failures during base backup. */ |
| 126 |
static long long int total_checksum_failures; |
| 127 |
|
| 128 |
/* Do not verify checksums. */ |
| 129 |
static bool noverify_checksums = false; |
| 130 |
|
| 131 |
/* |
| 132 |
* Definition of one element part of an exclusion list, used for paths part |
| 133 |
* of checksum validation or base backups. "name" is the name of the file |
| 134 |
* or path to check for exclusion. If "match_prefix" is true, any items |
| 135 |
* matching the name as prefix are excluded. |
| 136 |
*/ |
| 137 |
struct exclude_list_item |
| 138 |
{ |
| 139 |
const char *name; |
| 140 |
bool match_prefix; |
| 141 |
}; |
| 142 |
|
| 143 |
/* |
| 144 |
* The contents of these directories are removed or recreated during server |
| 145 |
* start so they are not included in backups. The directories themselves are |
| 146 |
* kept and included as empty to preserve access permissions. |
| 147 |
* |
| 148 |
* Note: this list should be kept in sync with the filter lists in pg_rewind's |
| 149 |
* filemap.c. |
| 150 |
*/ |
| 151 |
static const char *const excludeDirContents[] = |
| 152 |
{ |
| 153 |
/* |
| 154 |
* Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped |
| 155 |
* because extensions like pg_stat_statements store data there. |
| 156 |
*/ |
| 157 |
PG_STAT_TMP_DIR, |
| 158 |
|
| 159 |
/* |
| 160 |
* It is generally not useful to backup the contents of this directory |
| 161 |
* even if the intention is to restore to another primary. See backup.sgml |
| 162 |
* for a more detailed description. |
| 163 |
*/ |
| 164 |
PG_REPLSLOT_DIR, |
| 165 |
|
| 166 |
/* Contents removed on startup, see dsm_cleanup_for_mmap(). */ |
| 167 |
PG_DYNSHMEM_DIR, |
| 168 |
|
| 169 |
/* Contents removed on startup, see AsyncShmemInit(). */ |
| 170 |
"pg_notify", |
| 171 |
|
| 172 |
/* |
| 173 |
* Old contents are loaded for possible debugging but are not required for |
| 174 |
* normal operation, see SerialInit(). |
| 175 |
*/ |
| 176 |
"pg_serial", |
| 177 |
|
| 178 |
/* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ |
| 179 |
"pg_snapshots", |
| 180 |
|
| 181 |
/* Contents zeroed on startup, see StartupSUBTRANS(). */ |
| 182 |
"pg_subtrans", |
| 183 |
|
| 184 |
/* end of list */ |
| 185 |
NULL |
| 186 |
}; |
| 187 |
|
| 188 |
/* |
| 189 |
* List of files excluded from backups. |
| 190 |
*/ |
| 191 |
static const struct exclude_list_item excludeFiles[] = |
| 192 |
{ |
| 193 |
/* Skip auto conf temporary file. */ |
| 194 |
{PG_AUTOCONF_FILENAME ".tmp", false}, |
| 195 |
|
| 196 |
/* Skip current log file temporary file */ |
| 197 |
{LOG_METAINFO_DATAFILE_TMP, false}, |
| 198 |
|
| 199 |
/* |
| 200 |
* Skip relation cache because it is rebuilt on startup. This includes |
| 201 |
* temporary files. |
| 202 |
*/ |
| 203 |
{RELCACHE_INIT_FILENAME, true}, |
| 204 |
|
| 205 |
/* |
| 206 |
* backup_label and tablespace_map should not exist in a running cluster |
| 207 |
* capable of doing an online backup, but exclude them just in case. |
| 208 |
*/ |
| 209 |
{BACKUP_LABEL_FILE, false}, |
| 210 |
{TABLESPACE_MAP, false}, |
| 211 |
|
| 212 |
/* |
| 213 |
* If there's a backup_manifest, it belongs to a backup that was used to |
| 214 |
* start this server. It is *not* correct for this backup. Our |
| 215 |
* backup_manifest is injected into the backup separately if users want |
| 216 |
* it. |
| 217 |
*/ |
| 218 |
{"backup_manifest", false}, |
| 219 |
|
| 220 |
{"postmaster.pid", false}, |
| 221 |
{"postmaster.opts", false}, |
| 222 |
|
| 223 |
/* end of list */ |
| 224 |
{NULL, false} |
| 225 |
}; |
| 226 |
|
| 227 |
/* |
| 228 |
* Actually do a base backup for the specified tablespaces. |
| 229 |
* |
| 230 |
* This is split out mainly to avoid complaints about "variable might be |
| 231 |
* clobbered by longjmp" from stupider versions of gcc. |
| 232 |
*/ |
| 233 |
static void |
| 234 |
perform_base_backup(basebackup_options *opt, bbsink *sink, |
| 235 |
IncrementalBackupInfo *ib) |
| 236 |
{ |
| 237 |
bbsink_state state; |
| 238 |
XLogRecPtr endptr; |
| 239 |
TimeLineID endtli; |
| 240 |
backup_manifest_info manifest; |
| 241 |
BackupState *backup_state; |
| 242 |
StringInfo tablespace_map; |
| 243 |
|
| 244 |
/* Initial backup state, insofar as we know it now. */ |
| 245 |
state.tablespaces = NIL; |
| 246 |
state.tablespace_num = 0; |
| 247 |
state.bytes_done = 0; |
| 248 |
state.bytes_total = 0; |
| 249 |
state.bytes_total_is_valid = false; |
| 250 |
|
| 251 |
/* we're going to use a BufFile, so we need a ResourceOwner */ |
| 252 |
Assert(AuxProcessResourceOwner != NULL); |
| 253 |
Assert(CurrentResourceOwner == AuxProcessResourceOwner || |
| 254 |
CurrentResourceOwner == NULL); |
| 255 |
CurrentResourceOwner = AuxProcessResourceOwner; |
| 256 |
|
| 257 |
backup_started_in_recovery = RecoveryInProgress(); |
| 258 |
|
| 259 |
InitializeBackupManifest(&manifest, opt->manifest, |
| 260 |
opt->manifest_checksum_type); |
| 261 |
|
| 262 |
total_checksum_failures = 0; |
| 263 |
|
| 264 |
/* Allocate backup related variables. */ |
| 265 |
backup_state = (BackupState *) palloc0(sizeof(BackupState)); |
| 266 |
tablespace_map = makeStringInfo(); |
| 267 |
|
| 268 |
basebackup_progress_wait_checkpoint(); |
| 269 |
do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces, |
| 270 |
backup_state, tablespace_map); |
| 271 |
|
| 272 |
state.startptr = backup_state->startpoint; |
| 273 |
state.starttli = backup_state->starttli; |
| 274 |
|
| 275 |
/* |
| 276 |
* Once do_pg_backup_start has been called, ensure that any failure causes |
| 277 |
* us to abort the backup so we don't "leak" a backup counter. For this |
| 278 |
* reason, *all* functionality between do_pg_backup_start() and the end of |
| 279 |
* do_pg_backup_stop() should be inside the error cleanup block! |
| 280 |
*/ |
| 281 |
|
| 282 |
PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false)); |
| 283 |
{ |
| 284 |
ListCell *lc; |
| 285 |
tablespaceinfo *newti; |
| 286 |
|
| 287 |
/* If this is an incremental backup, execute preparatory steps. */ |
| 288 |
if (ib != NULL) |
| 289 |
PrepareForIncrementalBackup(ib, backup_state); |
| 290 |
|
| 291 |
/* Add a node for the base directory at the end */ |
| 292 |
newti = palloc0(sizeof(tablespaceinfo)); |
| 293 |
newti->size = -1; |
| 294 |
state.tablespaces = lappend(state.tablespaces, newti); |
| 295 |
|
| 296 |
/* |
| 297 |
* Calculate the total backup size by summing up the size of each |
| 298 |
* tablespace |
| 299 |
*/ |
| 300 |
if (opt->progress) |
| 301 |
{ |
| 302 |
basebackup_progress_estimate_backup_size(); |
| 303 |
|
| 304 |
foreach(lc, state.tablespaces) |
| 305 |
{ |
| 306 |
tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc); |
| 307 |
|
| 308 |
if (tmp->path == NULL) |
| 309 |
tmp->size = sendDir(sink, ".", 1, true, state.tablespaces, |
| 310 |
true, NULL, InvalidOid, NULL); |
| 311 |
else |
| 312 |
tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true, |
| 313 |
NULL, NULL); |
| 314 |
state.bytes_total += tmp->size; |
| 315 |
} |
| 316 |
state.bytes_total_is_valid = true; |
| 317 |
} |
| 318 |
|
| 319 |
/* notify basebackup sink about start of backup */ |
| 320 |
bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH); |
| 321 |
|
| 322 |
/* Send off our tablespaces one by one */ |
| 323 |
foreach(lc, state.tablespaces) |
| 324 |
{ |
| 325 |
tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc); |
| 326 |
|
| 327 |
if (ti->path == NULL) |
| 328 |
{ |
| 329 |
struct stat statbuf; |
| 330 |
bool sendtblspclinks = true; |
| 331 |
char *backup_label; |
| 332 |
|
| 333 |
bbsink_begin_archive(sink, "base.tar"); |
| 334 |
|
| 335 |
/* In the main tar, include the backup_label first... */ |
| 336 |
backup_label = build_backup_content(backup_state, false); |
| 337 |
sendFileWithContent(sink, BACKUP_LABEL_FILE, |
| 338 |
backup_label, -1, &manifest); |
| 339 |
pfree(backup_label); |
| 340 |
|
| 341 |
/* Then the tablespace_map file, if required... */ |
| 342 |
if (opt->sendtblspcmapfile) |
| 343 |
{ |
| 344 |
sendFileWithContent(sink, TABLESPACE_MAP, |
| 345 |
tablespace_map->data, -1, &manifest); |
| 346 |
sendtblspclinks = false; |
| 347 |
} |
| 348 |
|
| 349 |
/* Then the bulk of the files... */ |
| 350 |
sendDir(sink, ".", 1, false, state.tablespaces, |
| 351 |
sendtblspclinks, &manifest, InvalidOid, ib); |
| 352 |
|
| 353 |
/* ... and pg_control after everything else. */ |
| 354 |
if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) |
| 355 |
ereport(ERROR, |
| 356 |
(errcode_for_file_access(), |
| 357 |
errmsg("could not stat file \"%s\": %m", |
| 358 |
XLOG_CONTROL_FILE))); |
| 359 |
sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, |
| 360 |
false, InvalidOid, InvalidOid, |
| 361 |
InvalidRelFileNumber, 0, &manifest, 0, NULL, 0); |
| 362 |
} |
| 363 |
else |
| 364 |
{ |
| 365 |
char *archive_name = psprintf("%u.tar", ti->oid); |
| 366 |
|
| 367 |
bbsink_begin_archive(sink, archive_name); |
| 368 |
|
| 369 |
sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib); |
| 370 |
} |
| 371 |
|
| 372 |
/* |
| 373 |
* If we're including WAL, and this is the main data directory we |
| 374 |
* don't treat this as the end of the tablespace. Instead, we will |
| 375 |
* include the xlog files below and stop afterwards. This is safe |
| 376 |
* since the main data directory is always sent *last*. |
| 377 |
*/ |
| 378 |
if (opt->includewal && ti->path == NULL) |
| 379 |
{ |
| 380 |
Assert(lnext(state.tablespaces, lc) == NULL); |
| 381 |
} |
| 382 |
else |
| 383 |
{ |
| 384 |
/* Properly terminate the tarfile. */ |
| 385 |
StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ, |
| 386 |
"BLCKSZ too small for 2 tar blocks"); |
| 387 |
memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE); |
| 388 |
bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE); |
| 389 |
|
| 390 |
/* OK, that's the end of the archive. */ |
| 391 |
bbsink_end_archive(sink); |
| 392 |
} |
| 393 |
} |
| 394 |
|
| 395 |
basebackup_progress_wait_wal_archive(&state); |
| 396 |
do_pg_backup_stop(backup_state, !opt->nowait); |
| 397 |
|
| 398 |
endptr = backup_state->stoppoint; |
| 399 |
endtli = backup_state->stoptli; |
| 400 |
|
| 401 |
/* Deallocate backup-related variables. */ |
| 402 |
destroyStringInfo(tablespace_map); |
| 403 |
pfree(backup_state); |
| 404 |
} |
| 405 |
PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false)); |
| 406 |
|
| 407 |
|
| 408 |
if (opt->includewal) |
| 409 |
{ |
| 410 |
/* |
| 411 |
* We've left the last tar file "open", so we can now append the |
| 412 |
* required WAL files to it. |
| 413 |
*/ |
| 414 |
char pathbuf[MAXPGPATH]; |
| 415 |
XLogSegNo segno; |
| 416 |
XLogSegNo startsegno; |
| 417 |
XLogSegNo endsegno; |
| 418 |
struct stat statbuf; |
| 419 |
List *historyFileList = NIL; |
| 420 |
List *walFileList = NIL; |
| 421 |
char firstoff[MAXFNAMELEN]; |
| 422 |
char lastoff[MAXFNAMELEN]; |
| 423 |
DIR *dir; |
| 424 |
struct dirent *de; |
| 425 |
ListCell *lc; |
| 426 |
TimeLineID tli; |
| 427 |
|
| 428 |
basebackup_progress_transfer_wal(); |
| 429 |
|
| 430 |
/* |
| 431 |
* I'd rather not worry about timelines here, so scan pg_wal and |
| 432 |
* include all WAL files in the range between 'startptr' and 'endptr', |
| 433 |
* regardless of the timeline the file is stamped with. If there are |
| 434 |
* some spurious WAL files belonging to timelines that don't belong in |
| 435 |
* this server's history, they will be included too. Normally there |
| 436 |
* shouldn't be such files, but if there are, there's little harm in |
| 437 |
* including them. |
| 438 |
*/ |
| 439 |
XLByteToSeg(state.startptr, startsegno, wal_segment_size); |
| 440 |
XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size); |
| 441 |
XLByteToPrevSeg(endptr, endsegno, wal_segment_size); |
| 442 |
XLogFileName(lastoff, endtli, endsegno, wal_segment_size); |
| 443 |
|
| 444 |
dir = AllocateDir("pg_wal"); |
| 445 |
while ((de = ReadDir(dir, "pg_wal")) != NULL) |
| 446 |
{ |
| 447 |
/* Does it look like a WAL segment, and is it in the range? */ |
| 448 |
if (IsXLogFileName(de->d_name) && |
| 449 |
strcmp(de->d_name + 8, firstoff + 8) >= 0 && |
| 450 |
strcmp(de->d_name + 8, lastoff + 8) <= 0) |
| 451 |
{ |
| 452 |
walFileList = lappend(walFileList, pstrdup(de->d_name)); |
| 453 |
} |
| 454 |
/* Does it look like a timeline history file? */ |
| 455 |
else if (IsTLHistoryFileName(de->d_name)) |
| 456 |
{ |
| 457 |
historyFileList = lappend(historyFileList, pstrdup(de->d_name)); |
| 458 |
} |
| 459 |
} |
| 460 |
FreeDir(dir); |
| 461 |
|
| 462 |
/* |
| 463 |
* Before we go any further, check that none of the WAL segments we |
| 464 |
* need were removed. |
| 465 |
*/ |
| 466 |
CheckXLogRemoved(startsegno, state.starttli); |
| 467 |
|
| 468 |
/* |
| 469 |
* Sort the WAL filenames. We want to send the files in order from |
| 470 |
* oldest to newest, to reduce the chance that a file is recycled |
| 471 |
* before we get a chance to send it over. |
| 472 |
*/ |
| 473 |
list_sort(walFileList, compareWalFileNames); |
| 474 |
|
| 475 |
/* |
| 476 |
* There must be at least one xlog file in the pg_wal directory, since |
| 477 |
* we are doing backup-including-xlog. |
| 478 |
*/ |
| 479 |
if (walFileList == NIL) |
| 480 |
ereport(ERROR, |
| 481 |
(errmsg("could not find any WAL files"))); |
| 482 |
|
| 483 |
/* |
| 484 |
* Sanity check: the first and last segment should cover startptr and |
| 485 |
* endptr, with no gaps in between. |
| 486 |
*/ |
| 487 |
XLogFromFileName((char *) linitial(walFileList), |
| 488 |
&tli, &segno, wal_segment_size); |
| 489 |
if (segno != startsegno) |
| 490 |
{ |
| 491 |
char startfname[MAXFNAMELEN]; |
| 492 |
|
| 493 |
XLogFileName(startfname, state.starttli, startsegno, |
| 494 |
wal_segment_size); |
| 495 |
ereport(ERROR, |
| 496 |
(errmsg("could not find WAL file \"%s\"", startfname))); |
| 497 |
} |
| 498 |
foreach(lc, walFileList) |
| 499 |
{ |
| 500 |
char *walFileName = (char *) lfirst(lc); |
| 501 |
XLogSegNo currsegno = segno; |
| 502 |
XLogSegNo nextsegno = segno + 1; |
| 503 |
|
| 504 |
XLogFromFileName(walFileName, &tli, &segno, wal_segment_size); |
| 505 |
if (!(nextsegno == segno || currsegno == segno)) |
| 506 |
{ |
| 507 |
char nextfname[MAXFNAMELEN]; |
| 508 |
|
| 509 |
XLogFileName(nextfname, tli, nextsegno, wal_segment_size); |
| 510 |
ereport(ERROR, |
| 511 |
(errmsg("could not find WAL file \"%s\"", nextfname))); |
| 512 |
} |
| 513 |
} |
| 514 |
if (segno != endsegno) |
| 515 |
{ |
| 516 |
char endfname[MAXFNAMELEN]; |
| 517 |
|
| 518 |
XLogFileName(endfname, endtli, endsegno, wal_segment_size); |
| 519 |
ereport(ERROR, |
| 520 |
(errmsg("could not find WAL file \"%s\"", endfname))); |
| 521 |
} |
| 522 |
|
| 523 |
/* Ok, we have everything we need. Send the WAL files. */ |
| 524 |
foreach(lc, walFileList) |
| 525 |
{ |
| 526 |
char *walFileName = (char *) lfirst(lc); |
| 527 |
int fd; |
| 528 |
ssize_t cnt; |
| 529 |
pgoff_t len = 0; |
| 530 |
|
| 531 |
snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName); |
| 532 |
XLogFromFileName(walFileName, &tli, &segno, wal_segment_size); |
| 533 |
|
| 534 |
fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY); |
| 535 |
if (fd < 0) |
| 536 |
{ |
| 537 |
int save_errno = errno; |
| 538 |
|
| 539 |
/* |
| 540 |
* Most likely reason for this is that the file was already |
| 541 |
* removed by a checkpoint, so check for that to get a better |
| 542 |
* error message. |
| 543 |
*/ |
| 544 |
CheckXLogRemoved(segno, tli); |
| 545 |
|
| 546 |
errno = save_errno; |
| 547 |
ereport(ERROR, |
| 548 |
(errcode_for_file_access(), |
| 549 |
errmsg("could not open file \"%s\": %m", pathbuf))); |
| 550 |
} |
| 551 |
|
| 552 |
if (fstat(fd, &statbuf) != 0) |
| 553 |
ereport(ERROR, |
| 554 |
(errcode_for_file_access(), |
| 555 |
errmsg("could not stat file \"%s\": %m", |
| 556 |
pathbuf))); |
| 557 |
if (statbuf.st_size != wal_segment_size) |
| 558 |
{ |
| 559 |
CheckXLogRemoved(segno, tli); |
| 560 |
ereport(ERROR, |
| 561 |
(errcode_for_file_access(), |
| 562 |
errmsg("unexpected WAL file size \"%s\"", walFileName))); |
| 563 |
} |
| 564 |
|
| 565 |
/* send the WAL file itself */ |
| 566 |
_tarWriteHeader(sink, pathbuf, NULL, &statbuf, false); |
| 567 |
|
| 568 |
while ((cnt = basebackup_read_file(fd, sink->bbs_buffer, |
| 569 |
Min(sink->bbs_buffer_length, |
| 570 |
wal_segment_size - len), |
| 571 |
len, pathbuf, true)) > 0) |
| 572 |
{ |
| 573 |
CheckXLogRemoved(segno, tli); |
| 574 |
bbsink_archive_contents(sink, cnt); |
| 575 |
|
| 576 |
len += cnt; |
| 577 |
|
| 578 |
if (len == wal_segment_size) |
| 579 |
break; |
| 580 |
} |
| 581 |
|
| 582 |
if (len != wal_segment_size) |
| 583 |
{ |
| 584 |
CheckXLogRemoved(segno, tli); |
| 585 |
ereport(ERROR, |
| 586 |
(errcode_for_file_access(), |
| 587 |
errmsg("unexpected WAL file size \"%s\"", walFileName))); |
| 588 |
} |
| 589 |
|
| 590 |
/* |
| 591 |
* wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need |
| 592 |
* for padding. |
| 593 |
*/ |
| 594 |
Assert(wal_segment_size % TAR_BLOCK_SIZE == 0); |
| 595 |
|
| 596 |
CloseTransientFile(fd); |
| 597 |
|
| 598 |
/* |
| 599 |
* Mark file as archived, otherwise files can get archived again |
| 600 |
* after promotion of a new node. This is in line with |
| 601 |
* walreceiver.c always doing an XLogArchiveForceDone() after a |
| 602 |
* complete segment. |
| 603 |
*/ |
| 604 |
StatusFilePath(pathbuf, walFileName, ".done"); |
| 605 |
sendFileWithContent(sink, pathbuf, "", -1, &manifest); |
| 606 |
} |
| 607 |
|
| 608 |
/* |
| 609 |
* Send timeline history files too. Only the latest timeline history |
| 610 |
* file is required for recovery, and even that only if there happens |
| 611 |
* to be a timeline switch in the first WAL segment that contains the |
| 612 |
* checkpoint record, or if we're taking a base backup from a standby |
| 613 |
* server and the target timeline changes while the backup is taken. |
| 614 |
* But they are small and highly useful for debugging purposes, so |
| 615 |
* better include them all, always. |
| 616 |
*/ |
| 617 |
foreach(lc, historyFileList) |
| 618 |
{ |
| 619 |
char *fname = lfirst(lc); |
| 620 |
|
| 621 |
snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname); |
| 622 |
|
| 623 |
if (lstat(pathbuf, &statbuf) != 0) |
| 624 |
ereport(ERROR, |
| 625 |
(errcode_for_file_access(), |
| 626 |
errmsg("could not stat file \"%s\": %m", pathbuf))); |
| 627 |
|
| 628 |
sendFile(sink, pathbuf, pathbuf, &statbuf, false, |
| 629 |
InvalidOid, InvalidOid, InvalidRelFileNumber, 0, |
| 630 |
&manifest, 0, NULL, 0); |
| 631 |
|
| 632 |
/* unconditionally mark file as archived */ |
| 633 |
StatusFilePath(pathbuf, fname, ".done"); |
| 634 |
sendFileWithContent(sink, pathbuf, "", -1, &manifest); |
| 635 |
} |
| 636 |
|
| 637 |
/* Properly terminate the tar file. */ |
| 638 |
StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ, |
| 639 |
"BLCKSZ too small for 2 tar blocks"); |
| 640 |
memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE); |
| 641 |
bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE); |
| 642 |
|
| 643 |
/* OK, that's the end of the archive. */ |
| 644 |
bbsink_end_archive(sink); |
| 645 |
} |
| 646 |
|
| 647 |
AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli, |
| 648 |
endptr, endtli); |
| 649 |
|
| 650 |
SendBackupManifest(&manifest, sink); |
| 651 |
|
| 652 |
bbsink_end_backup(sink, endptr, endtli); |
| 653 |
|
| 654 |
if (total_checksum_failures) |
| 655 |
{ |
| 656 |
if (total_checksum_failures > 1) |
| 657 |
ereport(WARNING, |
| 658 |
(errmsg_plural("%lld total checksum verification failure", |
| 659 |
"%lld total checksum verification failures", |
| 660 |
total_checksum_failures, |
| 661 |
total_checksum_failures))); |
| 662 |
|
| 663 |
ereport(ERROR, |
| 664 |
(errcode(ERRCODE_DATA_CORRUPTED), |
| 665 |
errmsg("checksum verification failure during base backup"))); |
| 666 |
} |
| 667 |
|
| 668 |
/* |
| 669 |
* Make sure to free the manifest before the resource owners as manifests |
| 670 |
* use cryptohash contexts that may depend on resource owners (like |
| 671 |
* OpenSSL). |
| 672 |
*/ |
| 673 |
FreeBackupManifest(&manifest); |
| 674 |
|
| 675 |
/* clean up the resource owner we created */ |
| 676 |
ReleaseAuxProcessResources(true); |
| 677 |
} |
| 678 |
|
| 679 |
/* |
| 680 |
* list_sort comparison function, to compare log/seg portion of WAL segment |
| 681 |
* filenames, ignoring the timeline portion. |
| 682 |
*/ |
| 683 |
static int |
| 684 |
compareWalFileNames(const ListCell *a, const ListCell *b) |
| 685 |
{ |
| 686 |
char *fna = (char *) lfirst(a); |
| 687 |
char *fnb = (char *) lfirst(b); |
| 688 |
|
| 689 |
return strcmp(fna + 8, fnb + 8); |
| 690 |
} |
| 691 |
|
| 692 |
/* |
| 693 |
* Parse the base backup options passed down by the parser |
| 694 |
*/ |
| 695 |
static void |
| 696 |
parse_basebackup_options(List *options, basebackup_options *opt) |
| 697 |
{ |
| 698 |
ListCell *lopt; |
| 699 |
bool o_label = false; |
| 700 |
bool o_progress = false; |
| 701 |
bool o_checkpoint = false; |
| 702 |
bool o_nowait = false; |
| 703 |
bool o_wal = false; |
| 704 |
bool o_incremental = false; |
| 705 |
bool o_maxrate = false; |
| 706 |
bool o_tablespace_map = false; |
| 707 |
bool o_noverify_checksums = false; |
| 708 |
bool o_manifest = false; |
| 709 |
bool o_manifest_checksums = false; |
| 710 |
bool o_target = false; |
| 711 |
bool o_target_detail = false; |
| 712 |
char *target_str = NULL; |
| 713 |
char *target_detail_str = NULL; |
| 714 |
bool o_compression = false; |
| 715 |
bool o_compression_detail = false; |
| 716 |
char *compression_detail_str = NULL; |
| 717 |
|
| 718 |
MemSet(opt, 0, sizeof(*opt)); |
| 719 |
opt->manifest = MANIFEST_OPTION_NO; |
| 720 |
opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C; |
| 721 |
opt->compression = PG_COMPRESSION_NONE; |
| 722 |
opt->compression_specification.algorithm = PG_COMPRESSION_NONE; |
| 723 |
|
| 724 |
foreach(lopt, options) |
| 725 |
{ |
| 726 |
DefElem *defel = (DefElem *) lfirst(lopt); |
| 727 |
|
| 728 |
if (strcmp(defel->defname, "label") == 0) |
| 729 |
{ |
| 730 |
if (o_label) |
| 731 |
ereport(ERROR, |
| 732 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 733 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 734 |
opt->label = defGetString(defel); |
| 735 |
o_label = true; |
| 736 |
} |
| 737 |
else if (strcmp(defel->defname, "progress") == 0) |
| 738 |
{ |
| 739 |
if (o_progress) |
| 740 |
ereport(ERROR, |
| 741 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 742 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 743 |
opt->progress = defGetBoolean(defel); |
| 744 |
o_progress = true; |
| 745 |
} |
| 746 |
else if (strcmp(defel->defname, "checkpoint") == 0) |
| 747 |
{ |
| 748 |
char *optval = defGetString(defel); |
| 749 |
|
| 750 |
if (o_checkpoint) |
| 751 |
ereport(ERROR, |
| 752 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 753 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 754 |
if (pg_strcasecmp(optval, "fast") == 0) |
| 755 |
opt->fastcheckpoint = true; |
| 756 |
else if (pg_strcasecmp(optval, "spread") == 0) |
| 757 |
opt->fastcheckpoint = false; |
| 758 |
else |
| 759 |
ereport(ERROR, |
| 760 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 761 |
errmsg("unrecognized checkpoint type: \"%s\"", |
| 762 |
optval))); |
| 763 |
o_checkpoint = true; |
| 764 |
} |
| 765 |
else if (strcmp(defel->defname, "wait") == 0) |
| 766 |
{ |
| 767 |
if (o_nowait) |
| 768 |
ereport(ERROR, |
| 769 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 770 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 771 |
opt->nowait = !defGetBoolean(defel); |
| 772 |
o_nowait = true; |
| 773 |
} |
| 774 |
else if (strcmp(defel->defname, "wal") == 0) |
| 775 |
{ |
| 776 |
if (o_wal) |
| 777 |
ereport(ERROR, |
| 778 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 779 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 780 |
opt->includewal = defGetBoolean(defel); |
| 781 |
o_wal = true; |
| 782 |
} |
| 783 |
else if (strcmp(defel->defname, "incremental") == 0) |
| 784 |
{ |
| 785 |
if (o_incremental) |
| 786 |
ereport(ERROR, |
| 787 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 788 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 789 |
opt->incremental = defGetBoolean(defel); |
| 790 |
if (opt->incremental && !summarize_wal) |
| 791 |
ereport(ERROR, |
| 792 |
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), |
| 793 |
errmsg("incremental backups cannot be taken unless WAL summarization is enabled"))); |
| 794 |
o_incremental = true; |
| 795 |
} |
| 796 |
else if (strcmp(defel->defname, "max_rate") == 0) |
| 797 |
{ |
| 798 |
int64 maxrate; |
| 799 |
|
| 800 |
if (o_maxrate) |
| 801 |
ereport(ERROR, |
| 802 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 803 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 804 |
|
| 805 |
maxrate = defGetInt64(defel); |
| 806 |
if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER) |
| 807 |
ereport(ERROR, |
| 808 |
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), |
| 809 |
errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)", |
| 810 |
(int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER))); |
| 811 |
|
| 812 |
opt->maxrate = (uint32) maxrate; |
| 813 |
o_maxrate = true; |
| 814 |
} |
| 815 |
else if (strcmp(defel->defname, "tablespace_map") == 0) |
| 816 |
{ |
| 817 |
if (o_tablespace_map) |
| 818 |
ereport(ERROR, |
| 819 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 820 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 821 |
opt->sendtblspcmapfile = defGetBoolean(defel); |
| 822 |
o_tablespace_map = true; |
| 823 |
} |
| 824 |
else if (strcmp(defel->defname, "verify_checksums") == 0) |
| 825 |
{ |
| 826 |
if (o_noverify_checksums) |
| 827 |
ereport(ERROR, |
| 828 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 829 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 830 |
noverify_checksums = !defGetBoolean(defel); |
| 831 |
o_noverify_checksums = true; |
| 832 |
} |
| 833 |
else if (strcmp(defel->defname, "manifest") == 0) |
| 834 |
{ |
| 835 |
char *optval = defGetString(defel); |
| 836 |
bool manifest_bool; |
| 837 |
|
| 838 |
if (o_manifest) |
| 839 |
ereport(ERROR, |
| 840 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 841 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 842 |
if (parse_bool(optval, &manifest_bool)) |
| 843 |
{ |
| 844 |
if (manifest_bool) |
| 845 |
opt->manifest = MANIFEST_OPTION_YES; |
| 846 |
else |
| 847 |
opt->manifest = MANIFEST_OPTION_NO; |
| 848 |
} |
| 849 |
else if (pg_strcasecmp(optval, "force-encode") == 0) |
| 850 |
opt->manifest = MANIFEST_OPTION_FORCE_ENCODE; |
| 851 |
else |
| 852 |
ereport(ERROR, |
| 853 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 854 |
errmsg("unrecognized manifest option: \"%s\"", |
| 855 |
optval))); |
| 856 |
o_manifest = true; |
| 857 |
} |
| 858 |
else if (strcmp(defel->defname, "manifest_checksums") == 0) |
| 859 |
{ |
| 860 |
char *optval = defGetString(defel); |
| 861 |
|
| 862 |
if (o_manifest_checksums) |
| 863 |
ereport(ERROR, |
| 864 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 865 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 866 |
if (!pg_checksum_parse_type(optval, |
| 867 |
&opt->manifest_checksum_type)) |
| 868 |
ereport(ERROR, |
| 869 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 870 |
errmsg("unrecognized checksum algorithm: \"%s\"", |
| 871 |
optval))); |
| 872 |
o_manifest_checksums = true; |
| 873 |
} |
| 874 |
else if (strcmp(defel->defname, "target") == 0) |
| 875 |
{ |
| 876 |
if (o_target) |
| 877 |
ereport(ERROR, |
| 878 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 879 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 880 |
target_str = defGetString(defel); |
| 881 |
o_target = true; |
| 882 |
} |
| 883 |
else if (strcmp(defel->defname, "target_detail") == 0) |
| 884 |
{ |
| 885 |
char *optval = defGetString(defel); |
| 886 |
|
| 887 |
if (o_target_detail) |
| 888 |
ereport(ERROR, |
| 889 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 890 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 891 |
target_detail_str = optval; |
| 892 |
o_target_detail = true; |
| 893 |
} |
| 894 |
else if (strcmp(defel->defname, "compression") == 0) |
| 895 |
{ |
| 896 |
char *optval = defGetString(defel); |
| 897 |
|
| 898 |
if (o_compression) |
| 899 |
ereport(ERROR, |
| 900 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 901 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 902 |
if (!parse_compress_algorithm(optval, &opt->compression)) |
| 903 |
ereport(ERROR, |
| 904 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 905 |
errmsg("unrecognized compression algorithm: \"%s\"", |
| 906 |
optval))); |
| 907 |
o_compression = true; |
| 908 |
} |
| 909 |
else if (strcmp(defel->defname, "compression_detail") == 0) |
| 910 |
{ |
| 911 |
if (o_compression_detail) |
| 912 |
ereport(ERROR, |
| 913 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 914 |
errmsg("duplicate option \"%s\"", defel->defname))); |
| 915 |
compression_detail_str = defGetString(defel); |
| 916 |
o_compression_detail = true; |
| 917 |
} |
| 918 |
else |
| 919 |
ereport(ERROR, |
| 920 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 921 |
errmsg("unrecognized base backup option: \"%s\"", |
| 922 |
defel->defname))); |
| 923 |
} |
| 924 |
|
| 925 |
if (opt->label == NULL) |
| 926 |
opt->label = "base backup"; |
| 927 |
if (opt->manifest == MANIFEST_OPTION_NO) |
| 928 |
{ |
| 929 |
if (o_manifest_checksums) |
| 930 |
ereport(ERROR, |
| 931 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 932 |
errmsg("manifest checksums require a backup manifest"))); |
| 933 |
opt->manifest_checksum_type = CHECKSUM_TYPE_NONE; |
| 934 |
} |
| 935 |
|
| 936 |
if (target_str == NULL) |
| 937 |
{ |
| 938 |
if (target_detail_str != NULL) |
| 939 |
ereport(ERROR, |
| 940 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 941 |
errmsg("target detail cannot be used without target"))); |
| 942 |
opt->use_copytblspc = true; |
| 943 |
opt->send_to_client = true; |
| 944 |
} |
| 945 |
else if (strcmp(target_str, "client") == 0) |
| 946 |
{ |
| 947 |
if (target_detail_str != NULL) |
| 948 |
ereport(ERROR, |
| 949 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 950 |
errmsg("target \"%s\" does not accept a target detail", |
| 951 |
target_str))); |
| 952 |
opt->send_to_client = true; |
| 953 |
} |
| 954 |
else |
| 955 |
opt->target_handle = |
| 956 |
BaseBackupGetTargetHandle(target_str, target_detail_str); |
| 957 |
|
| 958 |
if (o_compression_detail && !o_compression) |
| 959 |
ereport(ERROR, |
| 960 |
(errcode(ERRCODE_SYNTAX_ERROR), |
| 961 |
errmsg("compression detail cannot be specified unless compression is enabled"))); |
| 962 |
|
| 963 |
if (o_compression) |
| 964 |
{ |
| 965 |
char *error_detail; |
| 966 |
|
| 967 |
parse_compress_specification(opt->compression, compression_detail_str, |
| 968 |
&opt->compression_specification); |
| 969 |
error_detail = |
| 970 |
validate_compress_specification(&opt->compression_specification); |
| 971 |
if (error_detail != NULL) |
| 972 |
ereport(ERROR, |
| 973 |
errcode(ERRCODE_SYNTAX_ERROR), |
| 974 |
errmsg("invalid compression specification: %s", |
| 975 |
error_detail)); |
| 976 |
} |
| 977 |
} |
| 978 |
|
| 979 |
|
| 980 |
/* |
| 981 |
* SendBaseBackup() - send a complete base backup. |
| 982 |
* |
| 983 |
* The function will put the system into backup mode like pg_backup_start() |
| 984 |
* does, so that the backup is consistent even though we read directly from |
| 985 |
* the filesystem, bypassing the buffer cache. |
| 986 |
*/ |
| 987 |
void |
| 988 |
SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib) |
| 989 |
{ |
| 990 |
basebackup_options opt; |
| 991 |
bbsink *sink; |
| 992 |
SessionBackupState status = get_backup_status(); |
| 993 |
|
| 994 |
if (status == SESSION_BACKUP_RUNNING) |
| 995 |
ereport(ERROR, |
| 996 |
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), |
| 997 |
errmsg("a backup is already in progress in this session"))); |
| 998 |
|
| 999 |
parse_basebackup_options(cmd->options, &opt); |
| 1000 |
|
| 1001 |
WalSndSetState(WALSNDSTATE_BACKUP); |
| 1002 |
|
| 1003 |
if (update_process_title) |
| 1004 |
{ |
| 1005 |
char activitymsg[50]; |
| 1006 |
|
| 1007 |
snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"", |
| 1008 |
opt.label); |
| 1009 |
set_ps_display(activitymsg); |
| 1010 |
} |
| 1011 |
|
| 1012 |
/* |
| 1013 |
* If we're asked to perform an incremental backup and the user has not |
| 1014 |
* supplied a manifest, that's an ERROR. |
| 1015 |
* |
| 1016 |
* If we're asked to perform a full backup and the user did supply a |
| 1017 |
* manifest, just ignore it. |
| 1018 |
*/ |
| 1019 |
if (!opt.incremental) |
| 1020 |
ib = NULL; |
| 1021 |
else if (ib == NULL) |
| 1022 |
ereport(ERROR, |
| 1023 |
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), |
| 1024 |
errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP"))); |
| 1025 |
|
| 1026 |
/* |
| 1027 |
* If the target is specifically 'client' then set up to stream the backup |
| 1028 |
* to the client; otherwise, it's being sent someplace else and should not |
| 1029 |
* be sent to the client. BaseBackupGetSink has the job of setting up a |
| 1030 |
* sink to send the backup data wherever it needs to go. |
| 1031 |
*/ |
| 1032 |
sink = bbsink_copystream_new(opt.send_to_client); |
| 1033 |
if (opt.target_handle != NULL) |
| 1034 |
sink = BaseBackupGetSink(opt.target_handle, sink); |
| 1035 |
|
| 1036 |
/* Set up network throttling, if client requested it */ |
| 1037 |
if (opt.maxrate > 0) |
| 1038 |
sink = bbsink_throttle_new(sink, opt.maxrate); |
| 1039 |
|
| 1040 |
/* Set up server-side compression, if client requested it */ |
| 1041 |
if (opt.compression == PG_COMPRESSION_GZIP) |
| 1042 |
sink = bbsink_gzip_new(sink, &opt.compression_specification); |
| 1043 |
else if (opt.compression == PG_COMPRESSION_LZ4) |
| 1044 |
sink = bbsink_lz4_new(sink, &opt.compression_specification); |
| 1045 |
else if (opt.compression == PG_COMPRESSION_ZSTD) |
| 1046 |
sink = bbsink_zstd_new(sink, &opt.compression_specification); |
| 1047 |
|
| 1048 |
/* Set up progress reporting. */ |
| 1049 |
sink = bbsink_progress_new(sink, opt.progress); |
| 1050 |
|
| 1051 |
/* |
| 1052 |
* Perform the base backup, but make sure we clean up the bbsink even if |
| 1053 |
* an error occurs. |
| 1054 |
*/ |
| 1055 |
PG_TRY(); |
| 1056 |
{ |
| 1057 |
perform_base_backup(&opt, sink, ib); |
| 1058 |
} |
| 1059 |
PG_FINALLY(); |
| 1060 |
{ |
| 1061 |
bbsink_cleanup(sink); |
| 1062 |
} |
| 1063 |
PG_END_TRY(); |
| 1064 |
} |
| 1065 |
|
| 1066 |
/* |
| 1067 |
* Inject a file with given name and content in the output tar stream. |
| 1068 |
* |
| 1069 |
* "len" can optionally be set to an arbitrary length of data sent. If set |
| 1070 |
* to -1, the content sent is treated as a string with strlen() as length. |
| 1071 |
*/ |
| 1072 |
static void |
| 1073 |
sendFileWithContent(bbsink *sink, const char *filename, const char *content, |
| 1074 |
int len, backup_manifest_info *manifest) |
| 1075 |
{ |
| 1076 |
struct stat statbuf; |
| 1077 |
int bytes_done = 0; |
| 1078 |
pg_checksum_context checksum_ctx; |
| 1079 |
|
| 1080 |
if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0) |
| 1081 |
elog(ERROR, "could not initialize checksum of file \"%s\"", |
| 1082 |
filename); |
| 1083 |
|
| 1084 |
if (len < 0) |
| 1085 |
len = strlen(content); |
| 1086 |
|
| 1087 |
/* |
| 1088 |
* Construct a stat struct for the file we're injecting in the tar. |
| 1089 |
*/ |
| 1090 |
|
| 1091 |
/* Windows doesn't have the concept of uid and gid */ |
| 1092 |
#ifdef WIN32 |
| 1093 |
statbuf.st_uid = 0; |
| 1094 |
statbuf.st_gid = 0; |
| 1095 |
#else |
| 1096 |
statbuf.st_uid = geteuid(); |
| 1097 |
statbuf.st_gid = getegid(); |
| 1098 |
#endif |
| 1099 |
statbuf.st_mtime = time(NULL); |
| 1100 |
statbuf.st_mode = pg_file_create_mode; |
| 1101 |
statbuf.st_size = len; |
| 1102 |
|
| 1103 |
_tarWriteHeader(sink, filename, NULL, &statbuf, false); |
| 1104 |
|
| 1105 |
if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0) |
| 1106 |
elog(ERROR, "could not update checksum of file \"%s\"", |
| 1107 |
filename); |
| 1108 |
|
| 1109 |
while (bytes_done < len) |
| 1110 |
{ |
| 1111 |
size_t remaining = len - bytes_done; |
| 1112 |
size_t nbytes = Min(sink->bbs_buffer_length, remaining); |
| 1113 |
|
| 1114 |
memcpy(sink->bbs_buffer, content, nbytes); |
| 1115 |
bbsink_archive_contents(sink, nbytes); |
| 1116 |
bytes_done += nbytes; |
| 1117 |
content += nbytes; |
| 1118 |
} |
| 1119 |
|
| 1120 |
_tarWritePadding(sink, len); |
| 1121 |
|
| 1122 |
AddFileToBackupManifest(manifest, InvalidOid, filename, len, |
| 1123 |
(pg_time_t) statbuf.st_mtime, &checksum_ctx); |
| 1124 |
} |
| 1125 |
|
| 1126 |
/* |
| 1127 |
* Include the tablespace directory pointed to by 'path' in the output tar |
| 1128 |
* stream. If 'sizeonly' is true, we just calculate a total length and return |
| 1129 |
* it, without actually sending anything. |
| 1130 |
* |
| 1131 |
* Only used to send auxiliary tablespaces, not PGDATA. |
| 1132 |
*/ |
| 1133 |
static int64 |
| 1134 |
sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly, |
| 1135 |
backup_manifest_info *manifest, IncrementalBackupInfo *ib) |
| 1136 |
{ |
| 1137 |
int64 size; |
| 1138 |
char pathbuf[MAXPGPATH]; |
| 1139 |
struct stat statbuf; |
| 1140 |
|
| 1141 |
/* |
| 1142 |
* 'path' points to the tablespace location, but we only want to include |
| 1143 |
* the version directory in it that belongs to us. |
| 1144 |
*/ |
| 1145 |
snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, |
| 1146 |
TABLESPACE_VERSION_DIRECTORY); |
| 1147 |
|
| 1148 |
/* |
| 1149 |
* Store a directory entry in the tar file so we get the permissions |
| 1150 |
* right. |
| 1151 |
*/ |
| 1152 |
if (lstat(pathbuf, &statbuf) != 0) |
| 1153 |
{ |
| 1154 |
if (errno != ENOENT) |
| 1155 |
ereport(ERROR, |
| 1156 |
(errcode_for_file_access(), |
| 1157 |
errmsg("could not stat file or directory \"%s\": %m", |
| 1158 |
pathbuf))); |
| 1159 |
|
| 1160 |
/* If the tablespace went away while scanning, it's no error. */ |
| 1161 |
return 0; |
| 1162 |
} |
| 1163 |
|
| 1164 |
size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf, |
| 1165 |
sizeonly); |
| 1166 |
|
| 1167 |
/* Send all the files in the tablespace version directory */ |
| 1168 |
size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest, |
| 1169 |
spcoid, ib); |
| 1170 |
|
| 1171 |
return size; |
| 1172 |
} |
| 1173 |
|
| 1174 |
/* |
| 1175 |
* Include all files from the given directory in the output tar stream. If |
| 1176 |
* 'sizeonly' is true, we just calculate a total length and return it, without |
| 1177 |
* actually sending anything. |
| 1178 |
* |
| 1179 |
* Omit any directory in the tablespaces list, to avoid backing up |
| 1180 |
* tablespaces twice when they were created inside PGDATA. |
| 1181 |
* |
| 1182 |
* If sendtblspclinks is true, we need to include symlink |
| 1183 |
* information in the tar file. If not, we can skip that |
| 1184 |
* as it will be sent separately in the tablespace_map file. |
| 1185 |
*/ |
| 1186 |
static int64 |
| 1187 |
sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, |
| 1188 |
List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, |
| 1189 |
Oid spcoid, IncrementalBackupInfo *ib) |
| 1190 |
{ |
| 1191 |
DIR *dir; |
| 1192 |
struct dirent *de; |
| 1193 |
char pathbuf[MAXPGPATH * 2]; |
| 1194 |
struct stat statbuf; |
| 1195 |
int64 size = 0; |
| 1196 |
const char *lastDir; /* Split last dir from parent path. */ |
| 1197 |
bool isRelationDir = false; /* Does directory contain relations? */ |
| 1198 |
bool isGlobalDir = false; |
| 1199 |
Oid dboid = InvalidOid; |
| 1200 |
BlockNumber *relative_block_numbers = NULL; |
| 1201 |
|
| 1202 |
/* |
| 1203 |
* Since this array is relatively large, avoid putting it on the stack. |
| 1204 |
* But we don't need it at all if this is not an incremental backup. |
| 1205 |
*/ |
| 1206 |
if (ib != NULL) |
| 1207 |
relative_block_numbers = palloc(sizeof(BlockNumber) * RELSEG_SIZE); |
| 1208 |
|
| 1209 |
/* |
| 1210 |
* Determine if the current path is a database directory that can contain |
| 1211 |
* relations. |
| 1212 |
* |
| 1213 |
* Start by finding the location of the delimiter between the parent path |
| 1214 |
* and the current path. |
| 1215 |
*/ |
| 1216 |
lastDir = last_dir_separator(path); |
| 1217 |
|
| 1218 |
/* Does this path look like a database path (i.e. all digits)? */ |
| 1219 |
if (lastDir != NULL && |
| 1220 |
strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1)) |
| 1221 |
{ |
| 1222 |
/* Part of path that contains the parent directory. */ |
| 1223 |
int parentPathLen = lastDir - path; |
| 1224 |
|
| 1225 |
/* |
| 1226 |
* Mark path as a database directory if the parent path is either |
| 1227 |
* $PGDATA/base or a tablespace version path. |
| 1228 |
*/ |
| 1229 |
if (strncmp(path, "./base", parentPathLen) == 0 || |
| 1230 |
(parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) && |
| 1231 |
strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1), |
| 1232 |
TABLESPACE_VERSION_DIRECTORY, |
| 1233 |
sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0)) |
| 1234 |
{ |
| 1235 |
isRelationDir = true; |
| 1236 |
dboid = atooid(lastDir + 1); |
| 1237 |
} |
| 1238 |
} |
| 1239 |
else if (strcmp(path, "./global") == 0) |
| 1240 |
{ |
| 1241 |
isRelationDir = true; |
| 1242 |
isGlobalDir = true; |
| 1243 |
} |
| 1244 |
|
| 1245 |
dir = AllocateDir(path); |
| 1246 |
while ((de = ReadDir(dir, path)) != NULL) |
| 1247 |
{ |
| 1248 |
int excludeIdx; |
| 1249 |
bool excludeFound; |
| 1250 |
RelFileNumber relfilenumber = InvalidRelFileNumber; |
| 1251 |
ForkNumber relForkNum = InvalidForkNumber; |
| 1252 |
unsigned segno = 0; |
| 1253 |
bool isRelationFile = false; |
| 1254 |
|
| 1255 |
/* Skip special stuff */ |
| 1256 |
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) |
| 1257 |
continue; |
| 1258 |
|
| 1259 |
/* Skip temporary files */ |
| 1260 |
if (strncmp(de->d_name, |
| 1261 |
PG_TEMP_FILE_PREFIX, |
| 1262 |
strlen(PG_TEMP_FILE_PREFIX)) == 0) |
| 1263 |
continue; |
| 1264 |
|
| 1265 |
/* Skip macOS system files */ |
| 1266 |
if (strcmp(de->d_name, ".DS_Store") == 0) |
| 1267 |
continue; |
| 1268 |
|
| 1269 |
/* |
| 1270 |
* Check if the postmaster has signaled us to exit, and abort with an |
| 1271 |
* error in that case. The error handler further up will call |
| 1272 |
* do_pg_abort_backup() for us. Also check that if the backup was |
| 1273 |
* started while still in recovery, the server wasn't promoted. |
| 1274 |
* do_pg_backup_stop() will check that too, but it's better to stop |
| 1275 |
* the backup early than continue to the end and fail there. |
| 1276 |
*/ |
| 1277 |
CHECK_FOR_INTERRUPTS(); |
| 1278 |
if (RecoveryInProgress() != backup_started_in_recovery) |
| 1279 |
ereport(ERROR, |
| 1280 |
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), |
| 1281 |
errmsg("the standby was promoted during online backup"), |
| 1282 |
errhint("This means that the backup being taken is corrupt " |
| 1283 |
"and should not be used. " |
| 1284 |
"Try taking another online backup."))); |
| 1285 |
|
| 1286 |
/* Scan for files that should be excluded */ |
| 1287 |
excludeFound = false; |
| 1288 |
for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++) |
| 1289 |
{ |
| 1290 |
int cmplen = strlen(excludeFiles[excludeIdx].name); |
| 1291 |
|
| 1292 |
if (!excludeFiles[excludeIdx].match_prefix) |
| 1293 |
cmplen++; |
| 1294 |
if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0) |
| 1295 |
{ |
| 1296 |
elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name); |
| 1297 |
excludeFound = true; |
| 1298 |
break; |
| 1299 |
} |
| 1300 |
} |
| 1301 |
|
| 1302 |
if (excludeFound) |
| 1303 |
continue; |
| 1304 |
|
| 1305 |
/* |
| 1306 |
* If there could be non-temporary relation files in this directory, |
| 1307 |
* try to parse the filename. |
| 1308 |
*/ |
| 1309 |
if (isRelationDir) |
| 1310 |
isRelationFile = |
| 1311 |
parse_filename_for_nontemp_relation(de->d_name, |
| 1312 |
&relfilenumber, |
| 1313 |
&relForkNum, &segno); |
| 1314 |
|
| 1315 |
/* Exclude all forks for unlogged tables except the init fork */ |
| 1316 |
if (isRelationFile && relForkNum != INIT_FORKNUM) |
| 1317 |
{ |
| 1318 |
char initForkFile[MAXPGPATH]; |
| 1319 |
|
| 1320 |
/* |
| 1321 |
* If any other type of fork, check if there is an init fork with |
| 1322 |
* the same RelFileNumber. If so, the file can be excluded. |
| 1323 |
*/ |
| 1324 |
snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init", |
| 1325 |
path, relfilenumber); |
| 1326 |
|
| 1327 |
if (lstat(initForkFile, &statbuf) == 0) |
| 1328 |
{ |
| 1329 |
elog(DEBUG2, |
| 1330 |
"unlogged relation file \"%s\" excluded from backup", |
| 1331 |
de->d_name); |
| 1332 |
|
| 1333 |
continue; |
| 1334 |
} |
| 1335 |
} |
| 1336 |
|
| 1337 |
/* Exclude temporary relations */ |
| 1338 |
if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name)) |
| 1339 |
{ |
| 1340 |
elog(DEBUG2, |
| 1341 |
"temporary relation file \"%s\" excluded from backup", |
| 1342 |
de->d_name); |
| 1343 |
|
| 1344 |
continue; |
| 1345 |
} |
| 1346 |
|
| 1347 |
snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name); |
| 1348 |
|
| 1349 |
/* Skip pg_control here to back up it last */ |
| 1350 |
if (strcmp(pathbuf, "./" XLOG_CONTROL_FILE) == 0) |
| 1351 |
continue; |
| 1352 |
|
| 1353 |
if (lstat(pathbuf, &statbuf) != 0) |
| 1354 |
{ |
| 1355 |
if (errno != ENOENT) |
| 1356 |
ereport(ERROR, |
| 1357 |
(errcode_for_file_access(), |
| 1358 |
errmsg("could not stat file or directory \"%s\": %m", |
| 1359 |
pathbuf))); |
| 1360 |
|
| 1361 |
/* If the file went away while scanning, it's not an error. */ |
| 1362 |
continue; |
| 1363 |
} |
| 1364 |
|
| 1365 |
/* Scan for directories whose contents should be excluded */ |
| 1366 |
excludeFound = false; |
| 1367 |
for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++) |
| 1368 |
{ |
| 1369 |
if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0) |
| 1370 |
{ |
| 1371 |
elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name); |
| 1372 |
convert_link_to_directory(pathbuf, &statbuf); |
| 1373 |
size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, |
| 1374 |
&statbuf, sizeonly); |
| 1375 |
excludeFound = true; |
| 1376 |
break; |
| 1377 |
} |
| 1378 |
} |
| 1379 |
|
| 1380 |
if (excludeFound) |
| 1381 |
continue; |
| 1382 |
|
| 1383 |
/* |
| 1384 |
* We can skip pg_wal, the WAL segments need to be fetched from the |
| 1385 |
* WAL archive anyway. But include it as an empty directory anyway, so |
| 1386 |
* we get permissions right. |
| 1387 |
*/ |
| 1388 |
if (strcmp(pathbuf, "./pg_wal") == 0) |
| 1389 |
{ |
| 1390 |
/* If pg_wal is a symlink, write it as a directory anyway */ |
| 1391 |
convert_link_to_directory(pathbuf, &statbuf); |
| 1392 |
size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, |
| 1393 |
&statbuf, sizeonly); |
| 1394 |
|
| 1395 |
/* |
| 1396 |
* Also send archive_status and summaries directories (by |
| 1397 |
* hackishly reusing statbuf from above ...). |
| 1398 |
*/ |
| 1399 |
size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL, |
| 1400 |
&statbuf, sizeonly); |
| 1401 |
size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL, |
| 1402 |
&statbuf, sizeonly); |
| 1403 |
|
| 1404 |
continue; /* don't recurse into pg_wal */ |
| 1405 |
} |
| 1406 |
|
| 1407 |
/* Allow symbolic links in pg_tblspc only */ |
| 1408 |
if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode)) |
| 1409 |
{ |
| 1410 |
char linkpath[MAXPGPATH]; |
| 1411 |
int rllen; |
| 1412 |
|
| 1413 |
rllen = readlink(pathbuf, linkpath, sizeof(linkpath)); |
| 1414 |
if (rllen < 0) |
| 1415 |
ereport(ERROR, |
| 1416 |
(errcode_for_file_access(), |
| 1417 |
errmsg("could not read symbolic link \"%s\": %m", |
| 1418 |
pathbuf))); |
| 1419 |
if (rllen >= sizeof(linkpath)) |
| 1420 |
ereport(ERROR, |
| 1421 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 1422 |
errmsg("symbolic link \"%s\" target is too long", |
| 1423 |
pathbuf))); |
| 1424 |
linkpath[rllen] = '\0'; |
| 1425 |
|
| 1426 |
size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath, |
| 1427 |
&statbuf, sizeonly); |
| 1428 |
} |
| 1429 |
else if (S_ISDIR(statbuf.st_mode)) |
| 1430 |
{ |
| 1431 |
bool skip_this_dir = false; |
| 1432 |
ListCell *lc; |
| 1433 |
|
| 1434 |
/* |
| 1435 |
* Store a directory entry in the tar file so we can get the |
| 1436 |
* permissions right. |
| 1437 |
*/ |
| 1438 |
size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf, |
| 1439 |
sizeonly); |
| 1440 |
|
| 1441 |
/* |
| 1442 |
* Call ourselves recursively for a directory, unless it happens |
| 1443 |
* to be a separate tablespace located within PGDATA. |
| 1444 |
*/ |
| 1445 |
foreach(lc, tablespaces) |
| 1446 |
{ |
| 1447 |
tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc); |
| 1448 |
|
| 1449 |
/* |
| 1450 |
* ti->rpath is the tablespace relative path within PGDATA, or |
| 1451 |
* NULL if the tablespace has been properly located somewhere |
| 1452 |
* else. |
| 1453 |
* |
| 1454 |
* Skip past the leading "./" in pathbuf when comparing. |
| 1455 |
*/ |
| 1456 |
if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0) |
| 1457 |
{ |
| 1458 |
skip_this_dir = true; |
| 1459 |
break; |
| 1460 |
} |
| 1461 |
} |
| 1462 |
|
| 1463 |
/* |
| 1464 |
* skip sending directories inside pg_tblspc, if not required. |
| 1465 |
*/ |
| 1466 |
if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks) |
| 1467 |
skip_this_dir = true; |
| 1468 |
|
| 1469 |
if (!skip_this_dir) |
| 1470 |
size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces, |
| 1471 |
sendtblspclinks, manifest, spcoid, ib); |
| 1472 |
} |
| 1473 |
else if (S_ISREG(statbuf.st_mode)) |
| 1474 |
{ |
| 1475 |
bool sent = false; |
| 1476 |
unsigned num_blocks_required = 0; |
| 1477 |
unsigned truncation_block_length = 0; |
| 1478 |
char tarfilenamebuf[MAXPGPATH * 2]; |
| 1479 |
char *tarfilename = pathbuf + basepathlen + 1; |
| 1480 |
FileBackupMethod method = BACK_UP_FILE_FULLY; |
| 1481 |
|
| 1482 |
if (ib != NULL && isRelationFile) |
| 1483 |
{ |
| 1484 |
Oid relspcoid; |
| 1485 |
char *lookup_path; |
| 1486 |
|
| 1487 |
if (OidIsValid(spcoid)) |
| 1488 |
{ |
| 1489 |
relspcoid = spcoid; |
| 1490 |
lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid, |
| 1491 |
tarfilename); |
| 1492 |
} |
| 1493 |
else |
| 1494 |
{ |
| 1495 |
if (isGlobalDir) |
| 1496 |
relspcoid = GLOBALTABLESPACE_OID; |
| 1497 |
else |
| 1498 |
relspcoid = DEFAULTTABLESPACE_OID; |
| 1499 |
lookup_path = pstrdup(tarfilename); |
| 1500 |
} |
| 1501 |
|
| 1502 |
method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid, |
| 1503 |
relfilenumber, relForkNum, |
| 1504 |
segno, statbuf.st_size, |
| 1505 |
&num_blocks_required, |
| 1506 |
relative_block_numbers, |
| 1507 |
&truncation_block_length); |
| 1508 |
if (method == BACK_UP_FILE_INCREMENTALLY) |
| 1509 |
{ |
| 1510 |
statbuf.st_size = |
| 1511 |
GetIncrementalFileSize(num_blocks_required); |
| 1512 |
snprintf(tarfilenamebuf, sizeof(tarfilenamebuf), |
| 1513 |
"%s/INCREMENTAL.%s", |
| 1514 |
path + basepathlen + 1, |
| 1515 |
de->d_name); |
| 1516 |
tarfilename = tarfilenamebuf; |
| 1517 |
} |
| 1518 |
|
| 1519 |
pfree(lookup_path); |
| 1520 |
} |
| 1521 |
|
| 1522 |
if (!sizeonly) |
| 1523 |
sent = sendFile(sink, pathbuf, tarfilename, &statbuf, |
| 1524 |
true, dboid, spcoid, |
| 1525 |
relfilenumber, segno, manifest, |
| 1526 |
num_blocks_required, |
| 1527 |
method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL, |
| 1528 |
truncation_block_length); |
| 1529 |
|
| 1530 |
if (sent || sizeonly) |
| 1531 |
{ |
| 1532 |
/* Add size. */ |
| 1533 |
size += statbuf.st_size; |
| 1534 |
|
| 1535 |
/* Pad to a multiple of the tar block size. */ |
| 1536 |
size += tarPaddingBytesRequired(statbuf.st_size); |
| 1537 |
|
| 1538 |
/* Size of the header for the file. */ |
| 1539 |
size += TAR_BLOCK_SIZE; |
| 1540 |
} |
| 1541 |
} |
| 1542 |
else |
| 1543 |
ereport(WARNING, |
| 1544 |
(errmsg("skipping special file \"%s\"", pathbuf))); |
| 1545 |
} |
| 1546 |
|
| 1547 |
if (relative_block_numbers != NULL) |
| 1548 |
pfree(relative_block_numbers); |
| 1549 |
|
| 1550 |
FreeDir(dir); |
| 1551 |
return size; |
| 1552 |
} |
| 1553 |
|
| 1554 |
/* |
| 1555 |
* Given the member, write the TAR header & send the file. |
| 1556 |
* |
| 1557 |
* If 'missing_ok' is true, will not throw an error if the file is not found. |
| 1558 |
* |
| 1559 |
* If dboid is anything other than InvalidOid then any checksum failures |
| 1560 |
* detected will get reported to the cumulative stats system. |
| 1561 |
* |
| 1562 |
* If the file is to be sent incrementally, then num_incremental_blocks |
| 1563 |
* should be the number of blocks to be sent, and incremental_blocks |
| 1564 |
* an array of block numbers relative to the start of the current segment. |
| 1565 |
* If the whole file is to be sent, then incremental_blocks should be NULL, |
| 1566 |
* and num_incremental_blocks can have any value, as it will be ignored. |
| 1567 |
* |
| 1568 |
* Returns true if the file was successfully sent, false if 'missing_ok', |
| 1569 |
* and the file did not exist. |
| 1570 |
*/ |
| 1571 |
static bool |
| 1572 |
sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, |
| 1573 |
struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid, |
| 1574 |
RelFileNumber relfilenumber, unsigned segno, |
| 1575 |
backup_manifest_info *manifest, unsigned num_incremental_blocks, |
| 1576 |
BlockNumber *incremental_blocks, unsigned truncation_block_length) |
| 1577 |
{ |
| 1578 |
int fd; |
| 1579 |
BlockNumber blkno = 0; |
| 1580 |
int checksum_failures = 0; |
| 1581 |
off_t cnt; |
| 1582 |
pgoff_t bytes_done = 0; |
| 1583 |
bool verify_checksum = false; |
| 1584 |
pg_checksum_context checksum_ctx; |
| 1585 |
int ibindex = 0; |
| 1586 |
|
| 1587 |
if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0) |
| 1588 |
elog(ERROR, "could not initialize checksum of file \"%s\"", |
| 1589 |
readfilename); |
| 1590 |
|
| 1591 |
fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY); |
| 1592 |
if (fd < 0) |
| 1593 |
{ |
| 1594 |
if (errno == ENOENT && missing_ok) |
| 1595 |
return false; |
| 1596 |
ereport(ERROR, |
| 1597 |
(errcode_for_file_access(), |
| 1598 |
errmsg("could not open file \"%s\": %m", readfilename))); |
| 1599 |
} |
| 1600 |
|
| 1601 |
_tarWriteHeader(sink, tarfilename, NULL, statbuf, false); |
| 1602 |
|
| 1603 |
/* |
| 1604 |
* Checksums are verified in multiples of BLCKSZ, so the buffer length |
| 1605 |
* should be a multiple of the block size as well. |
| 1606 |
*/ |
| 1607 |
Assert((sink->bbs_buffer_length % BLCKSZ) == 0); |
| 1608 |
|
| 1609 |
/* |
| 1610 |
* If we weren't told not to verify checksums, and if checksums are |
| 1611 |
* enabled for this cluster, and if this is a relation file, then verify |
| 1612 |
* the checksum. |
| 1613 |
*/ |
| 1614 |
if (!noverify_checksums && DataChecksumsEnabled() && |
| 1615 |
RelFileNumberIsValid(relfilenumber)) |
| 1616 |
verify_checksum = true; |
| 1617 |
|
| 1618 |
/* |
| 1619 |
* If we're sending an incremental file, write the file header. |
| 1620 |
*/ |
| 1621 |
if (incremental_blocks != NULL) |
| 1622 |
{ |
| 1623 |
unsigned magic = INCREMENTAL_MAGIC; |
| 1624 |
size_t header_bytes_done = 0; |
| 1625 |
char padding[BLCKSZ]; |
| 1626 |
size_t paddinglen; |
| 1627 |
|
| 1628 |
/* Emit header data. */ |
| 1629 |
push_to_sink(sink, &checksum_ctx, &header_bytes_done, |
| 1630 |
&magic, sizeof(magic)); |
| 1631 |
push_to_sink(sink, &checksum_ctx, &header_bytes_done, |
| 1632 |
&num_incremental_blocks, sizeof(num_incremental_blocks)); |
| 1633 |
push_to_sink(sink, &checksum_ctx, &header_bytes_done, |
| 1634 |
&truncation_block_length, sizeof(truncation_block_length)); |
| 1635 |
push_to_sink(sink, &checksum_ctx, &header_bytes_done, |
| 1636 |
incremental_blocks, |
| 1637 |
sizeof(BlockNumber) * num_incremental_blocks); |
| 1638 |
|
| 1639 |
/* |
| 1640 |
* Add padding to align header to a multiple of BLCKSZ, but only if |
| 1641 |
* the incremental file has some blocks, and the alignment is actually |
| 1642 |
* needed (i.e. header is not already a multiple of BLCKSZ). If there |
| 1643 |
* are no blocks we don't want to make the file unnecessarily large, |
| 1644 |
* as that might make some filesystem optimizations impossible. |
| 1645 |
*/ |
| 1646 |
if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0)) |
| 1647 |
{ |
| 1648 |
paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ)); |
| 1649 |
|
| 1650 |
memset(padding, 0, paddinglen); |
| 1651 |
bytes_done += paddinglen; |
| 1652 |
|
| 1653 |
push_to_sink(sink, &checksum_ctx, &header_bytes_done, |
| 1654 |
padding, paddinglen); |
| 1655 |
} |
| 1656 |
|
| 1657 |
/* Flush out any data still in the buffer so it's again empty. */ |
| 1658 |
if (header_bytes_done > 0) |
| 1659 |
{ |
| 1660 |
bbsink_archive_contents(sink, header_bytes_done); |
| 1661 |
if (pg_checksum_update(&checksum_ctx, |
| 1662 |
(uint8 *) sink->bbs_buffer, |
| 1663 |
header_bytes_done) < 0) |
| 1664 |
elog(ERROR, "could not update checksum of base backup"); |
| 1665 |
} |
| 1666 |
|
| 1667 |
/* Update our notion of file position. */ |
| 1668 |
bytes_done += sizeof(magic); |
| 1669 |
bytes_done += sizeof(num_incremental_blocks); |
| 1670 |
bytes_done += sizeof(truncation_block_length); |
| 1671 |
bytes_done += sizeof(BlockNumber) * num_incremental_blocks; |
| 1672 |
} |
| 1673 |
|
| 1674 |
/* |
| 1675 |
* Loop until we read the amount of data the caller told us to expect. The |
| 1676 |
* file could be longer, if it was extended while we were sending it, but |
| 1677 |
* for a base backup we can ignore such extended data. It will be restored |
| 1678 |
* from WAL. |
| 1679 |
*/ |
| 1680 |
while (1) |
| 1681 |
{ |
| 1682 |
/* |
| 1683 |
* Determine whether we've read all the data that we need, and if not, |
| 1684 |
* read some more. |
| 1685 |
*/ |
| 1686 |
if (incremental_blocks == NULL) |
| 1687 |
{ |
| 1688 |
size_t remaining = statbuf->st_size - bytes_done; |
| 1689 |
|
| 1690 |
/* |
| 1691 |
* If we've read the required number of bytes, then it's time to |
| 1692 |
* stop. |
| 1693 |
*/ |
| 1694 |
if (bytes_done >= statbuf->st_size) |
| 1695 |
break; |
| 1696 |
|
| 1697 |
/* |
| 1698 |
* Read as many bytes as will fit in the buffer, or however many |
| 1699 |
* are left to read, whichever is less. |
| 1700 |
*/ |
| 1701 |
cnt = read_file_data_into_buffer(sink, readfilename, fd, |
| 1702 |
bytes_done, remaining, |
| 1703 |
blkno + segno * RELSEG_SIZE, |
| 1704 |
verify_checksum, |
| 1705 |
&checksum_failures); |
| 1706 |
} |
| 1707 |
else |
| 1708 |
{ |
| 1709 |
BlockNumber relative_blkno; |
| 1710 |
|
| 1711 |
/* |
| 1712 |
* If we've read all the blocks, then it's time to stop. |
| 1713 |
*/ |
| 1714 |
if (ibindex >= num_incremental_blocks) |
| 1715 |
break; |
| 1716 |
|
| 1717 |
/* |
| 1718 |
* Read just one block, whichever one is the next that we're |
| 1719 |
* supposed to include. |
| 1720 |
*/ |
| 1721 |
relative_blkno = incremental_blocks[ibindex++]; |
| 1722 |
cnt = read_file_data_into_buffer(sink, readfilename, fd, |
| 1723 |
relative_blkno * BLCKSZ, |
| 1724 |
BLCKSZ, |
| 1725 |
relative_blkno + segno * RELSEG_SIZE, |
| 1726 |
verify_checksum, |
| 1727 |
&checksum_failures); |
| 1728 |
|
| 1729 |
/* |
| 1730 |
* If we get a partial read, that must mean that the relation is |
| 1731 |
* being truncated. Ultimately, it should be truncated to a |
| 1732 |
* multiple of BLCKSZ, since this path should only be reached for |
| 1733 |
* relation files, but we might transiently observe an |
| 1734 |
* intermediate value. |
| 1735 |
* |
| 1736 |
* It should be fine to treat this just as if the entire block had |
| 1737 |
* been truncated away - i.e. fill this and all later blocks with |
| 1738 |
* zeroes. WAL replay will fix things up. |
| 1739 |
*/ |
| 1740 |
if (cnt < BLCKSZ) |
| 1741 |
break; |
| 1742 |
} |
| 1743 |
|
| 1744 |
/* |
| 1745 |
* If the amount of data we were able to read was not a multiple of |
| 1746 |
* BLCKSZ, we cannot verify checksums, which are block-level. |
| 1747 |
*/ |
| 1748 |
if (verify_checksum && (cnt % BLCKSZ != 0)) |
| 1749 |
{ |
| 1750 |
ereport(WARNING, |
| 1751 |
(errmsg("could not verify checksum in file \"%s\", block " |
| 1752 |
"%u: read buffer size %d and page size %d " |
| 1753 |
"differ", |
| 1754 |
readfilename, blkno, (int) cnt, BLCKSZ))); |
| 1755 |
verify_checksum = false; |
| 1756 |
} |
| 1757 |
|
| 1758 |
/* |
| 1759 |
* If we hit end-of-file, a concurrent truncation must have occurred. |
| 1760 |
* That's not an error condition, because WAL replay will fix things |
| 1761 |
* up. |
| 1762 |
*/ |
| 1763 |
if (cnt == 0) |
| 1764 |
break; |
| 1765 |
|
| 1766 |
/* Update block number and # of bytes done for next loop iteration. */ |
| 1767 |
blkno += cnt / BLCKSZ; |
| 1768 |
bytes_done += cnt; |
| 1769 |
|
| 1770 |
/* |
| 1771 |
* Make sure incremental files with block data are properly aligned |
| 1772 |
* (header is a multiple of BLCKSZ, blocks are BLCKSZ too). |
| 1773 |
*/ |
| 1774 |
Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) && |
| 1775 |
(bytes_done % BLCKSZ != 0))); |
| 1776 |
|
| 1777 |
/* Archive the data we just read. */ |
| 1778 |
bbsink_archive_contents(sink, cnt); |
| 1779 |
|
| 1780 |
/* Also feed it to the checksum machinery. */ |
| 1781 |
if (pg_checksum_update(&checksum_ctx, |
| 1782 |
(uint8 *) sink->bbs_buffer, cnt) < 0) |
| 1783 |
elog(ERROR, "could not update checksum of base backup"); |
| 1784 |
} |
| 1785 |
|
| 1786 |
/* If the file was truncated while we were sending it, pad it with zeros */ |
| 1787 |
while (bytes_done < statbuf->st_size) |
| 1788 |
{ |
| 1789 |
size_t remaining = statbuf->st_size - bytes_done; |
| 1790 |
size_t nbytes = Min(sink->bbs_buffer_length, remaining); |
| 1791 |
|
| 1792 |
MemSet(sink->bbs_buffer, 0, nbytes); |
| 1793 |
if (pg_checksum_update(&checksum_ctx, |
| 1794 |
(uint8 *) sink->bbs_buffer, |
| 1795 |
nbytes) < 0) |
| 1796 |
elog(ERROR, "could not update checksum of base backup"); |
| 1797 |
bbsink_archive_contents(sink, nbytes); |
| 1798 |
bytes_done += nbytes; |
| 1799 |
} |
| 1800 |
|
| 1801 |
/* |
| 1802 |
* Pad to a block boundary, per tar format requirements. (This small piece |
| 1803 |
* of data is probably not worth throttling, and is not checksummed |
| 1804 |
* because it's not actually part of the file.) |
| 1805 |
*/ |
| 1806 |
_tarWritePadding(sink, bytes_done); |
| 1807 |
|
| 1808 |
CloseTransientFile(fd); |
| 1809 |
|
| 1810 |
if (checksum_failures > 1) |
| 1811 |
{ |
| 1812 |
ereport(WARNING, |
| 1813 |
(errmsg_plural("file \"%s\" has a total of %d checksum verification failure", |
| 1814 |
"file \"%s\" has a total of %d checksum verification failures", |
| 1815 |
checksum_failures, |
| 1816 |
readfilename, checksum_failures))); |
| 1817 |
|
| 1818 |
pgstat_prepare_report_checksum_failure(dboid); |
| 1819 |
pgstat_report_checksum_failures_in_db(dboid, checksum_failures); |
| 1820 |
} |
| 1821 |
|
| 1822 |
total_checksum_failures += checksum_failures; |
| 1823 |
|
| 1824 |
AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size, |
| 1825 |
(pg_time_t) statbuf->st_mtime, &checksum_ctx); |
| 1826 |
|
| 1827 |
return true; |
| 1828 |
} |
| 1829 |
|
| 1830 |
/* |
| 1831 |
* Read some more data from the file into the bbsink's buffer, verifying |
| 1832 |
* checksums as required. |
| 1833 |
* |
| 1834 |
* 'offset' is the file offset from which we should begin to read, and |
| 1835 |
* 'length' is the amount of data that should be read. The actual amount |
| 1836 |
* of data read will be less than the requested amount if the bbsink's |
| 1837 |
* buffer isn't big enough to hold it all, or if the underlying file has |
| 1838 |
* been truncated. The return value is the number of bytes actually read. |
| 1839 |
* |
| 1840 |
* 'blkno' is the block number of the first page in the bbsink's buffer |
| 1841 |
* relative to the start of the relation. |
| 1842 |
* |
| 1843 |
* 'verify_checksum' indicates whether we should try to verify checksums |
| 1844 |
* for the blocks we read. If we do this, we'll update *checksum_failures |
| 1845 |
* and issue warnings as appropriate. |
| 1846 |
*/ |
| 1847 |
static off_t |
| 1848 |
read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, |
| 1849 |
off_t offset, size_t length, BlockNumber blkno, |
| 1850 |
bool verify_checksum, int *checksum_failures) |
| 1851 |
{ |
| 1852 |
off_t cnt; |
| 1853 |
int i; |
| 1854 |
char *page; |
| 1855 |
|
| 1856 |
/* Try to read some more data. */ |
| 1857 |
cnt = basebackup_read_file(fd, sink->bbs_buffer, |
| 1858 |
Min(sink->bbs_buffer_length, length), |
| 1859 |
offset, readfilename, true); |
| 1860 |
|
| 1861 |
/* Can't verify checksums if read length is not a multiple of BLCKSZ. */ |
| 1862 |
if (!verify_checksum || (cnt % BLCKSZ) != 0) |
| 1863 |
return cnt; |
| 1864 |
|
| 1865 |
/* Verify checksum for each block. */ |
| 1866 |
for (i = 0; i < cnt / BLCKSZ; i++) |
| 1867 |
{ |
| 1868 |
int reread_cnt; |
| 1869 |
uint16 expected_checksum; |
| 1870 |
|
| 1871 |
page = sink->bbs_buffer + BLCKSZ * i; |
| 1872 |
|
| 1873 |
/* If the page is OK, go on to the next one. */ |
| 1874 |
if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i, |
| 1875 |
&expected_checksum)) |
| 1876 |
continue; |
| 1877 |
|
| 1878 |
/* |
| 1879 |
* Retry the block on the first failure. It's possible that we read |
| 1880 |
* the first 4K page of the block just before postgres updated the |
| 1881 |
* entire block so it ends up looking torn to us. If, before we retry |
| 1882 |
* the read, the concurrent write of the block finishes, the page LSN |
| 1883 |
* will be updated and we'll realize that we should ignore this block. |
| 1884 |
* |
| 1885 |
* There's no guarantee that this will actually happen, though: the |
| 1886 |
* torn write could take an arbitrarily long time to complete. |
| 1887 |
* Retrying multiple times wouldn't fix this problem, either, though |
| 1888 |
* it would reduce the chances of it happening in practice. The only |
| 1889 |
* real fix here seems to be to have some kind of interlock that |
| 1890 |
* allows us to wait until we can be certain that no write to the |
| 1891 |
* block is in progress. Since we don't have any such thing right now, |
| 1892 |
* we just do this and hope for the best. |
| 1893 |
*/ |
| 1894 |
reread_cnt = |
| 1895 |
basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i, |
| 1896 |
BLCKSZ, offset + BLCKSZ * i, |
| 1897 |
readfilename, false); |
| 1898 |
if (reread_cnt == 0) |
| 1899 |
{ |
| 1900 |
/* |
| 1901 |
* If we hit end-of-file, a concurrent truncation must have |
| 1902 |
* occurred, so reduce cnt to reflect only the blocks already |
| 1903 |
* processed and break out of this loop. |
| 1904 |
*/ |
| 1905 |
cnt = BLCKSZ * i; |
| 1906 |
break; |
| 1907 |
} |
| 1908 |
|
| 1909 |
/* If the page now looks OK, go on to the next one. */ |
| 1910 |
if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i, |
| 1911 |
&expected_checksum)) |
| 1912 |
continue; |
| 1913 |
|
| 1914 |
/* Handle checksum failure. */ |
| 1915 |
(*checksum_failures)++; |
| 1916 |
if (*checksum_failures <= 5) |
| 1917 |
ereport(WARNING, |
| 1918 |
(errmsg("checksum verification failed in " |
| 1919 |
"file \"%s\", block %u: calculated " |
| 1920 |
"%X but expected %X", |
| 1921 |
readfilename, blkno + i, expected_checksum, |
| 1922 |
((PageHeader) page)->pd_checksum))); |
| 1923 |
if (*checksum_failures == 5) |
| 1924 |
ereport(WARNING, |
| 1925 |
(errmsg("further checksum verification " |
| 1926 |
"failures in file \"%s\" will not " |
| 1927 |
"be reported", readfilename))); |
| 1928 |
} |
| 1929 |
|
| 1930 |
return cnt; |
| 1931 |
} |
| 1932 |
|
| 1933 |
/* |
| 1934 |
* Push data into a bbsink. |
| 1935 |
* |
| 1936 |
* It's better, when possible, to read data directly into the bbsink's buffer, |
| 1937 |
* rather than using this function to copy it into the buffer; this function is |
| 1938 |
* for cases where that approach is not practical. |
| 1939 |
* |
| 1940 |
* bytes_done should point to a count of the number of bytes that are |
| 1941 |
* currently used in the bbsink's buffer. Upon return, the bytes identified by |
| 1942 |
* data and length will have been copied into the bbsink's buffer, flushing |
| 1943 |
* as required, and *bytes_done will have been updated accordingly. If the |
| 1944 |
* buffer was flushed, the previous contents will also have been fed to |
| 1945 |
* checksum_ctx. |
| 1946 |
* |
| 1947 |
* Note that after one or more calls to this function it is the caller's |
| 1948 |
* responsibility to perform any required final flush. |
| 1949 |
*/ |
| 1950 |
static void |
| 1951 |
push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, |
| 1952 |
size_t *bytes_done, void *data, size_t length) |
| 1953 |
{ |
| 1954 |
while (length > 0) |
| 1955 |
{ |
| 1956 |
size_t bytes_to_copy; |
| 1957 |
|
| 1958 |
/* |
| 1959 |
* We use < here rather than <= so that if the data exactly fills the |
| 1960 |
* remaining buffer space, we trigger a flush now. |
| 1961 |
*/ |
| 1962 |
if (length < sink->bbs_buffer_length - *bytes_done) |
| 1963 |
{ |
| 1964 |
/* Append remaining data to buffer. */ |
| 1965 |
memcpy(sink->bbs_buffer + *bytes_done, data, length); |
| 1966 |
*bytes_done += length; |
| 1967 |
return; |
| 1968 |
} |
| 1969 |
|
| 1970 |
/* Copy until buffer is full and flush it. */ |
| 1971 |
bytes_to_copy = sink->bbs_buffer_length - *bytes_done; |
| 1972 |
memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy); |
| 1973 |
data = ((char *) data) + bytes_to_copy; |
| 1974 |
length -= bytes_to_copy; |
| 1975 |
bbsink_archive_contents(sink, sink->bbs_buffer_length); |
| 1976 |
if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer, |
| 1977 |
sink->bbs_buffer_length) < 0) |
| 1978 |
elog(ERROR, "could not update checksum"); |
| 1979 |
*bytes_done = 0; |
| 1980 |
} |
| 1981 |
} |
| 1982 |
|
| 1983 |
/* |
| 1984 |
* Try to verify the checksum for the provided page, if it seems appropriate |
| 1985 |
* to do so. |
| 1986 |
* |
| 1987 |
* Returns true if verification succeeds or if we decide not to check it, |
| 1988 |
* and false if verification fails. When return false, it also sets |
| 1989 |
* *expected_checksum to the computed value. |
| 1990 |
*/ |
| 1991 |
static bool |
| 1992 |
verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, |
| 1993 |
uint16 *expected_checksum) |
| 1994 |
{ |
| 1995 |
PageHeader phdr; |
| 1996 |
uint16 checksum; |
| 1997 |
|
| 1998 |
/* |
| 1999 |
* Only check pages which have not been modified since the start of the |
| 2000 |
* base backup. Otherwise, they might have been written only halfway and |
| 2001 |
* the checksum would not be valid. However, replaying WAL would |
| 2002 |
* reinstate the correct page in this case. We also skip completely new |
| 2003 |
* pages, since they don't have a checksum yet. |
| 2004 |
*/ |
| 2005 |
if (PageIsNew(page) || PageGetLSN(page) >= start_lsn) |
| 2006 |
return true; |
| 2007 |
|
| 2008 |
/* Perform the actual checksum calculation. */ |
| 2009 |
checksum = pg_checksum_page(page, blkno); |
| 2010 |
|
| 2011 |
/* See whether it matches the value from the page. */ |
| 2012 |
phdr = (PageHeader) page; |
| 2013 |
if (phdr->pd_checksum == checksum) |
| 2014 |
return true; |
| 2015 |
*expected_checksum = checksum; |
| 2016 |
return false; |
| 2017 |
} |
| 2018 |
|
| 2019 |
static int64 |
| 2020 |
_tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, |
| 2021 |
struct stat *statbuf, bool sizeonly) |
| 2022 |
{ |
| 2023 |
enum tarError rc; |
| 2024 |
|
| 2025 |
if (!sizeonly) |
| 2026 |
{ |
| 2027 |
/* |
| 2028 |
* As of this writing, the smallest supported block size is 1kB, which |
| 2029 |
* is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a |
| 2030 |
* multiple of BLCKSZ, it should be safe to assume that the buffer is |
| 2031 |
* large enough to fit an entire tar block. We double-check by means |
| 2032 |
* of these assertions. |
| 2033 |
*/ |
| 2034 |
StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ, |
| 2035 |
"BLCKSZ too small for tar block"); |
| 2036 |
Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE); |
| 2037 |
|
| 2038 |
rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget, |
| 2039 |
statbuf->st_size, statbuf->st_mode, |
| 2040 |
statbuf->st_uid, statbuf->st_gid, |
| 2041 |
statbuf->st_mtime); |
| 2042 |
|
| 2043 |
switch (rc) |
| 2044 |
{ |
| 2045 |
case TAR_OK: |
| 2046 |
break; |
| 2047 |
case TAR_NAME_TOO_LONG: |
| 2048 |
ereport(ERROR, |
| 2049 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 2050 |
errmsg("file name too long for tar format: \"%s\"", |
| 2051 |
filename))); |
| 2052 |
break; |
| 2053 |
case TAR_SYMLINK_TOO_LONG: |
| 2054 |
ereport(ERROR, |
| 2055 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 2056 |
errmsg("symbolic link target too long for tar format: " |
| 2057 |
"file name \"%s\", target \"%s\"", |
| 2058 |
filename, linktarget))); |
| 2059 |
break; |
| 2060 |
default: |
| 2061 |
elog(ERROR, "unrecognized tar error: %d", rc); |
| 2062 |
} |
| 2063 |
|
| 2064 |
bbsink_archive_contents(sink, TAR_BLOCK_SIZE); |
| 2065 |
} |
| 2066 |
|
| 2067 |
return TAR_BLOCK_SIZE; |
| 2068 |
} |
| 2069 |
|
| 2070 |
/* |
| 2071 |
* Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE. |
| 2072 |
*/ |
| 2073 |
static void |
| 2074 |
_tarWritePadding(bbsink *sink, int len) |
| 2075 |
{ |
| 2076 |
int pad = tarPaddingBytesRequired(len); |
| 2077 |
|
| 2078 |
/* |
| 2079 |
* As in _tarWriteHeader, it should be safe to assume that the buffer is |
| 2080 |
* large enough that we don't need to do this in multiple chunks. |
| 2081 |
*/ |
| 2082 |
Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE); |
| 2083 |
Assert(pad <= TAR_BLOCK_SIZE); |
| 2084 |
|
| 2085 |
if (pad > 0) |
| 2086 |
{ |
| 2087 |
MemSet(sink->bbs_buffer, 0, pad); |
| 2088 |
bbsink_archive_contents(sink, pad); |
| 2089 |
} |
| 2090 |
} |
| 2091 |
|
| 2092 |
/* |
| 2093 |
* If the entry in statbuf is a link, then adjust statbuf to make it look like a |
| 2094 |
* directory, so that it will be written that way. |
| 2095 |
*/ |
| 2096 |
static void |
| 2097 |
convert_link_to_directory(const char *pathbuf, struct stat *statbuf) |
| 2098 |
{ |
| 2099 |
/* If symlink, write it as a directory anyway */ |
| 2100 |
if (S_ISLNK(statbuf->st_mode)) |
| 2101 |
statbuf->st_mode = S_IFDIR | pg_dir_create_mode; |
| 2102 |
} |
| 2103 |
|
| 2104 |
/* |
| 2105 |
* Read some data from a file, setting a wait event and reporting any error |
| 2106 |
* encountered. |
| 2107 |
* |
| 2108 |
* If partial_read_ok is false, also report an error if the number of bytes |
| 2109 |
* read is not equal to the number of bytes requested. |
| 2110 |
* |
| 2111 |
* Returns the number of bytes read. |
| 2112 |
*/ |
| 2113 |
static ssize_t |
| 2114 |
basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, |
| 2115 |
const char *filename, bool partial_read_ok) |
| 2116 |
{ |
| 2117 |
ssize_t rc; |
| 2118 |
|
| 2119 |
pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ); |
| 2120 |
rc = pg_pread(fd, buf, nbytes, offset); |
| 2121 |
pgstat_report_wait_end(); |
| 2122 |
|
| 2123 |
if (rc < 0) |
| 2124 |
ereport(ERROR, |
| 2125 |
(errcode_for_file_access(), |
| 2126 |
errmsg("could not read file \"%s\": %m", filename))); |
| 2127 |
if (!partial_read_ok && rc > 0 && rc != nbytes) |
| 2128 |
ereport(ERROR, |
| 2129 |
(errcode_for_file_access(), |
| 2130 |
errmsg("could not read file \"%s\": read %zd of %zu", |
| 2131 |
filename, rc, nbytes))); |
| 2132 |
|
| 2133 |
return rc; |
| 2134 |
} |