Debian Codegraph @

src/backend/access/transam/xlogrecovery.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * xlogrecovery.c
4 * Functions for WAL recovery, standby mode
5 *
6 * This source file contains functions controlling WAL recovery.
7 * InitWalRecovery() initializes the system for crash or archive recovery,
8 * or standby mode, depending on configuration options and the state of
9 * the control file and possible backup label file. PerformWalRecovery()
10 * performs the actual WAL replay, calling the rmgr-specific redo routines.
11 * FinishWalRecovery() performs end-of-recovery checks and cleanup actions,
12 * and prepares information needed to initialize the WAL for writes. In
13 * addition to these three main functions, there are a bunch of functions
14 * for interrogating recovery state and controlling the recovery process.
15 *
16 *
17 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
18 * Portions Copyright (c) 1994, Regents of the University of California
19 *
20 * src/backend/access/transam/xlogrecovery.c
21 *
22 *-------------------------------------------------------------------------
23 */
24
25 #include "postgres.h"
26
27 #include <ctype.h>
28 #include <math.h>
29 #include <time.h>
30 #include <sys/stat.h>
31 #include <sys/time.h>
32 #include <unistd.h>
33
34 #include "access/timeline.h"
35 #include "access/transam.h"
36 #include "access/xact.h"
37 #include "access/xlog_internal.h"
38 #include "access/xlogarchive.h"
39 #include "access/xlogprefetcher.h"
40 #include "access/xlogreader.h"
41 #include "access/xlogrecovery.h"
42 #include "access/xlogutils.h"
43 #include "backup/basebackup.h"
44 #include "catalog/pg_control.h"
45 #include "commands/tablespace.h"
46 #include "common/file_utils.h"
47 #include "miscadmin.h"
48 #include "nodes/miscnodes.h"
49 #include "pgstat.h"
50 #include "postmaster/bgwriter.h"
51 #include "postmaster/startup.h"
52 #include "replication/slot.h"
53 #include "replication/slotsync.h"
54 #include "replication/walreceiver.h"
55 #include "storage/fd.h"
56 #include "storage/ipc.h"
57 #include "storage/latch.h"
58 #include "storage/pmsignal.h"
59 #include "storage/procarray.h"
60 #include "storage/spin.h"
61 #include "utils/datetime.h"
62 #include "utils/fmgrprotos.h"
63 #include "utils/guc_hooks.h"
64 #include "utils/pgstat_internal.h"
65 #include "utils/pg_lsn.h"
66 #include "utils/ps_status.h"
67 #include "utils/pg_rusage.h"
68
69 /* Unsupported old recovery command file names (relative to $PGDATA) */
70 #define RECOVERY_COMMAND_FILE "recovery.conf"
71 #define RECOVERY_COMMAND_DONE "recovery.done"
72
73 /*
74 * GUC support
75 */
76 const struct config_enum_entry recovery_target_action_options[] = {
77 {"pause", RECOVERY_TARGET_ACTION_PAUSE, false},
78 {"promote", RECOVERY_TARGET_ACTION_PROMOTE, false},
79 {"shutdown", RECOVERY_TARGET_ACTION_SHUTDOWN, false},
80 {NULL, 0, false}
81 };
82
83 /* options formerly taken from recovery.conf for archive recovery */
84 char *recoveryRestoreCommand = NULL;
85 char *recoveryEndCommand = NULL;
86 char *archiveCleanupCommand = NULL;
87 RecoveryTargetType recoveryTarget = RECOVERY_TARGET_UNSET;
88 bool recoveryTargetInclusive = true;
89 int recoveryTargetAction = RECOVERY_TARGET_ACTION_PAUSE;
90 TransactionId recoveryTargetXid;
91 char *recovery_target_time_string;
92 TimestampTz recoveryTargetTime;
93 const char *recoveryTargetName;
94 XLogRecPtr recoveryTargetLSN;
95 int recovery_min_apply_delay = 0;
96
97 /* options formerly taken from recovery.conf for XLOG streaming */
98 char *PrimaryConnInfo = NULL;
99 char *PrimarySlotName = NULL;
100 bool wal_receiver_create_temp_slot = false;
101
102 /*
103 * recoveryTargetTimeLineGoal: what the user requested, if any
104 *
105 * recoveryTargetTLIRequested: numeric value of requested timeline, if constant
106 *
107 * recoveryTargetTLI: the currently understood target timeline; changes
108 *
109 * expectedTLEs: a list of TimeLineHistoryEntries for recoveryTargetTLI and
110 * the timelines of its known parents, newest first (so recoveryTargetTLI is
111 * always the first list member). Only these TLIs are expected to be seen in
112 * the WAL segments we read, and indeed only these TLIs will be considered as
113 * candidate WAL files to open at all.
114 *
115 * curFileTLI: the TLI appearing in the name of the current input WAL file.
116 * (This is not necessarily the same as the timeline from which we are
117 * replaying WAL, which StartupXLOG calls replayTLI, because we could be
118 * scanning data that was copied from an ancestor timeline when the current
119 * file was created.) During a sequential scan we do not allow this value
120 * to decrease.
121 */
122 RecoveryTargetTimeLineGoal recoveryTargetTimeLineGoal = RECOVERY_TARGET_TIMELINE_LATEST;
123 TimeLineID recoveryTargetTLIRequested = 0;
124 TimeLineID recoveryTargetTLI = 0;
125 static List *expectedTLEs;
126 static TimeLineID curFileTLI;
127
128 /*
129 * When ArchiveRecoveryRequested is set, archive recovery was requested,
130 * ie. signal files were present. When InArchiveRecovery is set, we are
131 * currently recovering using offline XLOG archives. These variables are only
132 * valid in the startup process.
133 *
134 * When ArchiveRecoveryRequested is true, but InArchiveRecovery is false, we're
135 * currently performing crash recovery using only XLOG files in pg_wal, but
136 * will switch to using offline XLOG archives as soon as we reach the end of
137 * WAL in pg_wal.
138 */
139 bool ArchiveRecoveryRequested = false;
140 bool InArchiveRecovery = false;
141
142 /*
143 * When StandbyModeRequested is set, standby mode was requested, i.e.
144 * standby.signal file was present. When StandbyMode is set, we are currently
145 * in standby mode. These variables are only valid in the startup process.
146 * They work similarly to ArchiveRecoveryRequested and InArchiveRecovery.
147 */
148 static bool StandbyModeRequested = false;
149 bool StandbyMode = false;
150
151 /* was a signal file present at startup? */
152 static bool standby_signal_file_found = false;
153 static bool recovery_signal_file_found = false;
154
155 /*
156 * CheckPointLoc is the position of the checkpoint record that determines
157 * where to start the replay. It comes from the backup label file or the
158 * control file.
159 *
160 * RedoStartLSN is the checkpoint's REDO location, also from the backup label
161 * file or the control file. In standby mode, XLOG streaming usually starts
162 * from the position where an invalid record was found. But if we fail to
163 * read even the initial checkpoint record, we use the REDO location instead
164 * of the checkpoint location as the start position of XLOG streaming.
165 * Otherwise we would have to jump backwards to the REDO location after
166 * reading the checkpoint record, because the REDO record can precede the
167 * checkpoint record.
168 */
169 static XLogRecPtr CheckPointLoc = InvalidXLogRecPtr;
170 static TimeLineID CheckPointTLI = 0;
171 static XLogRecPtr RedoStartLSN = InvalidXLogRecPtr;
172 static TimeLineID RedoStartTLI = 0;
173
174 /*
175 * Local copy of SharedHotStandbyActive variable. False actually means "not
176 * known, need to check the shared state".
177 */
178 static bool LocalHotStandbyActive = false;
179
180 /*
181 * Local copy of SharedPromoteIsTriggered variable. False actually means "not
182 * known, need to check the shared state".
183 */
184 static bool LocalPromoteIsTriggered = false;
185
186 /* Has the recovery code requested a walreceiver wakeup? */
187 static bool doRequestWalReceiverReply;
188
189 /* XLogReader object used to parse the WAL records */
190 static XLogReaderState *xlogreader = NULL;
191
192 /* XLogPrefetcher object used to consume WAL records with read-ahead */
193 static XLogPrefetcher *xlogprefetcher = NULL;
194
195 /* Parameters passed down from ReadRecord to the XLogPageRead callback. */
196 typedef struct XLogPageReadPrivate
197 {
198 int emode;
199 bool fetching_ckpt; /* are we fetching a checkpoint record? */
200 bool randAccess;
201 TimeLineID replayTLI;
202 } XLogPageReadPrivate;
203
204 /* flag to tell XLogPageRead that we have started replaying */
205 static bool InRedo = false;
206
207 /*
208 * Codes indicating where we got a WAL file from during recovery, or where
209 * to attempt to get one.
210 */
211 typedef enum
212 {
213 XLOG_FROM_ANY = 0, /* request to read WAL from any source */
214 XLOG_FROM_ARCHIVE, /* restored using restore_command */
215 XLOG_FROM_PG_WAL, /* existing file in pg_wal */
216 XLOG_FROM_STREAM, /* streamed from primary */
217 } XLogSource;
218
219 /* human-readable names for XLogSources, for debugging output */
220 static const char *const xlogSourceNames[] = {"any", "archive", "pg_wal", "stream"};
221
222 /*
223 * readFile is -1 or a kernel FD for the log file segment that's currently
224 * open for reading. readSegNo identifies the segment. readOff is the offset
225 * of the page just read, readLen indicates how much of it has been read into
226 * readBuf, and readSource indicates where we got the currently open file from.
227 *
228 * Note: we could use Reserve/ReleaseExternalFD to track consumption of this
229 * FD too (like for openLogFile in xlog.c); but it doesn't currently seem
230 * worthwhile, since the XLOG is not read by general-purpose sessions.
231 */
232 static int readFile = -1;
233 static XLogSegNo readSegNo = 0;
234 static uint32 readOff = 0;
235 static uint32 readLen = 0;
236 static XLogSource readSource = XLOG_FROM_ANY;
237
238 /*
239 * Keeps track of which source we're currently reading from. This is
240 * different from readSource in that this is always set, even when we don't
241 * currently have a WAL file open. If lastSourceFailed is set, our last
242 * attempt to read from currentSource failed, and we should try another source
243 * next.
244 *
245 * pendingWalRcvRestart is set when a config change occurs that requires a
246 * walreceiver restart. This is only valid in XLOG_FROM_STREAM state.
247 */
248 static XLogSource currentSource = XLOG_FROM_ANY;
249 static bool lastSourceFailed = false;
250 static bool pendingWalRcvRestart = false;
251
252 /*
253 * These variables track when we last obtained some WAL data to process,
254 * and where we got it from. (XLogReceiptSource is initially the same as
255 * readSource, but readSource gets reset to zero when we don't have data
256 * to process right now. It is also different from currentSource, which
257 * also changes when we try to read from a source and fail, while
258 * XLogReceiptSource tracks where we last successfully read some WAL.)
259 */
260 static TimestampTz XLogReceiptTime = 0;
261 static XLogSource XLogReceiptSource = XLOG_FROM_ANY;
262
263 /* Local copy of WalRcv->flushedUpto */
264 static XLogRecPtr flushedUpto = 0;
265 static TimeLineID receiveTLI = 0;
266
267 /*
268 * Copy of minRecoveryPoint and backupEndPoint from the control file.
269 *
270 * In order to reach consistency, we must replay the WAL up to
271 * minRecoveryPoint. If backupEndRequired is true, we must also reach
272 * backupEndPoint, or if it's invalid, an end-of-backup record corresponding
273 * to backupStartPoint.
274 *
275 * Note: In archive recovery, after consistency has been reached, the
276 * functions in xlog.c will start updating minRecoveryPoint in the control
277 * file. But this copy of minRecoveryPoint variable reflects the value at the
278 * beginning of recovery, and is *not* updated after consistency is reached.
279 */
280 static XLogRecPtr minRecoveryPoint;
281 static TimeLineID minRecoveryPointTLI;
282
283 static XLogRecPtr backupStartPoint;
284 static XLogRecPtr backupEndPoint;
285 static bool backupEndRequired = false;
286
287 /*
288 * Have we reached a consistent database state? In crash recovery, we have
289 * to replay all the WAL, so reachedConsistency is never set. During archive
290 * recovery, the database is consistent once minRecoveryPoint is reached.
291 *
292 * Consistent state means that the system is internally consistent, all
293 * the WAL has been replayed up to a certain point, and importantly, there
294 * is no trace of later actions on disk.
295 *
296 * This flag is used only by the startup process and postmaster. When
297 * minRecoveryPoint is reached, the startup process sets it to true and
298 * sends a PMSIGNAL_RECOVERY_CONSISTENT signal to the postmaster,
299 * which then sets it to true upon receiving the signal.
300 */
301 bool reachedConsistency = false;
302
303 /* Buffers dedicated to consistency checks of size BLCKSZ */
304 static char *replay_image_masked = NULL;
305 static char *primary_image_masked = NULL;
306
307
308 /*
309 * Shared-memory state for WAL recovery.
310 */
311 typedef struct XLogRecoveryCtlData
312 {
313 /*
314 * SharedHotStandbyActive indicates if we allow hot standby queries to be
315 * run. Protected by info_lck.
316 */
317 bool SharedHotStandbyActive;
318
319 /*
320 * SharedPromoteIsTriggered indicates if a standby promotion has been
321 * triggered. Protected by info_lck.
322 */
323 bool SharedPromoteIsTriggered;
324
325 /*
326 * recoveryWakeupLatch is used to wake up the startup process to continue
327 * WAL replay, if it is waiting for WAL to arrive or promotion to be
328 * requested.
329 *
330 * Note that the startup process also uses another latch, its procLatch,
331 * to wait for recovery conflict. If we get rid of recoveryWakeupLatch for
332 * signaling the startup process in favor of using its procLatch, which
333 * comports better with possible generic signal handlers using that latch.
334 * But we should not do that because the startup process doesn't assume
335 * that it's waken up by walreceiver process or SIGHUP signal handler
336 * while it's waiting for recovery conflict. The separate latches,
337 * recoveryWakeupLatch and procLatch, should be used for inter-process
338 * communication for WAL replay and recovery conflict, respectively.
339 */
340 Latch recoveryWakeupLatch;
341
342 /*
343 * Last record successfully replayed.
344 */
345 XLogRecPtr lastReplayedReadRecPtr; /* start position */
346 XLogRecPtr lastReplayedEndRecPtr; /* end+1 position */
347 TimeLineID lastReplayedTLI; /* timeline */
348
349 /*
350 * When we're currently replaying a record, ie. in a redo function,
351 * replayEndRecPtr points to the end+1 of the record being replayed,
352 * otherwise it's equal to lastReplayedEndRecPtr.
353 */
354 XLogRecPtr replayEndRecPtr;
355 TimeLineID replayEndTLI;
356 /* timestamp of last COMMIT/ABORT record replayed (or being replayed) */
357 TimestampTz recoveryLastXTime;
358
359 /*
360 * timestamp of when we started replaying the current chunk of WAL data,
361 * only relevant for replication or archive recovery
362 */
363 TimestampTz currentChunkStartTime;
364 /* Recovery pause state */
365 RecoveryPauseState recoveryPauseState;
366 ConditionVariable recoveryNotPausedCV;
367
368 slock_t info_lck; /* locks shared variables shown above */
369 } XLogRecoveryCtlData;
370
371 static XLogRecoveryCtlData *XLogRecoveryCtl = NULL;
372
373 /*
374 * abortedRecPtr is the start pointer of a broken record at end of WAL when
375 * recovery completes; missingContrecPtr is the location of the first
376 * contrecord that went missing. See CreateOverwriteContrecordRecord for
377 * details.
378 */
379 static XLogRecPtr abortedRecPtr;
380 static XLogRecPtr missingContrecPtr;
381
382 /*
383 * if recoveryStopsBefore/After returns true, it saves information of the stop
384 * point here
385 */
386 static TransactionId recoveryStopXid;
387 static TimestampTz recoveryStopTime;
388 static XLogRecPtr recoveryStopLSN;
389 static char recoveryStopName[MAXFNAMELEN];
390 static bool recoveryStopAfter;
391
392 /* prototypes for local functions */
393 static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI);
394
395 static void EnableStandbyMode(void);
396 static void readRecoverySignalFile(void);
397 static void validateRecoveryParameters(void);
398 static bool read_backup_label(XLogRecPtr *checkPointLoc,
399 TimeLineID *backupLabelTLI,
400 bool *backupEndRequired, bool *backupFromStandby);
401 static bool read_tablespace_map(List **tablespaces);
402
403 static void xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI);
404 static void CheckRecoveryConsistency(void);
405 static void rm_redo_error_callback(void *arg);
406 #ifdef WAL_DEBUG
407 static void xlog_outrec(StringInfo buf, XLogReaderState *record);
408 #endif
409 static void xlog_block_info(StringInfo buf, XLogReaderState *record);
410 static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
411 TimeLineID prevTLI, TimeLineID replayTLI);
412 static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime);
413 static void verifyBackupPageConsistency(XLogReaderState *record);
414
415 static bool recoveryStopsBefore(XLogReaderState *record);
416 static bool recoveryStopsAfter(XLogReaderState *record);
417 static char *getRecoveryStopReason(void);
418 static void recoveryPausesHere(bool endOfRecovery);
419 static bool recoveryApplyDelay(XLogReaderState *record);
420 static void ConfirmRecoveryPaused(void);
421
422 static XLogRecord *ReadRecord(XLogPrefetcher *xlogprefetcher,
423 int emode, bool fetching_ckpt,
424 TimeLineID replayTLI);
425
426 static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
427 int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
428 static XLogPageReadResult WaitForWALToBecomeAvailable(XLogRecPtr RecPtr,
429 bool randAccess,
430 bool fetching_ckpt,
431 XLogRecPtr tliRecPtr,
432 TimeLineID replayTLI,
433 XLogRecPtr replayLSN,
434 bool nonblocking);
435 static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
436 static XLogRecord *ReadCheckpointRecord(XLogPrefetcher *xlogprefetcher,
437 XLogRecPtr RecPtr, TimeLineID replayTLI);
438 static bool rescanLatestTimeLine(TimeLineID replayTLI, XLogRecPtr replayLSN);
439 static int XLogFileRead(XLogSegNo segno, TimeLineID tli,
440 XLogSource source, bool notfoundOk);
441 static int XLogFileReadAnyTLI(XLogSegNo segno, XLogSource source);
442
443 static bool CheckForStandbyTrigger(void);
444 static void SetPromoteIsTriggered(void);
445 static bool HotStandbyActiveInReplay(void);
446
447 static void SetCurrentChunkStartTime(TimestampTz xtime);
448 static void SetLatestXTime(TimestampTz xtime);
449
450 /*
451 * Initialization of shared memory for WAL recovery
452 */
453 Size
454 XLogRecoveryShmemSize(void)
455 {
456 Size size;
457
458 /* XLogRecoveryCtl */
459 size = sizeof(XLogRecoveryCtlData);
460
461 return size;
462 }
463
464 void
465 XLogRecoveryShmemInit(void)
466 {
467 bool found;
468
469 XLogRecoveryCtl = (XLogRecoveryCtlData *)
470 ShmemInitStruct("XLOG Recovery Ctl", XLogRecoveryShmemSize(), &found);
471 if (found)
472 return;
473 memset(XLogRecoveryCtl, 0, sizeof(XLogRecoveryCtlData));
474
475 SpinLockInit(&XLogRecoveryCtl->info_lck);
476 InitSharedLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
477 ConditionVariableInit(&XLogRecoveryCtl->recoveryNotPausedCV);
478 }
479
480 /*
481 * A thin wrapper to enable StandbyMode and do other preparatory work as
482 * needed.
483 */
484 static void
485 EnableStandbyMode(void)
486 {
487 StandbyMode = true;
488
489 /*
490 * To avoid server log bloat, we don't report recovery progress in a
491 * standby as it will always be in recovery unless promoted. We disable
492 * startup progress timeout in standby mode to avoid calling
493 * startup_progress_timeout_handler() unnecessarily.
494 */
495 disable_startup_progress_timeout();
496 }
497
498 /*
499 * Prepare the system for WAL recovery, if needed.
500 *
501 * This is called by StartupXLOG() which coordinates the server startup
502 * sequence. This function analyzes the control file and the backup label
503 * file, if any, and figures out whether we need to perform crash recovery or
504 * archive recovery, and how far we need to replay the WAL to reach a
505 * consistent state.
506 *
507 * This doesn't yet change the on-disk state, except for creating the symlinks
508 * from table space map file if any, and for fetching WAL files needed to find
509 * the checkpoint record. On entry, the caller has already read the control
510 * file into memory, and passes it as argument. This function updates it to
511 * reflect the recovery state, and the caller is expected to write it back to
512 * disk does after initializing other subsystems, but before calling
513 * PerformWalRecovery().
514 *
515 * This initializes some global variables like ArchiveRecoveryRequested, and
516 * StandbyModeRequested and InRecovery.
517 */
518 void
519 InitWalRecovery(ControlFileData *ControlFile, bool *wasShutdown_ptr,
520 bool *haveBackupLabel_ptr, bool *haveTblspcMap_ptr)
521 {
522 XLogPageReadPrivate *private;
523 struct stat st;
524 bool wasShutdown;
525 XLogRecord *record;
526 DBState dbstate_at_startup;
527 bool haveTblspcMap = false;
528 bool haveBackupLabel = false;
529 CheckPoint checkPoint;
530 bool backupFromStandby = false;
531
532 dbstate_at_startup = ControlFile->state;
533
534 /*
535 * A startup process always starts with an inconsistent database. Set the
536 * flag accordingly, even if it was inherited from a postmaster that had
537 * already marked the database as consistent. This keeps the invariant
538 * local to the startup process without requiring every fork path to clear
539 * the flag.
540 */
541 reachedConsistency = false;
542
543 /*
544 * Initialize on the assumption we want to recover to the latest timeline
545 * that's active according to pg_control.
546 */
547 if (ControlFile->minRecoveryPointTLI >
548 ControlFile->checkPointCopy.ThisTimeLineID)
549 recoveryTargetTLI = ControlFile->minRecoveryPointTLI;
550 else
551 recoveryTargetTLI = ControlFile->checkPointCopy.ThisTimeLineID;
552
553 /*
554 * Check for signal files, and if so set up state for offline recovery
555 */
556 readRecoverySignalFile();
557 validateRecoveryParameters();
558
559 /*
560 * Take ownership of the wakeup latch if we're going to sleep during
561 * recovery, if required.
562 */
563 if (ArchiveRecoveryRequested)
564 OwnLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
565
566 /*
567 * Set the WAL reading processor now, as it will be needed when reading
568 * the checkpoint record required (backup_label or not).
569 */
570 private = palloc0(sizeof(XLogPageReadPrivate));
571 xlogreader =
572 XLogReaderAllocate(wal_segment_size, NULL,
573 XL_ROUTINE(.page_read = &XLogPageRead,
574 .segment_open = NULL,
575 .segment_close = wal_segment_close),
576 private);
577 if (!xlogreader)
578 ereport(ERROR,
579 (errcode(ERRCODE_OUT_OF_MEMORY),
580 errmsg("out of memory"),
581 errdetail("Failed while allocating a WAL reading processor.")));
582 xlogreader->system_identifier = ControlFile->system_identifier;
583
584 /*
585 * Set the WAL decode buffer size. This limits how far ahead we can read
586 * in the WAL.
587 */
588 XLogReaderSetDecodeBuffer(xlogreader, NULL, wal_decode_buffer_size);
589
590 /* Create a WAL prefetcher. */
591 xlogprefetcher = XLogPrefetcherAllocate(xlogreader);
592
593 /*
594 * Allocate two page buffers dedicated to WAL consistency checks. We do
595 * it this way, rather than just making static arrays, for two reasons:
596 * (1) no need to waste the storage in most instantiations of the backend;
597 * (2) a static char array isn't guaranteed to have any particular
598 * alignment, whereas palloc() will provide MAXALIGN'd storage.
599 */
600 replay_image_masked = (char *) palloc(BLCKSZ);
601 primary_image_masked = (char *) palloc(BLCKSZ);
602
603 /*
604 * Read the backup_label file. We want to run this part of the recovery
605 * process after checking for signal files and after performing validation
606 * of the recovery parameters.
607 */
608 if (read_backup_label(&CheckPointLoc, &CheckPointTLI, &backupEndRequired,
609 &backupFromStandby))
610 {
611 List *tablespaces = NIL;
612
613 /*
614 * Archive recovery was requested, and thanks to the backup label
615 * file, we know how far we need to replay to reach consistency. Enter
616 * archive recovery directly.
617 */
618 InArchiveRecovery = true;
619 if (StandbyModeRequested)
620 EnableStandbyMode();
621
622 /*
623 * Omitting backup_label when creating a new replica, PITR node etc.
624 * unfortunately is a common cause of corruption. Logging that
625 * backup_label was used makes it a bit easier to exclude that as the
626 * cause of observed corruption.
627 *
628 * Do so before we try to read the checkpoint record (which can fail),
629 * as otherwise it can be hard to understand why a checkpoint other
630 * than ControlFile->checkPoint is used.
631 */
632 ereport(LOG,
633 (errmsg("starting backup recovery with redo LSN %X/%X, checkpoint LSN %X/%X, on timeline ID %u",
634 LSN_FORMAT_ARGS(RedoStartLSN),
635 LSN_FORMAT_ARGS(CheckPointLoc),
636 CheckPointTLI)));
637
638 /*
639 * When a backup_label file is present, we want to roll forward from
640 * the checkpoint it identifies, rather than using pg_control.
641 */
642 record = ReadCheckpointRecord(xlogprefetcher, CheckPointLoc,
643 CheckPointTLI);
644 if (record != NULL)
645 {
646 memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
647 wasShutdown = ((record->xl_info & ~XLR_INFO_MASK) == XLOG_CHECKPOINT_SHUTDOWN);
648 ereport(DEBUG1,
649 (errmsg_internal("checkpoint record is at %X/%X",
650 LSN_FORMAT_ARGS(CheckPointLoc))));
651 InRecovery = true; /* force recovery even if SHUTDOWNED */
652
653 /*
654 * Make sure that REDO location exists. This may not be the case
655 * if there was a crash during an online backup, which left a
656 * backup_label around that references a WAL segment that's
657 * already been archived.
658 */
659 if (checkPoint.redo < CheckPointLoc)
660 {
661 XLogPrefetcherBeginRead(xlogprefetcher, checkPoint.redo);
662 if (!ReadRecord(xlogprefetcher, LOG, false,
663 checkPoint.ThisTimeLineID))
664 ereport(FATAL,
665 (errmsg("could not find redo location %X/%X referenced by checkpoint record at %X/%X",
666 LSN_FORMAT_ARGS(checkPoint.redo), LSN_FORMAT_ARGS(CheckPointLoc)),
667 errhint("If you are restoring from a backup, touch \"%s/recovery.signal\" or \"%s/standby.signal\" and add required recovery options.\n"
668 "If you are not restoring from a backup, try removing the file \"%s/backup_label\".\n"
669 "Be careful: removing \"%s/backup_label\" will result in a corrupt cluster if restoring from a backup.",
670 DataDir, DataDir, DataDir, DataDir)));
671 }
672 }
673 else
674 {
675 ereport(FATAL,
676 (errmsg("could not locate required checkpoint record at %X/%X",
677 LSN_FORMAT_ARGS(CheckPointLoc)),
678 errhint("If you are restoring from a backup, touch \"%s/recovery.signal\" or \"%s/standby.signal\" and add required recovery options.\n"
679 "If you are not restoring from a backup, try removing the file \"%s/backup_label\".\n"
680 "Be careful: removing \"%s/backup_label\" will result in a corrupt cluster if restoring from a backup.",
681 DataDir, DataDir, DataDir, DataDir)));
682 wasShutdown = false; /* keep compiler quiet */
683 }
684
685 /* Read the tablespace_map file if present and create symlinks. */
686 if (read_tablespace_map(&tablespaces))
687 {
688 ListCell *lc;
689
690 foreach(lc, tablespaces)
691 {
692 tablespaceinfo *ti = lfirst(lc);
693 char *linkloc;
694
695 linkloc = psprintf("%s/%u", PG_TBLSPC_DIR, ti->oid);
696
697 /*
698 * Remove the existing symlink if any and Create the symlink
699 * under PGDATA.
700 */
701 remove_tablespace_symlink(linkloc);
702
703 if (symlink(ti->path, linkloc) < 0)
704 ereport(ERROR,
705 (errcode_for_file_access(),
706 errmsg("could not create symbolic link \"%s\": %m",
707 linkloc)));
708
709 pfree(ti->path);
710 pfree(ti);
711 }
712
713 /* tell the caller to delete it later */
714 haveTblspcMap = true;
715 }
716
717 /* tell the caller to delete it later */
718 haveBackupLabel = true;
719 }
720 else
721 {
722 /* No backup_label file has been found if we are here. */
723
724 /*
725 * If tablespace_map file is present without backup_label file, there
726 * is no use of such file. There is no harm in retaining it, but it
727 * is better to get rid of the map file so that we don't have any
728 * redundant file in data directory and it will avoid any sort of
729 * confusion. It seems prudent though to just rename the file out of
730 * the way rather than delete it completely, also we ignore any error
731 * that occurs in rename operation as even if map file is present
732 * without backup_label file, it is harmless.
733 */
734 if (stat(TABLESPACE_MAP, &st) == 0)
735 {
736 unlink(TABLESPACE_MAP_OLD);
737 if (durable_rename(TABLESPACE_MAP, TABLESPACE_MAP_OLD, DEBUG1) == 0)
738 ereport(LOG,
739 (errmsg("ignoring file \"%s\" because no file \"%s\" exists",
740 TABLESPACE_MAP, BACKUP_LABEL_FILE),
741 errdetail("File \"%s\" was renamed to \"%s\".",
742 TABLESPACE_MAP, TABLESPACE_MAP_OLD)));
743 else
744 ereport(LOG,
745 (errmsg("ignoring file \"%s\" because no file \"%s\" exists",
746 TABLESPACE_MAP, BACKUP_LABEL_FILE),
747 errdetail("Could not rename file \"%s\" to \"%s\": %m.",
748 TABLESPACE_MAP, TABLESPACE_MAP_OLD)));
749 }
750
751 /*
752 * It's possible that archive recovery was requested, but we don't
753 * know how far we need to replay the WAL before we reach consistency.
754 * This can happen for example if a base backup is taken from a
755 * running server using an atomic filesystem snapshot, without calling
756 * pg_backup_start/stop. Or if you just kill a running primary server
757 * and put it into archive recovery by creating a recovery signal
758 * file.
759 *
760 * Our strategy in that case is to perform crash recovery first,
761 * replaying all the WAL present in pg_wal, and only enter archive
762 * recovery after that.
763 *
764 * But usually we already know how far we need to replay the WAL (up
765 * to minRecoveryPoint, up to backupEndPoint, or until we see an
766 * end-of-backup record), and we can enter archive recovery directly.
767 */
768 if (ArchiveRecoveryRequested &&
769 (ControlFile->minRecoveryPoint != InvalidXLogRecPtr ||
770 ControlFile->backupEndRequired ||
771 ControlFile->backupEndPoint != InvalidXLogRecPtr ||
772 ControlFile->state == DB_SHUTDOWNED))
773 {
774 InArchiveRecovery = true;
775 if (StandbyModeRequested)
776 EnableStandbyMode();
777 }
778
779 /*
780 * For the same reason as when starting up with backup_label present,
781 * emit a log message when we continue initializing from a base
782 * backup.
783 */
784 if (!XLogRecPtrIsInvalid(ControlFile->backupStartPoint))
785 ereport(LOG,
786 (errmsg("restarting backup recovery with redo LSN %X/%X",
787 LSN_FORMAT_ARGS(ControlFile->backupStartPoint))));
788
789 /* Get the last valid checkpoint record. */
790 CheckPointLoc = ControlFile->checkPoint;
791 CheckPointTLI = ControlFile->checkPointCopy.ThisTimeLineID;
792 RedoStartLSN = ControlFile->checkPointCopy.redo;
793 RedoStartTLI = ControlFile->checkPointCopy.ThisTimeLineID;
794 record = ReadCheckpointRecord(xlogprefetcher, CheckPointLoc,
795 CheckPointTLI);
796 if (record != NULL)
797 {
798 ereport(DEBUG1,
799 (errmsg_internal("checkpoint record is at %X/%X",
800 LSN_FORMAT_ARGS(CheckPointLoc))));
801 }
802 else
803 {
804 /*
805 * We used to attempt to go back to a secondary checkpoint record
806 * here, but only when not in standby mode. We now just fail if we
807 * can't read the last checkpoint because this allows us to
808 * simplify processing around checkpoints.
809 */
810 ereport(PANIC,
811 (errmsg("could not locate a valid checkpoint record at %X/%X",
812 LSN_FORMAT_ARGS(CheckPointLoc))));
813 }
814 memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
815 wasShutdown = ((record->xl_info & ~XLR_INFO_MASK) == XLOG_CHECKPOINT_SHUTDOWN);
816
817 /* Make sure that REDO location exists. */
818 if (checkPoint.redo < CheckPointLoc)
819 {
820 XLogPrefetcherBeginRead(xlogprefetcher, checkPoint.redo);
821 if (!ReadRecord(xlogprefetcher, LOG, false, checkPoint.ThisTimeLineID))
822 ereport(PANIC,
823 errmsg("could not find redo location %X/%08X referenced by checkpoint record at %X/%08X",
824 LSN_FORMAT_ARGS(checkPoint.redo), LSN_FORMAT_ARGS(CheckPointLoc)));
825 }
826 }
827
828 if (ArchiveRecoveryRequested)
829 {
830 if (StandbyModeRequested)
831 ereport(LOG,
832 (errmsg("entering standby mode")));
833 else if (recoveryTarget == RECOVERY_TARGET_XID)
834 ereport(LOG,
835 (errmsg("starting point-in-time recovery to XID %u",
836 recoveryTargetXid)));
837 else if (recoveryTarget == RECOVERY_TARGET_TIME)
838 ereport(LOG,
839 (errmsg("starting point-in-time recovery to %s",
840 timestamptz_to_str(recoveryTargetTime))));
841 else if (recoveryTarget == RECOVERY_TARGET_NAME)
842 ereport(LOG,
843 (errmsg("starting point-in-time recovery to \"%s\"",
844 recoveryTargetName)));
845 else if (recoveryTarget == RECOVERY_TARGET_LSN)
846 ereport(LOG,
847 (errmsg("starting point-in-time recovery to WAL location (LSN) \"%X/%X\"",
848 LSN_FORMAT_ARGS(recoveryTargetLSN))));
849 else if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE)
850 ereport(LOG,
851 (errmsg("starting point-in-time recovery to earliest consistent point")));
852 else
853 ereport(LOG,
854 (errmsg("starting archive recovery")));
855 }
856
857 /*
858 * If the location of the checkpoint record is not on the expected
859 * timeline in the history of the requested timeline, we cannot proceed:
860 * the backup is not part of the history of the requested timeline.
861 */
862 Assert(expectedTLEs); /* was initialized by reading checkpoint
863 * record */
864 if (tliOfPointInHistory(CheckPointLoc, expectedTLEs) !=
865 CheckPointTLI)
866 {
867 XLogRecPtr switchpoint;
868
869 /*
870 * tliSwitchPoint will throw an error if the checkpoint's timeline is
871 * not in expectedTLEs at all.
872 */
873 switchpoint = tliSwitchPoint(CheckPointTLI, expectedTLEs, NULL);
874 ereport(FATAL,
875 (errmsg("requested timeline %u is not a child of this server's history",
876 recoveryTargetTLI),
877 /* translator: %s is a backup_label file or a pg_control file */
878 errdetail("Latest checkpoint in file \"%s\" is at %X/%X on timeline %u, but in the history of the requested timeline, the server forked off from that timeline at %X/%X.",
879 haveBackupLabel ? "backup_label" : "pg_control",
880 LSN_FORMAT_ARGS(CheckPointLoc),
881 CheckPointTLI,
882 LSN_FORMAT_ARGS(switchpoint))));
883 }
884
885 /*
886 * The min recovery point should be part of the requested timeline's
887 * history, too.
888 */
889 if (!XLogRecPtrIsInvalid(ControlFile->minRecoveryPoint) &&
890 tliOfPointInHistory(ControlFile->minRecoveryPoint - 1, expectedTLEs) !=
891 ControlFile->minRecoveryPointTLI)
892 ereport(FATAL,
893 (errmsg("requested timeline %u does not contain minimum recovery point %X/%X on timeline %u",
894 recoveryTargetTLI,
895 LSN_FORMAT_ARGS(ControlFile->minRecoveryPoint),
896 ControlFile->minRecoveryPointTLI)));
897
898 ereport(DEBUG1,
899 (errmsg_internal("redo record is at %X/%X; shutdown %s",
900 LSN_FORMAT_ARGS(checkPoint.redo),
901 wasShutdown ? "true" : "false")));
902 ereport(DEBUG1,
903 (errmsg_internal("next transaction ID: " UINT64_FORMAT "; next OID: %u",
904 U64FromFullTransactionId(checkPoint.nextXid),
905 checkPoint.nextOid)));
906 ereport(DEBUG1,
907 (errmsg_internal("next MultiXactId: %u; next MultiXactOffset: %u",
908 checkPoint.nextMulti, checkPoint.nextMultiOffset)));
909 ereport(DEBUG1,
910 (errmsg_internal("oldest unfrozen transaction ID: %u, in database %u",
911 checkPoint.oldestXid, checkPoint.oldestXidDB)));
912 ereport(DEBUG1,
913 (errmsg_internal("oldest MultiXactId: %u, in database %u",
914 checkPoint.oldestMulti, checkPoint.oldestMultiDB)));
915 ereport(DEBUG1,
916 (errmsg_internal("commit timestamp Xid oldest/newest: %u/%u",
917 checkPoint.oldestCommitTsXid,
918 checkPoint.newestCommitTsXid)));
919 if (!TransactionIdIsNormal(XidFromFullTransactionId(checkPoint.nextXid)))
920 ereport(PANIC,
921 (errmsg("invalid next transaction ID")));
922
923 /* sanity check */
924 if (checkPoint.redo > CheckPointLoc)
925 ereport(PANIC,
926 (errmsg("invalid redo in checkpoint record")));
927
928 /*
929 * Check whether we need to force recovery from WAL. If it appears to
930 * have been a clean shutdown and we did not have a recovery signal file,
931 * then assume no recovery needed.
932 */
933 if (checkPoint.redo < CheckPointLoc)
934 {
935 if (wasShutdown)
936 ereport(PANIC,
937 (errmsg("invalid redo record in shutdown checkpoint")));
938 InRecovery = true;
939 }
940 else if (ControlFile->state != DB_SHUTDOWNED)
941 InRecovery = true;
942 else if (ArchiveRecoveryRequested)
943 {
944 /* force recovery due to presence of recovery signal file */
945 InRecovery = true;
946 }
947
948 /*
949 * If recovery is needed, update our in-memory copy of pg_control to show
950 * that we are recovering and to show the selected checkpoint as the place
951 * we are starting from. We also mark pg_control with any minimum recovery
952 * stop point obtained from a backup history file.
953 *
954 * We don't write the changes to disk yet, though. Only do that after
955 * initializing various subsystems.
956 */
957 if (InRecovery)
958 {
959 if (InArchiveRecovery)
960 {
961 ControlFile->state = DB_IN_ARCHIVE_RECOVERY;
962 }
963 else
964 {
965 ereport(LOG,
966 (errmsg("database system was not properly shut down; "
967 "automatic recovery in progress")));
968 if (recoveryTargetTLI > ControlFile->checkPointCopy.ThisTimeLineID)
969 ereport(LOG,
970 (errmsg("crash recovery starts in timeline %u "
971 "and has target timeline %u",
972 ControlFile->checkPointCopy.ThisTimeLineID,
973 recoveryTargetTLI)));
974 ControlFile->state = DB_IN_CRASH_RECOVERY;
975 }
976 ControlFile->checkPoint = CheckPointLoc;
977 ControlFile->checkPointCopy = checkPoint;
978 if (InArchiveRecovery)
979 {
980 /* initialize minRecoveryPoint if not set yet */
981 if (ControlFile->minRecoveryPoint < checkPoint.redo)
982 {
983 ControlFile->minRecoveryPoint = checkPoint.redo;
984 ControlFile->minRecoveryPointTLI = checkPoint.ThisTimeLineID;
985 }
986 }
987
988 /*
989 * Set backupStartPoint if we're starting recovery from a base backup.
990 *
991 * Also set backupEndPoint and use minRecoveryPoint as the backup end
992 * location if we're starting recovery from a base backup which was
993 * taken from a standby. In this case, the database system status in
994 * pg_control must indicate that the database was already in recovery.
995 * Usually that will be DB_IN_ARCHIVE_RECOVERY but also can be
996 * DB_SHUTDOWNED_IN_RECOVERY if recovery previously was interrupted
997 * before reaching this point; e.g. because restore_command or
998 * primary_conninfo were faulty.
999 *
1000 * Any other state indicates that the backup somehow became corrupted
1001 * and we can't sensibly continue with recovery.
1002 */
1003 if (haveBackupLabel)
1004 {
1005 ControlFile->backupStartPoint = checkPoint.redo;
1006 ControlFile->backupEndRequired = backupEndRequired;
1007
1008 if (backupFromStandby)
1009 {
1010 if (dbstate_at_startup != DB_IN_ARCHIVE_RECOVERY &&
1011 dbstate_at_startup != DB_SHUTDOWNED_IN_RECOVERY)
1012 ereport(FATAL,
1013 (errmsg("backup_label contains data inconsistent with control file"),
1014 errhint("This means that the backup is corrupted and you will "
1015 "have to use another backup for recovery.")));
1016 ControlFile->backupEndPoint = ControlFile->minRecoveryPoint;
1017 }
1018 }
1019 }
1020
1021 /* remember these, so that we know when we have reached consistency */
1022 backupStartPoint = ControlFile->backupStartPoint;
1023 backupEndRequired = ControlFile->backupEndRequired;
1024 backupEndPoint = ControlFile->backupEndPoint;
1025 if (InArchiveRecovery)
1026 {
1027 minRecoveryPoint = ControlFile->minRecoveryPoint;
1028 minRecoveryPointTLI = ControlFile->minRecoveryPointTLI;
1029 }
1030 else
1031 {
1032 minRecoveryPoint = InvalidXLogRecPtr;
1033 minRecoveryPointTLI = 0;
1034 }
1035
1036 /*
1037 * Start recovery assuming that the final record isn't lost.
1038 */
1039 abortedRecPtr = InvalidXLogRecPtr;
1040 missingContrecPtr = InvalidXLogRecPtr;
1041
1042 *wasShutdown_ptr = wasShutdown;
1043 *haveBackupLabel_ptr = haveBackupLabel;
1044 *haveTblspcMap_ptr = haveTblspcMap;
1045 }
1046
1047 /*
1048 * See if there are any recovery signal files and if so, set state for
1049 * recovery.
1050 *
1051 * See if there is a recovery command file (recovery.conf), and if so
1052 * throw an ERROR since as of PG12 we no longer recognize that.
1053 */
1054 static void
1055 readRecoverySignalFile(void)
1056 {
1057 struct stat stat_buf;
1058
1059 if (IsBootstrapProcessingMode())
1060 return;
1061
1062 /*
1063 * Check for old recovery API file: recovery.conf
1064 */
1065 if (stat(RECOVERY_COMMAND_FILE, &stat_buf) == 0)
1066 ereport(FATAL,
1067 (errcode_for_file_access(),
1068 errmsg("using recovery command file \"%s\" is not supported",
1069 RECOVERY_COMMAND_FILE)));
1070
1071 /*
1072 * Remove unused .done file, if present. Ignore if absent.
1073 */
1074 unlink(RECOVERY_COMMAND_DONE);
1075
1076 /*
1077 * Check for recovery signal files and if found, fsync them since they
1078 * represent server state information. We don't sweat too much about the
1079 * possibility of fsync failure, however.
1080 *
1081 * If present, standby signal file takes precedence. If neither is present
1082 * then we won't enter archive recovery.
1083 */
1084 if (stat(STANDBY_SIGNAL_FILE, &stat_buf) == 0)
1085 {
1086 int fd;
1087
1088 fd = BasicOpenFilePerm(STANDBY_SIGNAL_FILE, O_RDWR | PG_BINARY,
1089 S_IRUSR | S_IWUSR);
1090 if (fd >= 0)
1091 {
1092 (void) pg_fsync(fd);
1093 close(fd);
1094 }
1095 standby_signal_file_found = true;
1096 }
1097 else if (stat(RECOVERY_SIGNAL_FILE, &stat_buf) == 0)
1098 {
1099 int fd;
1100
1101 fd = BasicOpenFilePerm(RECOVERY_SIGNAL_FILE, O_RDWR | PG_BINARY,
1102 S_IRUSR | S_IWUSR);
1103 if (fd >= 0)
1104 {
1105 (void) pg_fsync(fd);
1106 close(fd);
1107 }
1108 recovery_signal_file_found = true;
1109 }
1110
1111 StandbyModeRequested = false;
1112 ArchiveRecoveryRequested = false;
1113 if (standby_signal_file_found)
1114 {
1115 StandbyModeRequested = true;
1116 ArchiveRecoveryRequested = true;
1117 }
1118 else if (recovery_signal_file_found)
1119 {
1120 StandbyModeRequested = false;
1121 ArchiveRecoveryRequested = true;
1122 }
1123 else
1124 return;
1125
1126 /*
1127 * We don't support standby mode in standalone backends; that requires
1128 * other processes such as the WAL receiver to be alive.
1129 */
1130 if (StandbyModeRequested && !IsUnderPostmaster)
1131 ereport(FATAL,
1132 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1133 errmsg("standby mode is not supported by single-user servers")));
1134 }
1135
1136 static void
1137 validateRecoveryParameters(void)
1138 {
1139 if (!ArchiveRecoveryRequested)
1140 return;
1141
1142 /*
1143 * Check for compulsory parameters
1144 */
1145 if (StandbyModeRequested)
1146 {
1147 if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
1148 (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
1149 ereport(WARNING,
1150 (errmsg("specified neither \"primary_conninfo\" nor \"restore_command\""),
1151 errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
1152 }
1153 else
1154 {
1155 if (recoveryRestoreCommand == NULL ||
1156 strcmp(recoveryRestoreCommand, "") == 0)
1157 ereport(FATAL,
1158 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1159 errmsg("must specify \"restore_command\" when standby mode is not enabled")));
1160 }
1161
1162 /*
1163 * Override any inconsistent requests. Note that this is a change of
1164 * behaviour in 9.5; prior to this we simply ignored a request to pause if
1165 * hot_standby = off, which was surprising behaviour.
1166 */
1167 if (recoveryTargetAction == RECOVERY_TARGET_ACTION_PAUSE &&
1168 !EnableHotStandby)
1169 recoveryTargetAction = RECOVERY_TARGET_ACTION_SHUTDOWN;
1170
1171 /*
1172 * Final parsing of recovery_target_time string; see also
1173 * check_recovery_target_time().
1174 */
1175 if (recoveryTarget == RECOVERY_TARGET_TIME)
1176 {
1177 recoveryTargetTime = DatumGetTimestampTz(DirectFunctionCall3(timestamptz_in,
1178 CStringGetDatum(recovery_target_time_string),
1179 ObjectIdGetDatum(InvalidOid),
1180 Int32GetDatum(-1)));
1181 }
1182
1183 /*
1184 * If user specified recovery_target_timeline, validate it or compute the
1185 * "latest" value. We can't do this until after we've gotten the restore
1186 * command and set InArchiveRecovery, because we need to fetch timeline
1187 * history files from the archive.
1188 */
1189 if (recoveryTargetTimeLineGoal == RECOVERY_TARGET_TIMELINE_NUMERIC)
1190 {
1191 TimeLineID rtli = recoveryTargetTLIRequested;
1192
1193 /* Timeline 1 does not have a history file, all else should */
1194 if (rtli != 1 && !existsTimeLineHistory(rtli))
1195 ereport(FATAL,
1196 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1197 errmsg("recovery target timeline %u does not exist",
1198 rtli)));
1199 recoveryTargetTLI = rtli;
1200 }
1201 else if (recoveryTargetTimeLineGoal == RECOVERY_TARGET_TIMELINE_LATEST)
1202 {
1203 /* We start the "latest" search from pg_control's timeline */
1204 recoveryTargetTLI = findNewestTimeLine(recoveryTargetTLI);
1205 }
1206 else
1207 {
1208 /*
1209 * else we just use the recoveryTargetTLI as already read from
1210 * ControlFile
1211 */
1212 Assert(recoveryTargetTimeLineGoal == RECOVERY_TARGET_TIMELINE_CONTROLFILE);
1213 }
1214 }
1215
1216 /*
1217 * read_backup_label: check to see if a backup_label file is present
1218 *
1219 * If we see a backup_label during recovery, we assume that we are recovering
1220 * from a backup dump file, and we therefore roll forward from the checkpoint
1221 * identified by the label file, NOT what pg_control says. This avoids the
1222 * problem that pg_control might have been archived one or more checkpoints
1223 * later than the start of the dump, and so if we rely on it as the start
1224 * point, we will fail to restore a consistent database state.
1225 *
1226 * Returns true if a backup_label was found (and fills the checkpoint
1227 * location and TLI into *checkPointLoc and *backupLabelTLI, respectively);
1228 * returns false if not. If this backup_label came from a streamed backup,
1229 * *backupEndRequired is set to true. If this backup_label was created during
1230 * recovery, *backupFromStandby is set to true.
1231 *
1232 * Also sets the global variables RedoStartLSN and RedoStartTLI with the LSN
1233 * and TLI read from the backup file.
1234 */
1235 static bool
1236 read_backup_label(XLogRecPtr *checkPointLoc, TimeLineID *backupLabelTLI,
1237 bool *backupEndRequired, bool *backupFromStandby)
1238 {
1239 char startxlogfilename[MAXFNAMELEN];
1240 TimeLineID tli_from_walseg,
1241 tli_from_file;
1242 FILE *lfp;
1243 char ch;
1244 char backuptype[20];
1245 char backupfrom[20];
1246 char backuplabel[MAXPGPATH];
1247 char backuptime[128];
1248 uint32 hi,
1249 lo;
1250
1251 /* suppress possible uninitialized-variable warnings */
1252 *checkPointLoc = InvalidXLogRecPtr;
1253 *backupLabelTLI = 0;
1254 *backupEndRequired = false;
1255 *backupFromStandby = false;
1256
1257 /*
1258 * See if label file is present
1259 */
1260 lfp = AllocateFile(BACKUP_LABEL_FILE, "r");
1261 if (!lfp)
1262 {
1263 if (errno != ENOENT)
1264 ereport(FATAL,
1265 (errcode_for_file_access(),
1266 errmsg("could not read file \"%s\": %m",
1267 BACKUP_LABEL_FILE)));
1268 return false; /* it's not there, all is fine */
1269 }
1270
1271 /*
1272 * Read and parse the START WAL LOCATION and CHECKPOINT lines (this code
1273 * is pretty crude, but we are not expecting any variability in the file
1274 * format).
1275 */
1276 if (fscanf(lfp, "START WAL LOCATION: %X/%X (file %08X%16s)%c",
1277 &hi, &lo, &tli_from_walseg, startxlogfilename, &ch) != 5 || ch != '\n')
1278 ereport(FATAL,
1279 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1280 errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE)));
1281 RedoStartLSN = ((uint64) hi) << 32 | lo;
1282 RedoStartTLI = tli_from_walseg;
1283 if (fscanf(lfp, "CHECKPOINT LOCATION: %X/%X%c",
1284 &hi, &lo, &ch) != 3 || ch != '\n')
1285 ereport(FATAL,
1286 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1287 errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE)));
1288 *checkPointLoc = ((uint64) hi) << 32 | lo;
1289 *backupLabelTLI = tli_from_walseg;
1290
1291 /*
1292 * BACKUP METHOD lets us know if this was a typical backup ("streamed",
1293 * which could mean either pg_basebackup or the pg_backup_start/stop
1294 * method was used) or if this label came from somewhere else (the only
1295 * other option today being from pg_rewind). If this was a streamed
1296 * backup then we know that we need to play through until we get to the
1297 * end of the WAL which was generated during the backup (at which point we
1298 * will have reached consistency and backupEndRequired will be reset to be
1299 * false).
1300 */
1301 if (fscanf(lfp, "BACKUP METHOD: %19s\n", backuptype) == 1)
1302 {
1303 if (strcmp(backuptype, "streamed") == 0)
1304 *backupEndRequired = true;
1305 }
1306
1307 /*
1308 * BACKUP FROM lets us know if this was from a primary or a standby. If
1309 * it was from a standby, we'll double-check that the control file state
1310 * matches that of a standby.
1311 */
1312 if (fscanf(lfp, "BACKUP FROM: %19s\n", backupfrom) == 1)
1313 {
1314 if (strcmp(backupfrom, "standby") == 0)
1315 *backupFromStandby = true;
1316 }
1317
1318 /*
1319 * Parse START TIME and LABEL. Those are not mandatory fields for recovery
1320 * but checking for their presence is useful for debugging and the next
1321 * sanity checks. Cope also with the fact that the result buffers have a
1322 * pre-allocated size, hence if the backup_label file has been generated
1323 * with strings longer than the maximum assumed here an incorrect parsing
1324 * happens. That's fine as only minor consistency checks are done
1325 * afterwards.
1326 */
1327 if (fscanf(lfp, "START TIME: %127[^\n]\n", backuptime) == 1)
1328 ereport(DEBUG1,
1329 (errmsg_internal("backup time %s in file \"%s\"",
1330 backuptime, BACKUP_LABEL_FILE)));
1331
1332 if (fscanf(lfp, "LABEL: %1023[^\n]\n", backuplabel) == 1)
1333 ereport(DEBUG1,
1334 (errmsg_internal("backup label %s in file \"%s\"",
1335 backuplabel, BACKUP_LABEL_FILE)));
1336
1337 /*
1338 * START TIMELINE is new as of 11. Its parsing is not mandatory, still use
1339 * it as a sanity check if present.
1340 */
1341 if (fscanf(lfp, "START TIMELINE: %u\n", &tli_from_file) == 1)
1342 {
1343 if (tli_from_walseg != tli_from_file)
1344 ereport(FATAL,
1345 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1346 errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE),
1347 errdetail("Timeline ID parsed is %u, but expected %u.",
1348 tli_from_file, tli_from_walseg)));
1349
1350 ereport(DEBUG1,
1351 (errmsg_internal("backup timeline %u in file \"%s\"",
1352 tli_from_file, BACKUP_LABEL_FILE)));
1353 }
1354
1355 if (fscanf(lfp, "INCREMENTAL FROM LSN: %X/%X\n", &hi, &lo) > 0)
1356 ereport(FATAL,
1357 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1358 errmsg("this is an incremental backup, not a data directory"),
1359 errhint("Use pg_combinebackup to reconstruct a valid data directory.")));
1360
1361 if (ferror(lfp) || FreeFile(lfp))
1362 ereport(FATAL,
1363 (errcode_for_file_access(),
1364 errmsg("could not read file \"%s\": %m",
1365 BACKUP_LABEL_FILE)));
1366
1367 return true;
1368 }
1369
1370 /*
1371 * read_tablespace_map: check to see if a tablespace_map file is present
1372 *
1373 * If we see a tablespace_map file during recovery, we assume that we are
1374 * recovering from a backup dump file, and we therefore need to create symlinks
1375 * as per the information present in tablespace_map file.
1376 *
1377 * Returns true if a tablespace_map file was found (and fills *tablespaces
1378 * with a tablespaceinfo struct for each tablespace listed in the file);
1379 * returns false if not.
1380 */
1381 static bool
1382 read_tablespace_map(List **tablespaces)
1383 {
1384 tablespaceinfo *ti;
1385 FILE *lfp;
1386 char str[MAXPGPATH];
1387 int ch,
1388 i,
1389 n;
1390 bool was_backslash;
1391
1392 /*
1393 * See if tablespace_map file is present
1394 */
1395 lfp = AllocateFile(TABLESPACE_MAP, "r");
1396 if (!lfp)
1397 {
1398 if (errno != ENOENT)
1399 ereport(FATAL,
1400 (errcode_for_file_access(),
1401 errmsg("could not read file \"%s\": %m",
1402 TABLESPACE_MAP)));
1403 return false; /* it's not there, all is fine */
1404 }
1405
1406 /*
1407 * Read and parse the link name and path lines from tablespace_map file
1408 * (this code is pretty crude, but we are not expecting any variability in
1409 * the file format). De-escape any backslashes that were inserted.
1410 */
1411 i = 0;
1412 was_backslash = false;
1413 while ((ch = fgetc(lfp)) != EOF)
1414 {
1415 if (!was_backslash && (ch == '\n' || ch == '\r'))
1416 {
1417 char *endp;
1418
1419 if (i == 0)
1420 continue; /* \r immediately followed by \n */
1421
1422 /*
1423 * The de-escaped line should contain an OID followed by exactly
1424 * one space followed by a path. The path might start with
1425 * spaces, so don't be too liberal about parsing.
1426 */
1427 str[i] = '\0';
1428 n = 0;
1429 while (str[n] && str[n] != ' ')
1430 n++;
1431 if (n < 1 || n >= i - 1)
1432 ereport(FATAL,
1433 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1434 errmsg("invalid data in file \"%s\"", TABLESPACE_MAP)));
1435 str[n++] = '\0';
1436
1437 ti = palloc0(sizeof(tablespaceinfo));
1438 errno = 0;
1439 ti->oid = strtoul(str, &endp, 10);
1440 if (*endp != '\0' || errno == EINVAL || errno == ERANGE)
1441 ereport(FATAL,
1442 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1443 errmsg("invalid data in file \"%s\"", TABLESPACE_MAP)));
1444 ti->path = pstrdup(str + n);
1445 *tablespaces = lappend(*tablespaces, ti);
1446
1447 i = 0;
1448 continue;
1449 }
1450 else if (!was_backslash && ch == '\\')
1451 was_backslash = true;
1452 else
1453 {
1454 if (i < sizeof(str) - 1)
1455 str[i++] = ch;
1456 was_backslash = false;
1457 }
1458 }
1459
1460 if (i != 0 || was_backslash) /* last line not terminated? */
1461 ereport(FATAL,
1462 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1463 errmsg("invalid data in file \"%s\"", TABLESPACE_MAP)));
1464
1465 if (ferror(lfp) || FreeFile(lfp))
1466 ereport(FATAL,
1467 (errcode_for_file_access(),
1468 errmsg("could not read file \"%s\": %m",
1469 TABLESPACE_MAP)));
1470
1471 return true;
1472 }
1473
1474 /*
1475 * Finish WAL recovery.
1476 *
1477 * This does not close the 'xlogreader' yet, because in some cases the caller
1478 * still wants to re-read the last checkpoint record by calling
1479 * ReadCheckpointRecord().
1480 *
1481 * Returns the position of the last valid or applied record, after which new
1482 * WAL should be appended, information about why recovery was ended, and some
1483 * other things. See the EndOfWalRecoveryInfo struct for details.
1484 */
1485 EndOfWalRecoveryInfo *
1486 FinishWalRecovery(void)
1487 {
1488 EndOfWalRecoveryInfo *result = palloc(sizeof(EndOfWalRecoveryInfo));
1489 XLogRecPtr lastRec;
1490 TimeLineID lastRecTLI;
1491 XLogRecPtr endOfLog;
1492
1493 /*
1494 * Kill WAL receiver, if it's still running, before we continue to write
1495 * the startup checkpoint and aborted-contrecord records. It will trump
1496 * over these records and subsequent ones if it's still alive when we
1497 * start writing WAL.
1498 */
1499 XLogShutdownWalRcv();
1500
1501 /*
1502 * Shutdown the slot sync worker to drop any temporary slots acquired by
1503 * it and to prevent it from keep trying to fetch the failover slots.
1504 *
1505 * We do not update the 'synced' column in 'pg_replication_slots' system
1506 * view from true to false here, as any failed update could leave 'synced'
1507 * column false for some slots. This could cause issues during slot sync
1508 * after restarting the server as a standby. While updating the 'synced'
1509 * column after switching to the new timeline is an option, it does not
1510 * simplify the handling for the 'synced' column. Therefore, we retain the
1511 * 'synced' column as true after promotion as it may provide useful
1512 * information about the slot origin.
1513 */
1514 ShutDownSlotSync();
1515
1516 /*
1517 * We are now done reading the xlog from stream. Turn off streaming
1518 * recovery to force fetching the files (which would be required at end of
1519 * recovery, e.g., timeline history file) from archive or pg_wal.
1520 *
1521 * Note that standby mode must be turned off after killing WAL receiver,
1522 * i.e., calling XLogShutdownWalRcv().
1523 */
1524 Assert(!WalRcvStreaming());
1525 StandbyMode = false;
1526
1527 /*
1528 * Determine where to start writing WAL next.
1529 *
1530 * Re-fetch the last valid or last applied record, so we can identify the
1531 * exact endpoint of what we consider the valid portion of WAL. There may
1532 * be an incomplete continuation record after that, in which case
1533 * 'abortedRecPtr' and 'missingContrecPtr' are set and the caller will
1534 * write a special OVERWRITE_CONTRECORD message to mark that the rest of
1535 * it is intentionally missing. See CreateOverwriteContrecordRecord().
1536 *
1537 * An important side-effect of this is to load the last page into
1538 * xlogreader. The caller uses it to initialize the WAL for writing.
1539 */
1540 if (!InRecovery)
1541 {
1542 lastRec = CheckPointLoc;
1543 lastRecTLI = CheckPointTLI;
1544 }
1545 else
1546 {
1547 lastRec = XLogRecoveryCtl->lastReplayedReadRecPtr;
1548 lastRecTLI = XLogRecoveryCtl->lastReplayedTLI;
1549 }
1550 XLogPrefetcherBeginRead(xlogprefetcher, lastRec);
1551 (void) ReadRecord(xlogprefetcher, PANIC, false, lastRecTLI);
1552 endOfLog = xlogreader->EndRecPtr;
1553
1554 /*
1555 * Remember the TLI in the filename of the XLOG segment containing the
1556 * end-of-log. It could be different from the timeline that endOfLog
1557 * nominally belongs to, if there was a timeline switch in that segment,
1558 * and we were reading the old WAL from a segment belonging to a higher
1559 * timeline.
1560 */
1561 result->endOfLogTLI = xlogreader->seg.ws_tli;
1562
1563 if (ArchiveRecoveryRequested)
1564 {
1565 /*
1566 * We are no longer in archive recovery state.
1567 *
1568 * We are now done reading the old WAL. Turn off archive fetching if
1569 * it was active.
1570 */
1571 Assert(InArchiveRecovery);
1572 InArchiveRecovery = false;
1573
1574 /*
1575 * If the ending log segment is still open, close it (to avoid
1576 * problems on Windows with trying to rename or delete an open file).
1577 */
1578 if (readFile >= 0)
1579 {
1580 close(readFile);
1581 readFile = -1;
1582 }
1583 }
1584
1585 /*
1586 * Copy the last partial block to the caller, for initializing the WAL
1587 * buffer for appending new WAL.
1588 */
1589 if (endOfLog % XLOG_BLCKSZ != 0)
1590 {
1591 char *page;
1592 int len;
1593 XLogRecPtr pageBeginPtr;
1594
1595 pageBeginPtr = endOfLog - (endOfLog % XLOG_BLCKSZ);
1596 Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
1597
1598 /* Copy the valid part of the last block */
1599 len = endOfLog % XLOG_BLCKSZ;
1600 page = palloc(len);
1601 memcpy(page, xlogreader->readBuf, len);
1602
1603 result->lastPageBeginPtr = pageBeginPtr;
1604 result->lastPage = page;
1605 }
1606 else
1607 {
1608 /* There is no partial block to copy. */
1609 result->lastPageBeginPtr = endOfLog;
1610 result->lastPage = NULL;
1611 }
1612
1613 /*
1614 * Create a comment for the history file to explain why and where timeline
1615 * changed.
1616 */
1617 result->recoveryStopReason = getRecoveryStopReason();
1618
1619 result->lastRec = lastRec;
1620 result->lastRecTLI = lastRecTLI;
1621 result->endOfLog = endOfLog;
1622
1623 result->abortedRecPtr = abortedRecPtr;
1624 result->missingContrecPtr = missingContrecPtr;
1625
1626 result->standby_signal_file_found = standby_signal_file_found;
1627 result->recovery_signal_file_found = recovery_signal_file_found;
1628
1629 return result;
1630 }
1631
1632 /*
1633 * Clean up the WAL reader and leftovers from restoring WAL from archive
1634 */
1635 void
1636 ShutdownWalRecovery(void)
1637 {
1638 char recoveryPath[MAXPGPATH];
1639
1640 /* Final update of pg_stat_recovery_prefetch. */
1641 XLogPrefetcherComputeStats(xlogprefetcher);
1642
1643 /* Shut down xlogreader */
1644 if (readFile >= 0)
1645 {
1646 close(readFile);
1647 readFile = -1;
1648 }
1649 XLogReaderFree(xlogreader);
1650 XLogPrefetcherFree(xlogprefetcher);
1651
1652 if (ArchiveRecoveryRequested)
1653 {
1654 /*
1655 * Since there might be a partial WAL segment named RECOVERYXLOG, get
1656 * rid of it.
1657 */
1658 snprintf(recoveryPath, MAXPGPATH, XLOGDIR "/RECOVERYXLOG");
1659 unlink(recoveryPath); /* ignore any error */
1660
1661 /* Get rid of any remaining recovered timeline-history file, too */
1662 snprintf(recoveryPath, MAXPGPATH, XLOGDIR "/RECOVERYHISTORY");
1663 unlink(recoveryPath); /* ignore any error */
1664 }
1665
1666 /*
1667 * We don't need the latch anymore. It's not strictly necessary to disown
1668 * it, but let's do it for the sake of tidiness.
1669 */
1670 if (ArchiveRecoveryRequested)
1671 DisownLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
1672 }
1673
1674 /*
1675 * Perform WAL recovery.
1676 *
1677 * If the system was shut down cleanly, this is never called.
1678 */
1679 void
1680 PerformWalRecovery(void)
1681 {
1682 XLogRecord *record;
1683 bool reachedRecoveryTarget = false;
1684 TimeLineID replayTLI;
1685
1686 /*
1687 * Initialize shared variables for tracking progress of WAL replay, as if
1688 * we had just replayed the record before the REDO location (or the
1689 * checkpoint record itself, if it's a shutdown checkpoint).
1690 */
1691 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
1692 if (RedoStartLSN < CheckPointLoc)
1693 {
1694 XLogRecoveryCtl->lastReplayedReadRecPtr = InvalidXLogRecPtr;
1695 XLogRecoveryCtl->lastReplayedEndRecPtr = RedoStartLSN;
1696 XLogRecoveryCtl->lastReplayedTLI = RedoStartTLI;
1697 }
1698 else
1699 {
1700 XLogRecoveryCtl->lastReplayedReadRecPtr = xlogreader->ReadRecPtr;
1701 XLogRecoveryCtl->lastReplayedEndRecPtr = xlogreader->EndRecPtr;
1702 XLogRecoveryCtl->lastReplayedTLI = CheckPointTLI;
1703 }
1704 XLogRecoveryCtl->replayEndRecPtr = XLogRecoveryCtl->lastReplayedEndRecPtr;
1705 XLogRecoveryCtl->replayEndTLI = XLogRecoveryCtl->lastReplayedTLI;
1706 XLogRecoveryCtl->recoveryLastXTime = 0;
1707 XLogRecoveryCtl->currentChunkStartTime = 0;
1708 XLogRecoveryCtl->recoveryPauseState = RECOVERY_NOT_PAUSED;
1709 SpinLockRelease(&XLogRecoveryCtl->info_lck);
1710
1711 /* Also ensure XLogReceiptTime has a sane value */
1712 XLogReceiptTime = GetCurrentTimestamp();
1713
1714 /*
1715 * Let postmaster know we've started redo now, so that it can launch the
1716 * archiver if necessary.
1717 */
1718 if (IsUnderPostmaster)
1719 SendPostmasterSignal(PMSIGNAL_RECOVERY_STARTED);
1720
1721 /*
1722 * Allow read-only connections immediately if we're consistent already.
1723 */
1724 CheckRecoveryConsistency();
1725
1726 /*
1727 * Find the first record that logically follows the checkpoint --- it
1728 * might physically precede it, though.
1729 */
1730 if (RedoStartLSN < CheckPointLoc)
1731 {
1732 /* back up to find the record */
1733 replayTLI = RedoStartTLI;
1734 XLogPrefetcherBeginRead(xlogprefetcher, RedoStartLSN);
1735 record = ReadRecord(xlogprefetcher, PANIC, false, replayTLI);
1736
1737 /*
1738 * If a checkpoint record's redo pointer points back to an earlier
1739 * LSN, the record at that LSN should be an XLOG_CHECKPOINT_REDO
1740 * record.
1741 */
1742 if (record->xl_rmid != RM_XLOG_ID ||
1743 (record->xl_info & ~XLR_INFO_MASK) != XLOG_CHECKPOINT_REDO)
1744 ereport(FATAL,
1745 (errmsg("unexpected record type found at redo point %X/%X",
1746 LSN_FORMAT_ARGS(xlogreader->ReadRecPtr))));
1747 }
1748 else
1749 {
1750 /* just have to read next record after CheckPoint */
1751 Assert(xlogreader->ReadRecPtr == CheckPointLoc);
1752 replayTLI = CheckPointTLI;
1753 record = ReadRecord(xlogprefetcher, LOG, false, replayTLI);
1754 }
1755
1756 if (record != NULL)
1757 {
1758 TimestampTz xtime;
1759 PGRUsage ru0;
1760
1761 pg_rusage_init(&ru0);
1762
1763 InRedo = true;
1764
1765 RmgrStartup();
1766
1767 ereport(LOG,
1768 (errmsg("redo starts at %X/%X",
1769 LSN_FORMAT_ARGS(xlogreader->ReadRecPtr))));
1770
1771 /* Prepare to report progress of the redo phase. */
1772 if (!StandbyMode)
1773 begin_startup_progress_phase();
1774
1775 /*
1776 * main redo apply loop
1777 */
1778 do
1779 {
1780 if (!StandbyMode)
1781 ereport_startup_progress("redo in progress, elapsed time: %ld.%02d s, current LSN: %X/%X",
1782 LSN_FORMAT_ARGS(xlogreader->ReadRecPtr));
1783
1784 #ifdef WAL_DEBUG
1785 if (XLOG_DEBUG)
1786 {
1787 StringInfoData buf;
1788
1789 initStringInfo(&buf);
1790 appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ",
1791 LSN_FORMAT_ARGS(xlogreader->ReadRecPtr),
1792 LSN_FORMAT_ARGS(xlogreader->EndRecPtr));
1793 xlog_outrec(&buf, xlogreader);
1794 appendStringInfoString(&buf, " - ");
1795 xlog_outdesc(&buf, xlogreader);
1796 elog(LOG, "%s", buf.data);
1797 pfree(buf.data);
1798 }
1799 #endif
1800
1801 /* Handle interrupt signals of startup process */
1802 ProcessStartupProcInterrupts();
1803
1804 /*
1805 * Pause WAL replay, if requested by a hot-standby session via
1806 * SetRecoveryPause().
1807 *
1808 * Note that we intentionally don't take the info_lck spinlock
1809 * here. We might therefore read a slightly stale value of the
1810 * recoveryPause flag, but it can't be very stale (no worse than
1811 * the last spinlock we did acquire). Since a pause request is a
1812 * pretty asynchronous thing anyway, possibly responding to it one
1813 * WAL record later than we otherwise would is a minor issue, so
1814 * it doesn't seem worth adding another spinlock cycle to prevent
1815 * that.
1816 */
1817 if (((volatile XLogRecoveryCtlData *) XLogRecoveryCtl)->recoveryPauseState !=
1818 RECOVERY_NOT_PAUSED)
1819 recoveryPausesHere(false);
1820
1821 /*
1822 * Have we reached our recovery target?
1823 */
1824 if (recoveryStopsBefore(xlogreader))
1825 {
1826 reachedRecoveryTarget = true;
1827 break;
1828 }
1829
1830 /*
1831 * If we've been asked to lag the primary, wait on latch until
1832 * enough time has passed.
1833 */
1834 if (recoveryApplyDelay(xlogreader))
1835 {
1836 /*
1837 * We test for paused recovery again here. If user sets
1838 * delayed apply, it may be because they expect to pause
1839 * recovery in case of problems, so we must test again here
1840 * otherwise pausing during the delay-wait wouldn't work.
1841 */
1842 if (((volatile XLogRecoveryCtlData *) XLogRecoveryCtl)->recoveryPauseState !=
1843 RECOVERY_NOT_PAUSED)
1844 recoveryPausesHere(false);
1845 }
1846
1847 /*
1848 * Apply the record
1849 */
1850 ApplyWalRecord(xlogreader, record, &replayTLI);
1851
1852 /* Exit loop if we reached inclusive recovery target */
1853 if (recoveryStopsAfter(xlogreader))
1854 {
1855 reachedRecoveryTarget = true;
1856 break;
1857 }
1858
1859 /* Else, try to fetch the next WAL record */
1860 record = ReadRecord(xlogprefetcher, LOG, false, replayTLI);
1861 } while (record != NULL);
1862
1863 /*
1864 * end of main redo apply loop
1865 */
1866
1867 if (reachedRecoveryTarget)
1868 {
1869 if (!reachedConsistency)
1870 ereport(FATAL,
1871 (errmsg("requested recovery stop point is before consistent recovery point")));
1872
1873 /*
1874 * This is the last point where we can restart recovery with a new
1875 * recovery target, if we shutdown and begin again. After this,
1876 * Resource Managers may choose to do permanent corrective actions
1877 * at end of recovery.
1878 */
1879 switch (recoveryTargetAction)
1880 {
1881 case RECOVERY_TARGET_ACTION_SHUTDOWN:
1882
1883 /*
1884 * exit with special return code to request shutdown of
1885 * postmaster. Log messages issued from postmaster.
1886 */
1887 proc_exit(3);
1888
1889 case RECOVERY_TARGET_ACTION_PAUSE:
1890 SetRecoveryPause(true);
1891 recoveryPausesHere(true);
1892
1893 /* drop into promote */
1894
1895 case RECOVERY_TARGET_ACTION_PROMOTE:
1896 break;
1897 }
1898 }
1899
1900 RmgrCleanup();
1901
1902 ereport(LOG,
1903 (errmsg("redo done at %X/%X system usage: %s",
1904 LSN_FORMAT_ARGS(xlogreader->ReadRecPtr),
1905 pg_rusage_show(&ru0))));
1906 xtime = GetLatestXTime();
1907 if (xtime)
1908 ereport(LOG,
1909 (errmsg("last completed transaction was at log time %s",
1910 timestamptz_to_str(xtime))));
1911
1912 InRedo = false;
1913 }
1914 else
1915 {
1916 /* there are no WAL records following the checkpoint */
1917 ereport(LOG,
1918 (errmsg("redo is not required")));
1919 }
1920
1921 /*
1922 * This check is intentionally after the above log messages that indicate
1923 * how far recovery went.
1924 */
1925 if (ArchiveRecoveryRequested &&
1926 recoveryTarget != RECOVERY_TARGET_UNSET &&
1927 !reachedRecoveryTarget)
1928 ereport(FATAL,
1929 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1930 errmsg("recovery ended before configured recovery target was reached")));
1931 }
1932
1933 /*
1934 * Subroutine of PerformWalRecovery, to apply one WAL record.
1935 */
1936 static void
1937 ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI)
1938 {
1939 ErrorContextCallback errcallback;
1940 bool switchedTLI = false;
1941
1942 /* Setup error traceback support for ereport() */
1943 errcallback.callback = rm_redo_error_callback;
1944 errcallback.arg = xlogreader;
1945 errcallback.previous = error_context_stack;
1946 error_context_stack = &errcallback;
1947
1948 /*
1949 * TransamVariables->nextXid must be beyond record's xid.
1950 */
1951 AdvanceNextFullTransactionIdPastXid(record->xl_xid);
1952
1953 /*
1954 * Before replaying this record, check if this record causes the current
1955 * timeline to change. The record is already considered to be part of the
1956 * new timeline, so we update replayTLI before replaying it. That's
1957 * important so that replayEndTLI, which is recorded as the minimum
1958 * recovery point's TLI if recovery stops after this record, is set
1959 * correctly.
1960 */
1961 if (record->xl_rmid == RM_XLOG_ID)
1962 {
1963 TimeLineID newReplayTLI = *replayTLI;
1964 TimeLineID prevReplayTLI = *replayTLI;
1965 uint8 info = record->xl_info & ~XLR_INFO_MASK;
1966
1967 if (info == XLOG_CHECKPOINT_SHUTDOWN)
1968 {
1969 CheckPoint checkPoint;
1970
1971 memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
1972 newReplayTLI = checkPoint.ThisTimeLineID;
1973 prevReplayTLI = checkPoint.PrevTimeLineID;
1974 }
1975 else if (info == XLOG_END_OF_RECOVERY)
1976 {
1977 xl_end_of_recovery xlrec;
1978
1979 memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery));
1980 newReplayTLI = xlrec.ThisTimeLineID;
1981 prevReplayTLI = xlrec.PrevTimeLineID;
1982 }
1983
1984 if (newReplayTLI != *replayTLI)
1985 {
1986 /* Check that it's OK to switch to this TLI */
1987 checkTimeLineSwitch(xlogreader->EndRecPtr,
1988 newReplayTLI, prevReplayTLI, *replayTLI);
1989
1990 /* Following WAL records should be run with new TLI */
1991 *replayTLI = newReplayTLI;
1992 switchedTLI = true;
1993 }
1994 }
1995
1996 /*
1997 * Update shared replayEndRecPtr before replaying this record, so that
1998 * XLogFlush will update minRecoveryPoint correctly.
1999 */
2000 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
2001 XLogRecoveryCtl->replayEndRecPtr = xlogreader->EndRecPtr;
2002 XLogRecoveryCtl->replayEndTLI = *replayTLI;
2003 SpinLockRelease(&XLogRecoveryCtl->info_lck);
2004
2005 /*
2006 * If we are attempting to enter Hot Standby mode, process XIDs we see
2007 */
2008 if (standbyState >= STANDBY_INITIALIZED &&
2009 TransactionIdIsValid(record->xl_xid))
2010 RecordKnownAssignedTransactionIds(record->xl_xid);
2011
2012 /*
2013 * Some XLOG record types that are related to recovery are processed
2014 * directly here, rather than in xlog_redo()
2015 */
2016 if (record->xl_rmid == RM_XLOG_ID)
2017 xlogrecovery_redo(xlogreader, *replayTLI);
2018
2019 /* Now apply the WAL record itself */
2020 GetRmgr(record->xl_rmid).rm_redo(xlogreader);
2021
2022 /*
2023 * After redo, check whether the backup pages associated with the WAL
2024 * record are consistent with the existing pages. This check is done only
2025 * if consistency check is enabled for this record.
2026 */
2027 if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0)
2028 verifyBackupPageConsistency(xlogreader);
2029
2030 /* Pop the error context stack */
2031 error_context_stack = errcallback.previous;
2032
2033 /*
2034 * Update lastReplayedEndRecPtr after this record has been successfully
2035 * replayed.
2036 */
2037 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
2038 XLogRecoveryCtl->lastReplayedReadRecPtr = xlogreader->ReadRecPtr;
2039 XLogRecoveryCtl->lastReplayedEndRecPtr = xlogreader->EndRecPtr;
2040 XLogRecoveryCtl->lastReplayedTLI = *replayTLI;
2041 SpinLockRelease(&XLogRecoveryCtl->info_lck);
2042
2043 /* ------
2044 * Wakeup walsenders:
2045 *
2046 * On the standby, the WAL is flushed first (which will only wake up
2047 * physical walsenders) and then applied, which will only wake up logical
2048 * walsenders.
2049 *
2050 * Indeed, logical walsenders on standby can't decode and send data until
2051 * it's been applied.
2052 *
2053 * Physical walsenders don't need to be woken up during replay unless
2054 * cascading replication is allowed and time line change occurred (so that
2055 * they can notice that they are on a new time line).
2056 *
2057 * That's why the wake up conditions are for:
2058 *
2059 * - physical walsenders in case of new time line and cascade
2060 * replication is allowed
2061 * - logical walsenders in case cascade replication is allowed (could not
2062 * be created otherwise)
2063 * ------
2064 */
2065 if (AllowCascadeReplication())
2066 WalSndWakeup(switchedTLI, true);
2067
2068 /*
2069 * If rm_redo called XLogRequestWalReceiverReply, then we wake up the
2070 * receiver so that it notices the updated lastReplayedEndRecPtr and sends
2071 * a reply to the primary.
2072 */
2073 if (doRequestWalReceiverReply)
2074 {
2075 doRequestWalReceiverReply = false;
2076 WalRcvForceReply();
2077 }
2078
2079 /* Allow read-only connections if we're consistent now */
2080 CheckRecoveryConsistency();
2081
2082 /* Is this a timeline switch? */
2083 if (switchedTLI)
2084 {
2085 /*
2086 * Before we continue on the new timeline, clean up any (possibly
2087 * bogus) future WAL segments on the old timeline.
2088 */
2089 RemoveNonParentXlogFiles(xlogreader->EndRecPtr, *replayTLI);
2090
2091 /* Reset the prefetcher. */
2092 XLogPrefetchReconfigure();
2093 }
2094 }
2095
2096 /*
2097 * Some XLOG RM record types that are directly related to WAL recovery are
2098 * handled here rather than in the xlog_redo()
2099 */
2100 static void
2101 xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
2102 {
2103 uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
2104 XLogRecPtr lsn = record->EndRecPtr;
2105
2106 Assert(XLogRecGetRmid(record) == RM_XLOG_ID);
2107
2108 if (info == XLOG_OVERWRITE_CONTRECORD)
2109 {
2110 /* Verify the payload of a XLOG_OVERWRITE_CONTRECORD record. */
2111 xl_overwrite_contrecord xlrec;
2112
2113 memcpy(&xlrec, XLogRecGetData(record), sizeof(xl_overwrite_contrecord));
2114 if (xlrec.overwritten_lsn != record->overwrittenRecPtr)
2115 elog(FATAL, "mismatching overwritten LSN %X/%X -> %X/%X",
2116 LSN_FORMAT_ARGS(xlrec.overwritten_lsn),
2117 LSN_FORMAT_ARGS(record->overwrittenRecPtr));
2118
2119 /* We have safely skipped the aborted record */
2120 abortedRecPtr = InvalidXLogRecPtr;
2121 missingContrecPtr = InvalidXLogRecPtr;
2122
2123 ereport(LOG,
2124 (errmsg("successfully skipped missing contrecord at %X/%X, overwritten at %s",
2125 LSN_FORMAT_ARGS(xlrec.overwritten_lsn),
2126 timestamptz_to_str(xlrec.overwrite_time))));
2127
2128 /* Verifying the record should only happen once */
2129 record->overwrittenRecPtr = InvalidXLogRecPtr;
2130 }
2131 else if (info == XLOG_BACKUP_END)
2132 {
2133 XLogRecPtr startpoint;
2134
2135 memcpy(&startpoint, XLogRecGetData(record), sizeof(startpoint));
2136
2137 if (backupStartPoint == startpoint)
2138 {
2139 /*
2140 * We have reached the end of base backup, the point where
2141 * pg_backup_stop() was done. The data on disk is now consistent
2142 * (assuming we have also reached minRecoveryPoint). Set
2143 * backupEndPoint to the current LSN, so that the next call to
2144 * CheckRecoveryConsistency() will notice it and do the
2145 * end-of-backup processing.
2146 */
2147 elog(DEBUG1, "end of backup record reached");
2148
2149 backupEndPoint = lsn;
2150 }
2151 else
2152 elog(DEBUG1, "saw end-of-backup record for backup starting at %X/%X, waiting for %X/%X",
2153 LSN_FORMAT_ARGS(startpoint), LSN_FORMAT_ARGS(backupStartPoint));
2154 }
2155 }
2156
2157 /*
2158 * Verify that, in non-test mode, ./pg_tblspc doesn't contain any real
2159 * directories.
2160 *
2161 * Replay of database creation XLOG records for databases that were later
2162 * dropped can create fake directories in pg_tblspc. By the time consistency
2163 * is reached these directories should have been removed; here we verify
2164 * that this did indeed happen. This is to be called at the point where
2165 * consistent state is reached.
2166 *
2167 * allow_in_place_tablespaces turns the PANIC into a WARNING, which is
2168 * useful for testing purposes, and also allows for an escape hatch in case
2169 * things go south.
2170 */
2171 static void
2172 CheckTablespaceDirectory(void)
2173 {
2174 DIR *dir;
2175 struct dirent *de;
2176
2177 dir = AllocateDir(PG_TBLSPC_DIR);
2178 while ((de = ReadDir(dir, PG_TBLSPC_DIR)) != NULL)
2179 {
2180 char path[MAXPGPATH + sizeof(PG_TBLSPC_DIR)];
2181
2182 /* Skip entries of non-oid names */
2183 if (strspn(de->d_name, "0123456789") != strlen(de->d_name))
2184 continue;
2185
2186 snprintf(path, sizeof(path), "%s/%s", PG_TBLSPC_DIR, de->d_name);
2187
2188 if (get_dirent_type(path, de, false, ERROR) != PGFILETYPE_LNK)
2189 ereport(allow_in_place_tablespaces ? WARNING : PANIC,
2190 (errcode(ERRCODE_DATA_CORRUPTED),
2191 errmsg("unexpected directory entry \"%s\" found in %s",
2192 de->d_name, PG_TBLSPC_DIR),
2193 errdetail("All directory entries in %s/ should be symbolic links.",
2194 PG_TBLSPC_DIR),
2195 errhint("Remove those directories, or set \"allow_in_place_tablespaces\" to ON transiently to let recovery complete.")));
2196 }
2197 }
2198
2199 /*
2200 * Checks if recovery has reached a consistent state. When consistency is
2201 * reached and we have a valid starting standby snapshot, tell postmaster
2202 * that it can start accepting read-only connections.
2203 */
2204 static void
2205 CheckRecoveryConsistency(void)
2206 {
2207 XLogRecPtr lastReplayedEndRecPtr;
2208 TimeLineID lastReplayedTLI;
2209
2210 /*
2211 * During crash recovery, we don't reach a consistent state until we've
2212 * replayed all the WAL.
2213 */
2214 if (XLogRecPtrIsInvalid(minRecoveryPoint))
2215 return;
2216
2217 Assert(InArchiveRecovery);
2218
2219 /*
2220 * assume that we are called in the startup process, and hence don't need
2221 * a lock to read lastReplayedEndRecPtr
2222 */
2223 lastReplayedEndRecPtr = XLogRecoveryCtl->lastReplayedEndRecPtr;
2224 lastReplayedTLI = XLogRecoveryCtl->lastReplayedTLI;
2225
2226 /*
2227 * Have we reached the point where our base backup was completed?
2228 */
2229 if (!XLogRecPtrIsInvalid(backupEndPoint) &&
2230 backupEndPoint <= lastReplayedEndRecPtr)
2231 {
2232 XLogRecPtr saveBackupStartPoint = backupStartPoint;
2233 XLogRecPtr saveBackupEndPoint = backupEndPoint;
2234
2235 elog(DEBUG1, "end of backup reached");
2236
2237 /*
2238 * We have reached the end of base backup, as indicated by pg_control.
2239 * Update the control file accordingly.
2240 */
2241 ReachedEndOfBackup(lastReplayedEndRecPtr, lastReplayedTLI);
2242 backupStartPoint = InvalidXLogRecPtr;
2243 backupEndPoint = InvalidXLogRecPtr;
2244 backupEndRequired = false;
2245
2246 ereport(LOG,
2247 (errmsg("completed backup recovery with redo LSN %X/%X and end LSN %X/%X",
2248 LSN_FORMAT_ARGS(saveBackupStartPoint),
2249 LSN_FORMAT_ARGS(saveBackupEndPoint))));
2250 }
2251
2252 /*
2253 * Have we passed our safe starting point? Note that minRecoveryPoint is
2254 * known to be incorrectly set if recovering from a backup, until the
2255 * XLOG_BACKUP_END arrives to advise us of the correct minRecoveryPoint.
2256 * All we know prior to that is that we're not consistent yet.
2257 */
2258 if (!reachedConsistency && !backupEndRequired &&
2259 minRecoveryPoint <= lastReplayedEndRecPtr)
2260 {
2261 /*
2262 * Check to see if the XLOG sequence contained any unresolved
2263 * references to uninitialized pages.
2264 */
2265 XLogCheckInvalidPages();
2266
2267 /*
2268 * Check that pg_tblspc doesn't contain any real directories. Replay
2269 * of Database/CREATE_* records may have created fictitious tablespace
2270 * directories that should have been removed by the time consistency
2271 * was reached.
2272 */
2273 CheckTablespaceDirectory();
2274
2275 reachedConsistency = true;
2276 SendPostmasterSignal(PMSIGNAL_RECOVERY_CONSISTENT);
2277 ereport(LOG,
2278 (errmsg("consistent recovery state reached at %X/%X",
2279 LSN_FORMAT_ARGS(lastReplayedEndRecPtr))));
2280 }
2281
2282 /*
2283 * Have we got a valid starting snapshot that will allow queries to be
2284 * run? If so, we can tell postmaster that the database is consistent now,
2285 * enabling connections.
2286 */
2287 if (standbyState == STANDBY_SNAPSHOT_READY &&
2288 !LocalHotStandbyActive &&
2289 reachedConsistency &&
2290 IsUnderPostmaster)
2291 {
2292 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
2293 XLogRecoveryCtl->SharedHotStandbyActive = true;
2294 SpinLockRelease(&XLogRecoveryCtl->info_lck);
2295
2296 LocalHotStandbyActive = true;
2297
2298 SendPostmasterSignal(PMSIGNAL_BEGIN_HOT_STANDBY);
2299 }
2300 }
2301
2302 /*
2303 * Error context callback for errors occurring during rm_redo().
2304 */
2305 static void
2306 rm_redo_error_callback(void *arg)
2307 {
2308 XLogReaderState *record = (XLogReaderState *) arg;
2309 StringInfoData buf;
2310
2311 initStringInfo(&buf);
2312 xlog_outdesc(&buf, record);
2313 xlog_block_info(&buf, record);
2314
2315 /* translator: %s is a WAL record description */
2316 errcontext("WAL redo at %X/%X for %s",
2317 LSN_FORMAT_ARGS(record->ReadRecPtr),
2318 buf.data);
2319
2320 pfree(buf.data);
2321 }
2322
2323 /*
2324 * Returns a string describing an XLogRecord, consisting of its identity
2325 * optionally followed by a colon, a space, and a further description.
2326 */
2327 void
2328 xlog_outdesc(StringInfo buf, XLogReaderState *record)
2329 {
2330 RmgrData rmgr = GetRmgr(XLogRecGetRmid(record));
2331 uint8 info = XLogRecGetInfo(record);
2332 const char *id;
2333
2334 appendStringInfoString(buf, rmgr.rm_name);
2335 appendStringInfoChar(buf, '/');
2336
2337 id = rmgr.rm_identify(info);
2338 if (id == NULL)
2339 appendStringInfo(buf, "UNKNOWN (%X): ", info & ~XLR_INFO_MASK);
2340 else
2341 appendStringInfo(buf, "%s: ", id);
2342
2343 rmgr.rm_desc(buf, record);
2344 }
2345
2346 #ifdef WAL_DEBUG
2347
2348 static void
2349 xlog_outrec(StringInfo buf, XLogReaderState *record)
2350 {
2351 appendStringInfo(buf, "prev %X/%X; xid %u",
2352 LSN_FORMAT_ARGS(XLogRecGetPrev(record)),
2353 XLogRecGetXid(record));
2354
2355 appendStringInfo(buf, "; len %u",
2356 XLogRecGetDataLen(record));
2357
2358 xlog_block_info(buf, record);
2359 }
2360 #endif /* WAL_DEBUG */
2361
2362 /*
2363 * Returns a string giving information about all the blocks in an
2364 * XLogRecord.
2365 */
2366 static void
2367 xlog_block_info(StringInfo buf, XLogReaderState *record)
2368 {
2369 int block_id;
2370
2371 /* decode block references */
2372 for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
2373 {
2374 RelFileLocator rlocator;
2375 ForkNumber forknum;
2376 BlockNumber blk;
2377
2378 if (!XLogRecGetBlockTagExtended(record, block_id,
2379 &rlocator, &forknum, &blk, NULL))
2380 continue;
2381
2382 if (forknum != MAIN_FORKNUM)
2383 appendStringInfo(buf, "; blkref #%d: rel %u/%u/%u, fork %u, blk %u",
2384 block_id,
2385 rlocator.spcOid, rlocator.dbOid,
2386 rlocator.relNumber,
2387 forknum,
2388 blk);
2389 else
2390 appendStringInfo(buf, "; blkref #%d: rel %u/%u/%u, blk %u",
2391 block_id,
2392 rlocator.spcOid, rlocator.dbOid,
2393 rlocator.relNumber,
2394 blk);
2395 if (XLogRecHasBlockImage(record, block_id))
2396 appendStringInfoString(buf, " FPW");
2397 }
2398 }
2399
2400
2401 /*
2402 * Check that it's OK to switch to new timeline during recovery.
2403 *
2404 * 'lsn' is the address of the shutdown checkpoint record we're about to
2405 * replay. (Currently, timeline can only change at a shutdown checkpoint).
2406 */
2407 static void
2408 checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI, TimeLineID prevTLI,
2409 TimeLineID replayTLI)
2410 {
2411 /* Check that the record agrees on what the current (old) timeline is */
2412 if (prevTLI != replayTLI)
2413 ereport(PANIC,
2414 (errmsg("unexpected previous timeline ID %u (current timeline ID %u) in checkpoint record",
2415 prevTLI, replayTLI)));
2416
2417 /*
2418 * The new timeline better be in the list of timelines we expect to see,
2419 * according to the timeline history. It should also not decrease.
2420 */
2421 if (newTLI < replayTLI || !tliInHistory(newTLI, expectedTLEs))
2422 ereport(PANIC,
2423 (errmsg("unexpected timeline ID %u (after %u) in checkpoint record",
2424 newTLI, replayTLI)));
2425
2426 /*
2427 * If we have not yet reached min recovery point, and we're about to
2428 * switch to a timeline greater than the timeline of the min recovery
2429 * point: trouble. After switching to the new timeline, we could not
2430 * possibly visit the min recovery point on the correct timeline anymore.
2431 * This can happen if there is a newer timeline in the archive that
2432 * branched before the timeline the min recovery point is on, and you
2433 * attempt to do PITR to the new timeline.
2434 */
2435 if (!XLogRecPtrIsInvalid(minRecoveryPoint) &&
2436 lsn < minRecoveryPoint &&
2437 newTLI > minRecoveryPointTLI)
2438 ereport(PANIC,
2439 (errmsg("unexpected timeline ID %u in checkpoint record, before reaching minimum recovery point %X/%X on timeline %u",
2440 newTLI,
2441 LSN_FORMAT_ARGS(minRecoveryPoint),
2442 minRecoveryPointTLI)));
2443
2444 /* Looks good */
2445 }
2446
2447
2448 /*
2449 * Extract timestamp from WAL record.
2450 *
2451 * If the record contains a timestamp, returns true, and saves the timestamp
2452 * in *recordXtime. If the record type has no timestamp, returns false.
2453 * Currently, only transaction commit/abort records and restore points contain
2454 * timestamps.
2455 */
2456 static bool
2457 getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime)
2458 {
2459 uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
2460 uint8 xact_info = info & XLOG_XACT_OPMASK;
2461 uint8 rmid = XLogRecGetRmid(record);
2462
2463 if (rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT)
2464 {
2465 *recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time;
2466 return true;
2467 }
2468 if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT ||
2469 xact_info == XLOG_XACT_COMMIT_PREPARED))
2470 {
2471 *recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time;
2472 return true;
2473 }
2474 if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT ||
2475 xact_info == XLOG_XACT_ABORT_PREPARED))
2476 {
2477 *recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time;
2478 return true;
2479 }
2480 return false;
2481 }
2482
2483 /*
2484 * Checks whether the current buffer page and backup page stored in the
2485 * WAL record are consistent or not. Before comparing the two pages, a
2486 * masking can be applied to the pages to ignore certain areas like hint bits,
2487 * unused space between pd_lower and pd_upper among other things. This
2488 * function should be called once WAL replay has been completed for a
2489 * given record.
2490 */
2491 static void
2492 verifyBackupPageConsistency(XLogReaderState *record)
2493 {
2494 RmgrData rmgr = GetRmgr(XLogRecGetRmid(record));
2495 RelFileLocator rlocator;
2496 ForkNumber forknum;
2497 BlockNumber blkno;
2498 int block_id;
2499
2500 /* Records with no backup blocks have no need for consistency checks. */
2501 if (!XLogRecHasAnyBlockRefs(record))
2502 return;
2503
2504 Assert((XLogRecGetInfo(record) & XLR_CHECK_CONSISTENCY) != 0);
2505
2506 for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
2507 {
2508 Buffer buf;
2509 Page page;
2510
2511 if (!XLogRecGetBlockTagExtended(record, block_id,
2512 &rlocator, &forknum, &blkno, NULL))
2513 {
2514 /*
2515 * WAL record doesn't contain a block reference with the given id.
2516 * Do nothing.
2517 */
2518 continue;
2519 }
2520
2521 Assert(XLogRecHasBlockImage(record, block_id));
2522
2523 if (XLogRecBlockImageApply(record, block_id))
2524 {
2525 /*
2526 * WAL record has already applied the page, so bypass the
2527 * consistency check as that would result in comparing the full
2528 * page stored in the record with itself.
2529 */
2530 continue;
2531 }
2532
2533 /*
2534 * Read the contents from the current buffer and store it in a
2535 * temporary page.
2536 */
2537 buf = XLogReadBufferExtended(rlocator, forknum, blkno,
2538 RBM_NORMAL_NO_LOG,
2539 InvalidBuffer);
2540 if (!BufferIsValid(buf))
2541 continue;
2542
2543 LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
2544 page = BufferGetPage(buf);
2545
2546 /*
2547 * Take a copy of the local page where WAL has been applied to have a
2548 * comparison base before masking it...
2549 */
2550 memcpy(replay_image_masked, page, BLCKSZ);
2551
2552 /* No need for this page anymore now that a copy is in. */
2553 UnlockReleaseBuffer(buf);
2554
2555 /*
2556 * If the block LSN is already ahead of this WAL record, we can't
2557 * expect contents to match. This can happen if recovery is
2558 * restarted.
2559 */
2560 if (PageGetLSN(replay_image_masked) > record->EndRecPtr)
2561 continue;
2562
2563 /*
2564 * Read the contents from the backup copy, stored in WAL record and
2565 * store it in a temporary page. There is no need to allocate a new
2566 * page here, a local buffer is fine to hold its contents and a mask
2567 * can be directly applied on it.
2568 */
2569 if (!RestoreBlockImage(record, block_id, primary_image_masked))
2570 ereport(ERROR,
2571 (errcode(ERRCODE_INTERNAL_ERROR),
2572 errmsg_internal("%s", record->errormsg_buf)));
2573
2574 /*
2575 * If masking function is defined, mask both the primary and replay
2576 * images
2577 */
2578 if (rmgr.rm_mask != NULL)
2579 {
2580 rmgr.rm_mask(replay_image_masked, blkno);
2581 rmgr.rm_mask(primary_image_masked, blkno);
2582 }
2583
2584 /* Time to compare the primary and replay images. */
2585 if (memcmp(replay_image_masked, primary_image_masked, BLCKSZ) != 0)
2586 {
2587 elog(FATAL,
2588 "inconsistent page found, rel %u/%u/%u, forknum %u, blkno %u",
2589 rlocator.spcOid, rlocator.dbOid, rlocator.relNumber,
2590 forknum, blkno);
2591 }
2592 }
2593 }
2594
2595 /*
2596 * For point-in-time recovery, this function decides whether we want to
2597 * stop applying the XLOG before the current record.
2598 *
2599 * Returns true if we are stopping, false otherwise. If stopping, some
2600 * information is saved in recoveryStopXid et al for use in annotating the
2601 * new timeline's history file.
2602 */
2603 static bool
2604 recoveryStopsBefore(XLogReaderState *record)
2605 {
2606 bool stopsHere = false;
2607 uint8 xact_info;
2608 bool isCommit;
2609 TimestampTz recordXtime = 0;
2610 TransactionId recordXid;
2611
2612 /*
2613 * Ignore recovery target settings when not in archive recovery (meaning
2614 * we are in crash recovery).
2615 */
2616 if (!ArchiveRecoveryRequested)
2617 return false;
2618
2619 /* Check if we should stop as soon as reaching consistency */
2620 if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE && reachedConsistency)
2621 {
2622 ereport(LOG,
2623 (errmsg("recovery stopping after reaching consistency")));
2624
2625 recoveryStopAfter = false;
2626 recoveryStopXid = InvalidTransactionId;
2627 recoveryStopLSN = InvalidXLogRecPtr;
2628 recoveryStopTime = 0;
2629 recoveryStopName[0] = '\0';
2630 return true;
2631 }
2632
2633 /* Check if target LSN has been reached */
2634 if (recoveryTarget == RECOVERY_TARGET_LSN &&
2635 !recoveryTargetInclusive &&
2636 record->ReadRecPtr >= recoveryTargetLSN)
2637 {
2638 recoveryStopAfter = false;
2639 recoveryStopXid = InvalidTransactionId;
2640 recoveryStopLSN = record->ReadRecPtr;
2641 recoveryStopTime = 0;
2642 recoveryStopName[0] = '\0';
2643 ereport(LOG,
2644 (errmsg("recovery stopping before WAL location (LSN) \"%X/%X\"",
2645 LSN_FORMAT_ARGS(recoveryStopLSN))));
2646 return true;
2647 }
2648
2649 /* Otherwise we only consider stopping before COMMIT or ABORT records. */
2650 if (XLogRecGetRmid(record) != RM_XACT_ID)
2651 return false;
2652
2653 xact_info = XLogRecGetInfo(record) & XLOG_XACT_OPMASK;
2654
2655 if (xact_info == XLOG_XACT_COMMIT)
2656 {
2657 isCommit = true;
2658 recordXid = XLogRecGetXid(record);
2659 }
2660 else if (xact_info == XLOG_XACT_COMMIT_PREPARED)
2661 {
2662 xl_xact_commit *xlrec = (xl_xact_commit *) XLogRecGetData(record);
2663 xl_xact_parsed_commit parsed;
2664
2665 isCommit = true;
2666 ParseCommitRecord(XLogRecGetInfo(record),
2667 xlrec,
2668 &parsed);
2669 recordXid = parsed.twophase_xid;
2670 }
2671 else if (xact_info == XLOG_XACT_ABORT)
2672 {
2673 isCommit = false;
2674 recordXid = XLogRecGetXid(record);
2675 }
2676 else if (xact_info == XLOG_XACT_ABORT_PREPARED)
2677 {
2678 xl_xact_abort *xlrec = (xl_xact_abort *) XLogRecGetData(record);
2679 xl_xact_parsed_abort parsed;
2680
2681 isCommit = false;
2682 ParseAbortRecord(XLogRecGetInfo(record),
2683 xlrec,
2684 &parsed);
2685 recordXid = parsed.twophase_xid;
2686 }
2687 else
2688 return false;
2689
2690 if (recoveryTarget == RECOVERY_TARGET_XID && !recoveryTargetInclusive)
2691 {
2692 /*
2693 * There can be only one transaction end record with this exact
2694 * transactionid
2695 *
2696 * when testing for an xid, we MUST test for equality only, since
2697 * transactions are numbered in the order they start, not the order
2698 * they complete. A higher numbered xid will complete before you about
2699 * 50% of the time...
2700 */
2701 stopsHere = (recordXid == recoveryTargetXid);
2702 }
2703
2704 /*
2705 * Note: we must fetch recordXtime regardless of recoveryTarget setting.
2706 * We don't expect getRecordTimestamp ever to fail, since we already know
2707 * this is a commit or abort record; but test its result anyway.
2708 */
2709 if (getRecordTimestamp(record, &recordXtime) &&
2710 recoveryTarget == RECOVERY_TARGET_TIME)
2711 {
2712 /*
2713 * There can be many transactions that share the same commit time, so
2714 * we stop after the last one, if we are inclusive, or stop at the
2715 * first one if we are exclusive
2716 */
2717 if (recoveryTargetInclusive)
2718 stopsHere = (recordXtime > recoveryTargetTime);
2719 else
2720 stopsHere = (recordXtime >= recoveryTargetTime);
2721 }
2722
2723 if (stopsHere)
2724 {
2725 recoveryStopAfter = false;
2726 recoveryStopXid = recordXid;
2727 recoveryStopTime = recordXtime;
2728 recoveryStopLSN = InvalidXLogRecPtr;
2729 recoveryStopName[0] = '\0';
2730
2731 if (isCommit)
2732 {
2733 ereport(LOG,
2734 (errmsg("recovery stopping before commit of transaction %u, time %s",
2735 recoveryStopXid,
2736 timestamptz_to_str(recoveryStopTime))));
2737 }
2738 else
2739 {
2740 ereport(LOG,
2741 (errmsg("recovery stopping before abort of transaction %u, time %s",
2742 recoveryStopXid,
2743 timestamptz_to_str(recoveryStopTime))));
2744 }
2745 }
2746
2747 return stopsHere;
2748 }
2749
2750 /*
2751 * Same as recoveryStopsBefore, but called after applying the record.
2752 *
2753 * We also track the timestamp of the latest applied COMMIT/ABORT
2754 * record in XLogRecoveryCtl->recoveryLastXTime.
2755 */
2756 static bool
2757 recoveryStopsAfter(XLogReaderState *record)
2758 {
2759 uint8 info;
2760 uint8 xact_info;
2761 uint8 rmid;
2762 TimestampTz recordXtime = 0;
2763
2764 /*
2765 * Ignore recovery target settings when not in archive recovery (meaning
2766 * we are in crash recovery).
2767 */
2768 if (!ArchiveRecoveryRequested)
2769 return false;
2770
2771 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
2772 rmid = XLogRecGetRmid(record);
2773
2774 /*
2775 * There can be many restore points that share the same name; we stop at
2776 * the first one.
2777 */
2778 if (recoveryTarget == RECOVERY_TARGET_NAME &&
2779 rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT)
2780 {
2781 xl_restore_point *recordRestorePointData;
2782
2783 recordRestorePointData = (xl_restore_point *) XLogRecGetData(record);
2784
2785 if (strcmp(recordRestorePointData->rp_name, recoveryTargetName) == 0)
2786 {
2787 recoveryStopAfter = true;
2788 recoveryStopXid = InvalidTransactionId;
2789 recoveryStopLSN = InvalidXLogRecPtr;
2790 (void) getRecordTimestamp(record, &recoveryStopTime);
2791 strlcpy(recoveryStopName, recordRestorePointData->rp_name, MAXFNAMELEN);
2792
2793 ereport(LOG,
2794 (errmsg("recovery stopping at restore point \"%s\", time %s",
2795 recoveryStopName,
2796 timestamptz_to_str(recoveryStopTime))));
2797 return true;
2798 }
2799 }
2800
2801 /* Check if the target LSN has been reached */
2802 if (recoveryTarget == RECOVERY_TARGET_LSN &&
2803 recoveryTargetInclusive &&
2804 record->ReadRecPtr >= recoveryTargetLSN)
2805 {
2806 recoveryStopAfter = true;
2807 recoveryStopXid = InvalidTransactionId;
2808 recoveryStopLSN = record->ReadRecPtr;
2809 recoveryStopTime = 0;
2810 recoveryStopName[0] = '\0';
2811 ereport(LOG,
2812 (errmsg("recovery stopping after WAL location (LSN) \"%X/%X\"",
2813 LSN_FORMAT_ARGS(recoveryStopLSN))));
2814 return true;
2815 }
2816
2817 if (rmid != RM_XACT_ID)
2818 return false;
2819
2820 xact_info = info & XLOG_XACT_OPMASK;
2821
2822 if (xact_info == XLOG_XACT_COMMIT ||
2823 xact_info == XLOG_XACT_COMMIT_PREPARED ||
2824 xact_info == XLOG_XACT_ABORT ||
2825 xact_info == XLOG_XACT_ABORT_PREPARED)
2826 {
2827 TransactionId recordXid;
2828
2829 /* Update the last applied transaction timestamp */
2830 if (getRecordTimestamp(record, &recordXtime))
2831 SetLatestXTime(recordXtime);
2832
2833 /* Extract the XID of the committed/aborted transaction */
2834 if (xact_info == XLOG_XACT_COMMIT_PREPARED)
2835 {
2836 xl_xact_commit *xlrec = (xl_xact_commit *) XLogRecGetData(record);
2837 xl_xact_parsed_commit parsed;
2838
2839 ParseCommitRecord(XLogRecGetInfo(record),
2840 xlrec,
2841 &parsed);
2842 recordXid = parsed.twophase_xid;
2843 }
2844 else if (xact_info == XLOG_XACT_ABORT_PREPARED)
2845 {
2846 xl_xact_abort *xlrec = (xl_xact_abort *) XLogRecGetData(record);
2847 xl_xact_parsed_abort parsed;
2848
2849 ParseAbortRecord(XLogRecGetInfo(record),
2850 xlrec,
2851 &parsed);
2852 recordXid = parsed.twophase_xid;
2853 }
2854 else
2855 recordXid = XLogRecGetXid(record);
2856
2857 /*
2858 * There can be only one transaction end record with this exact
2859 * transactionid
2860 *
2861 * when testing for an xid, we MUST test for equality only, since
2862 * transactions are numbered in the order they start, not the order
2863 * they complete. A higher numbered xid will complete before you about
2864 * 50% of the time...
2865 */
2866 if (recoveryTarget == RECOVERY_TARGET_XID && recoveryTargetInclusive &&
2867 recordXid == recoveryTargetXid)
2868 {
2869 recoveryStopAfter = true;
2870 recoveryStopXid = recordXid;
2871 recoveryStopTime = recordXtime;
2872 recoveryStopLSN = InvalidXLogRecPtr;
2873 recoveryStopName[0] = '\0';
2874
2875 if (xact_info == XLOG_XACT_COMMIT ||
2876 xact_info == XLOG_XACT_COMMIT_PREPARED)
2877 {
2878 ereport(LOG,
2879 (errmsg("recovery stopping after commit of transaction %u, time %s",
2880 recoveryStopXid,
2881 timestamptz_to_str(recoveryStopTime))));
2882 }
2883 else if (xact_info == XLOG_XACT_ABORT ||
2884 xact_info == XLOG_XACT_ABORT_PREPARED)
2885 {
2886 ereport(LOG,
2887 (errmsg("recovery stopping after abort of transaction %u, time %s",
2888 recoveryStopXid,
2889 timestamptz_to_str(recoveryStopTime))));
2890 }
2891 return true;
2892 }
2893 }
2894
2895 /* Check if we should stop as soon as reaching consistency */
2896 if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE && reachedConsistency)
2897 {
2898 ereport(LOG,
2899 (errmsg("recovery stopping after reaching consistency")));
2900
2901 recoveryStopAfter = true;
2902 recoveryStopXid = InvalidTransactionId;
2903 recoveryStopTime = 0;
2904 recoveryStopLSN = InvalidXLogRecPtr;
2905 recoveryStopName[0] = '\0';
2906 return true;
2907 }
2908
2909 return false;
2910 }
2911
2912 /*
2913 * Create a comment for the history file to explain why and where
2914 * timeline changed.
2915 */
2916 static char *
2917 getRecoveryStopReason(void)
2918 {
2919 char reason[200];
2920
2921 if (recoveryTarget == RECOVERY_TARGET_XID)
2922 snprintf(reason, sizeof(reason),
2923 "%s transaction %u",
2924 recoveryStopAfter ? "after" : "before",
2925 recoveryStopXid);
2926 else if (recoveryTarget == RECOVERY_TARGET_TIME)
2927 snprintf(reason, sizeof(reason),
2928 "%s %s\n",
2929 recoveryStopAfter ? "after" : "before",
2930 timestamptz_to_str(recoveryStopTime));
2931 else if (recoveryTarget == RECOVERY_TARGET_LSN)
2932 snprintf(reason, sizeof(reason),
2933 "%s LSN %X/%X\n",
2934 recoveryStopAfter ? "after" : "before",
2935 LSN_FORMAT_ARGS(recoveryStopLSN));
2936 else if (recoveryTarget == RECOVERY_TARGET_NAME)
2937 snprintf(reason, sizeof(reason),
2938 "at restore point \"%s\"",
2939 recoveryStopName);
2940 else if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE)
2941 snprintf(reason, sizeof(reason), "reached consistency");
2942 else
2943 snprintf(reason, sizeof(reason), "no recovery target specified");
2944
2945 return pstrdup(reason);
2946 }
2947
2948 /*
2949 * Wait until shared recoveryPauseState is set to RECOVERY_NOT_PAUSED.
2950 *
2951 * endOfRecovery is true if the recovery target is reached and
2952 * the paused state starts at the end of recovery because of
2953 * recovery_target_action=pause, and false otherwise.
2954 */
2955 static void
2956 recoveryPausesHere(bool endOfRecovery)
2957 {
2958 /* Don't pause unless users can connect! */
2959 if (!LocalHotStandbyActive)
2960 return;
2961
2962 /* Don't pause after standby promotion has been triggered */
2963 if (LocalPromoteIsTriggered)
2964 return;
2965
2966 if (endOfRecovery)
2967 ereport(LOG,
2968 (errmsg("pausing at the end of recovery"),
2969 errhint("Execute pg_wal_replay_resume() to promote.")));
2970 else
2971 ereport(LOG,
2972 (errmsg("recovery has paused"),
2973 errhint("Execute pg_wal_replay_resume() to continue.")));
2974
2975 /* loop until recoveryPauseState is set to RECOVERY_NOT_PAUSED */
2976 while (GetRecoveryPauseState() != RECOVERY_NOT_PAUSED)
2977 {
2978 ProcessStartupProcInterrupts();
2979 if (CheckForStandbyTrigger())
2980 return;
2981
2982 /*
2983 * If recovery pause is requested then set it paused. While we are in
2984 * the loop, user might resume and pause again so set this every time.
2985 */
2986 ConfirmRecoveryPaused();
2987
2988 /*
2989 * We wait on a condition variable that will wake us as soon as the
2990 * pause ends, but we use a timeout so we can check the above exit
2991 * condition periodically too.
2992 */
2993 ConditionVariableTimedSleep(&XLogRecoveryCtl->recoveryNotPausedCV, 1000,
2994 WAIT_EVENT_RECOVERY_PAUSE);
2995 }
2996 ConditionVariableCancelSleep();
2997 }
2998
2999 /*
3000 * When recovery_min_apply_delay is set, we wait long enough to make sure
3001 * certain record types are applied at least that interval behind the primary.
3002 *
3003 * Returns true if we waited.
3004 *
3005 * Note that the delay is calculated between the WAL record log time and
3006 * the current time on standby. We would prefer to keep track of when this
3007 * standby received each WAL record, which would allow a more consistent
3008 * approach and one not affected by time synchronisation issues, but that
3009 * is significantly more effort and complexity for little actual gain in
3010 * usability.
3011 */
3012 static bool
3013 recoveryApplyDelay(XLogReaderState *record)
3014 {
3015 uint8 xact_info;
3016 TimestampTz xtime;
3017 TimestampTz delayUntil;
3018 long msecs;
3019
3020 /* nothing to do if no delay configured */
3021 if (recovery_min_apply_delay <= 0)
3022 return false;
3023
3024 /* no delay is applied on a database not yet consistent */
3025 if (!reachedConsistency)
3026 return false;
3027
3028 /* nothing to do if crash recovery is requested */
3029 if (!ArchiveRecoveryRequested)
3030 return false;
3031
3032 /*
3033 * Is it a COMMIT record?
3034 *
3035 * We deliberately choose not to delay aborts since they have no effect on
3036 * MVCC. We already allow replay of records that don't have a timestamp,
3037 * so there is already opportunity for issues caused by early conflicts on
3038 * standbys.
3039 */
3040 if (XLogRecGetRmid(record) != RM_XACT_ID)
3041 return false;
3042
3043 xact_info = XLogRecGetInfo(record) & XLOG_XACT_OPMASK;
3044
3045 if (xact_info != XLOG_XACT_COMMIT &&
3046 xact_info != XLOG_XACT_COMMIT_PREPARED)
3047 return false;
3048
3049 if (!getRecordTimestamp(record, &xtime))
3050 return false;
3051
3052 delayUntil = TimestampTzPlusMilliseconds(xtime, recovery_min_apply_delay);
3053
3054 /*
3055 * Exit without arming the latch if it's already past time to apply this
3056 * record
3057 */
3058 msecs = TimestampDifferenceMilliseconds(GetCurrentTimestamp(), delayUntil);
3059 if (msecs <= 0)
3060 return false;
3061
3062 while (true)
3063 {
3064 ResetLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
3065
3066 /* This might change recovery_min_apply_delay. */
3067 ProcessStartupProcInterrupts();
3068
3069 if (CheckForStandbyTrigger())
3070 break;
3071
3072 /*
3073 * Recalculate delayUntil as recovery_min_apply_delay could have
3074 * changed while waiting in this loop.
3075 */
3076 delayUntil = TimestampTzPlusMilliseconds(xtime, recovery_min_apply_delay);
3077
3078 /*
3079 * Wait for difference between GetCurrentTimestamp() and delayUntil.
3080 */
3081 msecs = TimestampDifferenceMilliseconds(GetCurrentTimestamp(),
3082 delayUntil);
3083
3084 if (msecs <= 0)
3085 break;
3086
3087 elog(DEBUG2, "recovery apply delay %ld milliseconds", msecs);
3088
3089 (void) WaitLatch(&XLogRecoveryCtl->recoveryWakeupLatch,
3090 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
3091 msecs,
3092 WAIT_EVENT_RECOVERY_APPLY_DELAY);
3093 }
3094 return true;
3095 }
3096
3097 /*
3098 * Get the current state of the recovery pause request.
3099 */
3100 RecoveryPauseState
3101 GetRecoveryPauseState(void)
3102 {
3103 RecoveryPauseState state;
3104
3105 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
3106 state = XLogRecoveryCtl->recoveryPauseState;
3107 SpinLockRelease(&XLogRecoveryCtl->info_lck);
3108
3109 return state;
3110 }
3111
3112 /*
3113 * Set the recovery pause state.
3114 *
3115 * If recovery pause is requested then sets the recovery pause state to
3116 * 'pause requested' if it is not already 'paused'. Otherwise, sets it
3117 * to 'not paused' to resume the recovery. The recovery pause will be
3118 * confirmed by the ConfirmRecoveryPaused.
3119 */
3120 void
3121 SetRecoveryPause(bool recoveryPause)
3122 {
3123 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
3124
3125 if (!recoveryPause)
3126 XLogRecoveryCtl->recoveryPauseState = RECOVERY_NOT_PAUSED;
3127 else if (XLogRecoveryCtl->recoveryPauseState == RECOVERY_NOT_PAUSED)
3128 XLogRecoveryCtl->recoveryPauseState = RECOVERY_PAUSE_REQUESTED;
3129
3130 SpinLockRelease(&XLogRecoveryCtl->info_lck);
3131
3132 if (!recoveryPause)
3133 ConditionVariableBroadcast(&XLogRecoveryCtl->recoveryNotPausedCV);
3134 }
3135
3136 /*
3137 * Confirm the recovery pause by setting the recovery pause state to
3138 * RECOVERY_PAUSED.
3139 */
3140 static void
3141 ConfirmRecoveryPaused(void)
3142 {
3143 /* If recovery pause is requested then set it paused */
3144 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
3145 if (XLogRecoveryCtl->recoveryPauseState == RECOVERY_PAUSE_REQUESTED)
3146 XLogRecoveryCtl->recoveryPauseState = RECOVERY_PAUSED;
3147 SpinLockRelease(&XLogRecoveryCtl->info_lck);
3148 }
3149
3150
3151 /*
3152 * Attempt to read the next XLOG record.
3153 *
3154 * Before first call, the reader needs to be positioned to the first record
3155 * by calling XLogPrefetcherBeginRead().
3156 *
3157 * If no valid record is available, returns NULL, or fails if emode is PANIC.
3158 * (emode must be either PANIC, LOG). In standby mode, retries until a valid
3159 * record is available.
3160 */
3161 static XLogRecord *
3162 ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
3163 bool fetching_ckpt, TimeLineID replayTLI)
3164 {
3165 XLogRecord *record;
3166 XLogReaderState *xlogreader = XLogPrefetcherGetReader(xlogprefetcher);
3167 XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
3168
3169 /* Pass through parameters to XLogPageRead */
3170 private->fetching_ckpt = fetching_ckpt;
3171 private->emode = emode;
3172 private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
3173 private->replayTLI = replayTLI;
3174
3175 /* This is the first attempt to read this page. */
3176 lastSourceFailed = false;
3177
3178 for (;;)
3179 {
3180 char *errormsg;
3181
3182 record = XLogPrefetcherReadRecord(xlogprefetcher, &errormsg);
3183 if (record == NULL)
3184 {
3185 /*
3186 * When we find that WAL ends in an incomplete record, keep track
3187 * of that record. After recovery is done, we'll write a record
3188 * to indicate to downstream WAL readers that that portion is to
3189 * be ignored.
3190 *
3191 * However, when ArchiveRecoveryRequested = true, we're going to
3192 * switch to a new timeline at the end of recovery. We will only
3193 * copy WAL over to the new timeline up to the end of the last
3194 * complete record, so if we did this, we would later create an
3195 * overwrite contrecord in the wrong place, breaking everything.
3196 */
3197 if (!ArchiveRecoveryRequested &&
3198 !XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))
3199 {
3200 abortedRecPtr = xlogreader->abortedRecPtr;
3201 missingContrecPtr = xlogreader->missingContrecPtr;
3202 }
3203
3204 if (readFile >= 0)
3205 {
3206 close(readFile);
3207 readFile = -1;
3208 }
3209
3210 /*
3211 * We only end up here without a message when XLogPageRead()
3212 * failed - in that case we already logged something. In
3213 * StandbyMode that only happens if we have been triggered, so we
3214 * shouldn't loop anymore in that case.
3215 */
3216 if (errormsg)
3217 ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
3218 (errmsg_internal("%s", errormsg) /* already translated */ ));
3219 }
3220
3221 /*
3222 * Check page TLI is one of the expected values.
3223 */
3224 else if (!tliInHistory(xlogreader->latestPageTLI, expectedTLEs))
3225 {
3226 char fname[MAXFNAMELEN];
3227 XLogSegNo segno;
3228 int32 offset;
3229
3230 XLByteToSeg(xlogreader->latestPagePtr, segno, wal_segment_size);
3231 offset = XLogSegmentOffset(xlogreader->latestPagePtr,
3232 wal_segment_size);
3233 XLogFileName(fname, xlogreader->seg.ws_tli, segno,
3234 wal_segment_size);
3235 ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
3236 (errmsg("unexpected timeline ID %u in WAL segment %s, LSN %X/%X, offset %u",
3237 xlogreader->latestPageTLI,
3238 fname,
3239 LSN_FORMAT_ARGS(xlogreader->latestPagePtr),
3240 offset)));
3241 record = NULL;
3242 }
3243
3244 if (record)
3245 {
3246 /* Great, got a record */
3247 return record;
3248 }
3249 else
3250 {
3251 /* No valid record available from this source */
3252 lastSourceFailed = true;
3253
3254 /*
3255 * If archive recovery was requested, but we were still doing
3256 * crash recovery, switch to archive recovery and retry using the
3257 * offline archive. We have now replayed all the valid WAL in
3258 * pg_wal, so we are presumably now consistent.
3259 *
3260 * We require that there's at least some valid WAL present in
3261 * pg_wal, however (!fetching_ckpt). We could recover using the
3262 * WAL from the archive, even if pg_wal is completely empty, but
3263 * we'd have no idea how far we'd have to replay to reach
3264 * consistency. So err on the safe side and give up.
3265 */
3266 if (!InArchiveRecovery && ArchiveRecoveryRequested &&
3267 !fetching_ckpt)
3268 {
3269 ereport(DEBUG1,
3270 (errmsg_internal("reached end of WAL in pg_wal, entering archive recovery")));
3271 InArchiveRecovery = true;
3272 if (StandbyModeRequested)
3273 EnableStandbyMode();
3274
3275 SwitchIntoArchiveRecovery(xlogreader->EndRecPtr, replayTLI);
3276 minRecoveryPoint = xlogreader->EndRecPtr;
3277 minRecoveryPointTLI = replayTLI;
3278
3279 CheckRecoveryConsistency();
3280
3281 /*
3282 * Before we retry, reset lastSourceFailed and currentSource
3283 * so that we will check the archive next.
3284 */
3285 lastSourceFailed = false;
3286 currentSource = XLOG_FROM_ANY;
3287
3288 continue;
3289 }
3290
3291 /* In standby mode, loop back to retry. Otherwise, give up. */
3292 if (StandbyMode && !CheckForStandbyTrigger())
3293 continue;
3294 else
3295 return NULL;
3296 }
3297 }
3298 }
3299
3300 /*
3301 * Read the XLOG page containing targetPagePtr into readBuf (if not read
3302 * already). Returns number of bytes read, if the page is read successfully,
3303 * or XLREAD_FAIL in case of errors. When errors occur, they are ereport'ed,
3304 * but only if they have not been previously reported.
3305 *
3306 * See XLogReaderRoutine.page_read for more details.
3307 *
3308 * While prefetching, xlogreader->nonblocking may be set. In that case,
3309 * returns XLREAD_WOULDBLOCK if we'd otherwise have to wait for more WAL.
3310 *
3311 * This is responsible for restoring files from archive as needed, as well
3312 * as for waiting for the requested WAL record to arrive in standby mode.
3313 *
3314 * xlogreader->private_data->emode specifies the log level used for reporting
3315 * "file not found" or "end of WAL" situations in archive recovery, or in
3316 * standby mode when promotion is triggered. If set to WARNING or below,
3317 * XLogPageRead() returns XLREAD_FAIL in those situations, on higher log
3318 * levels the ereport() won't return.
3319 *
3320 * In standby mode, if after a successful return of XLogPageRead() the
3321 * caller finds the record it's interested in to be broken, it should
3322 * ereport the error with the level determined by
3323 * emode_for_corrupt_record(), and then set lastSourceFailed
3324 * and call XLogPageRead() again with the same arguments. This lets
3325 * XLogPageRead() to try fetching the record from another source, or to
3326 * sleep and retry.
3327 */
3328 static int
3329 XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
3330 XLogRecPtr targetRecPtr, char *readBuf)
3331 {
3332 XLogPageReadPrivate *private =
3333 (XLogPageReadPrivate *) xlogreader->private_data;
3334 int emode = private->emode;
3335 uint32 targetPageOff;
3336 XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
3337 int r;
3338 instr_time io_start;
3339
3340 XLByteToSeg(targetPagePtr, targetSegNo, wal_segment_size);
3341 targetPageOff = XLogSegmentOffset(targetPagePtr, wal_segment_size);
3342
3343 /*
3344 * See if we need to switch to a new segment because the requested record
3345 * is not in the currently open one.
3346 */
3347 if (readFile >= 0 &&
3348 !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
3349 {
3350 /*
3351 * Request a restartpoint if we've replayed too much xlog since the
3352 * last one.
3353 */
3354 if (ArchiveRecoveryRequested && IsUnderPostmaster)
3355 {
3356 if (XLogCheckpointNeeded(readSegNo))
3357 {
3358 (void) GetRedoRecPtr();
3359 if (XLogCheckpointNeeded(readSegNo))
3360 RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
3361 }
3362 }
3363
3364 close(readFile);
3365 readFile = -1;
3366 readSource = XLOG_FROM_ANY;
3367 }
3368
3369 XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
3370
3371 retry:
3372 /* See if we need to retrieve more data */
3373 if (readFile < 0 ||
3374 (readSource == XLOG_FROM_STREAM &&
3375 flushedUpto < targetPagePtr + reqLen))
3376 {
3377 if (readFile >= 0 &&
3378 xlogreader->nonblocking &&
3379 readSource == XLOG_FROM_STREAM &&
3380 flushedUpto < targetPagePtr + reqLen)
3381 return XLREAD_WOULDBLOCK;
3382
3383 switch (WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
3384 private->randAccess,
3385 private->fetching_ckpt,
3386 targetRecPtr,
3387 private->replayTLI,
3388 xlogreader->EndRecPtr,
3389 xlogreader->nonblocking))
3390 {
3391 case XLREAD_WOULDBLOCK:
3392 return XLREAD_WOULDBLOCK;
3393 case XLREAD_FAIL:
3394 if (readFile >= 0)
3395 close(readFile);
3396 readFile = -1;
3397 readLen = 0;
3398 readSource = XLOG_FROM_ANY;
3399 return XLREAD_FAIL;
3400 case XLREAD_SUCCESS:
3401 break;
3402 }
3403 }
3404
3405 /*
3406 * At this point, we have the right segment open and if we're streaming we
3407 * know the requested record is in it.
3408 */
3409 Assert(readFile != -1);
3410
3411 /*
3412 * If the current segment is being streamed from the primary, calculate
3413 * how much of the current page we have received already. We know the
3414 * requested record has been received, but this is for the benefit of
3415 * future calls, to allow quick exit at the top of this function.
3416 */
3417 if (readSource == XLOG_FROM_STREAM)
3418 {
3419 if (((targetPagePtr) / XLOG_BLCKSZ) != (flushedUpto / XLOG_BLCKSZ))
3420 readLen = XLOG_BLCKSZ;
3421 else
3422 readLen = XLogSegmentOffset(flushedUpto, wal_segment_size) -
3423 targetPageOff;
3424 }
3425 else
3426 readLen = XLOG_BLCKSZ;
3427
3428 /* Read the requested page */
3429 readOff = targetPageOff;
3430
3431 /* Measure I/O timing when reading segment */
3432 io_start = pgstat_prepare_io_time(track_wal_io_timing);
3433
3434 pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
3435 r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
3436 if (r != XLOG_BLCKSZ)
3437 {
3438 char fname[MAXFNAMELEN];
3439 int save_errno = errno;
3440
3441 pgstat_report_wait_end();
3442
3443 /* Count I/O stats only for successful short reads */
3444 if (r > 0)
3445 pgstat_count_io_op_time(IOOBJECT_WAL, IOCONTEXT_NORMAL, IOOP_READ,
3446 io_start, 1, r);
3447
3448 XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
3449 if (r < 0)
3450 {
3451 errno = save_errno;
3452 ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
3453 (errcode_for_file_access(),
3454 errmsg("could not read from WAL segment %s, LSN %X/%X, offset %u: %m",
3455 fname, LSN_FORMAT_ARGS(targetPagePtr),
3456 readOff)));
3457 }
3458 else
3459 ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
3460 (errcode(ERRCODE_DATA_CORRUPTED),
3461 errmsg("could not read from WAL segment %s, LSN %X/%X, offset %u: read %d of %zu",
3462 fname, LSN_FORMAT_ARGS(targetPagePtr),
3463 readOff, r, (Size) XLOG_BLCKSZ)));
3464 goto next_record_is_invalid;
3465 }
3466 pgstat_report_wait_end();
3467
3468 pgstat_count_io_op_time(IOOBJECT_WAL, IOCONTEXT_NORMAL, IOOP_READ,
3469 io_start, 1, r);
3470
3471 Assert(targetSegNo == readSegNo);
3472 Assert(targetPageOff == readOff);
3473 Assert(reqLen <= readLen);
3474
3475 xlogreader->seg.ws_tli = curFileTLI;
3476
3477 /*
3478 * Check the page header immediately, so that we can retry immediately if
3479 * it's not valid. This may seem unnecessary, because ReadPageInternal()
3480 * validates the page header anyway, and would propagate the failure up to
3481 * ReadRecord(), which would retry. However, there's a corner case with
3482 * continuation records, if a record is split across two pages such that
3483 * we would need to read the two pages from different sources across two
3484 * WAL segments.
3485 *
3486 * The first page is only available locally, in pg_wal, because it's
3487 * already been recycled on the primary. The second page, however, is not
3488 * present in pg_wal, and we should stream it from the primary. There is a
3489 * recycled WAL segment present in pg_wal, with garbage contents, however.
3490 * We would read the first page from the local WAL segment, but when
3491 * reading the second page, we would read the bogus, recycled, WAL
3492 * segment. If we didn't catch that case here, we would never recover,
3493 * because ReadRecord() would retry reading the whole record from the
3494 * beginning.
3495 *
3496 * Of course, this only catches errors in the page header, which is what
3497 * happens in the case of a recycled WAL segment. Other kinds of errors or
3498 * corruption still has the same problem. But this at least fixes the
3499 * common case, which can happen as part of normal operation.
3500 *
3501 * Validating the page header is cheap enough that doing it twice
3502 * shouldn't be a big deal from a performance point of view.
3503 *
3504 * When not in standby mode, an invalid page header should cause recovery
3505 * to end, not retry reading the page, so we don't need to validate the
3506 * page header here for the retry. Instead, ReadPageInternal() is
3507 * responsible for the validation.
3508 */
3509 if (StandbyMode &&
3510 (targetPagePtr % wal_segment_size) == 0 &&
3511 !XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
3512 {
3513 /*
3514 * Emit this error right now then retry this page immediately. Use
3515 * errmsg_internal() because the message was already translated.
3516 */
3517 if (xlogreader->errormsg_buf[0])
3518 ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
3519 (errmsg_internal("%s", xlogreader->errormsg_buf)));
3520
3521 /* reset any error XLogReaderValidatePageHeader() might have set */
3522 XLogReaderResetError(xlogreader);
3523 goto next_record_is_invalid;
3524 }
3525
3526 return readLen;
3527
3528 next_record_is_invalid:
3529
3530 /*
3531 * If we're reading ahead, give up fast. Retries and error reporting will
3532 * be handled by a later read when recovery catches up to this point.
3533 */
3534 if (xlogreader->nonblocking)
3535 return XLREAD_WOULDBLOCK;
3536
3537 lastSourceFailed = true;
3538
3539 if (readFile >= 0)
3540 close(readFile);
3541 readFile = -1;
3542 readLen = 0;
3543 readSource = XLOG_FROM_ANY;
3544
3545 /* In standby-mode, keep trying */
3546 if (StandbyMode)
3547 goto retry;
3548 else
3549 return XLREAD_FAIL;
3550 }
3551
3552 /*
3553 * Open the WAL segment containing WAL location 'RecPtr'.
3554 *
3555 * The segment can be fetched via restore_command, or via walreceiver having
3556 * streamed the record, or it can already be present in pg_wal. Checking
3557 * pg_wal is mainly for crash recovery, but it will be polled in standby mode
3558 * too, in case someone copies a new segment directly to pg_wal. That is not
3559 * documented or recommended, though.
3560 *
3561 * If 'fetching_ckpt' is true, we're fetching a checkpoint record, and should
3562 * prepare to read WAL starting from RedoStartLSN after this.
3563 *
3564 * 'RecPtr' might not point to the beginning of the record we're interested
3565 * in, it might also point to the page or segment header. In that case,
3566 * 'tliRecPtr' is the position of the WAL record we're interested in. It is
3567 * used to decide which timeline to stream the requested WAL from.
3568 *
3569 * 'replayLSN' is the current replay LSN, so that if we scan for new
3570 * timelines, we can reject a switch to a timeline that branched off before
3571 * this point.
3572 *
3573 * If the record is not immediately available, the function returns false
3574 * if we're not in standby mode. In standby mode, waits for it to become
3575 * available.
3576 *
3577 * When the requested record becomes available, the function opens the file
3578 * containing it (if not open already), and returns XLREAD_SUCCESS. When end
3579 * of standby mode is triggered by the user, and there is no more WAL
3580 * available, returns XLREAD_FAIL.
3581 *
3582 * If nonblocking is true, then give up immediately if we can't satisfy the
3583 * request, returning XLREAD_WOULDBLOCK instead of waiting.
3584 */
3585 static XLogPageReadResult
3586 WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
3587 bool fetching_ckpt, XLogRecPtr tliRecPtr,
3588 TimeLineID replayTLI, XLogRecPtr replayLSN,
3589 bool nonblocking)
3590 {
3591 static TimestampTz last_fail_time = 0;
3592 TimestampTz now;
3593 bool streaming_reply_sent = false;
3594
3595 /*-------
3596 * Standby mode is implemented by a state machine:
3597 *
3598 * 1. Read from either archive or pg_wal (XLOG_FROM_ARCHIVE), or just
3599 * pg_wal (XLOG_FROM_PG_WAL)
3600 * 2. Check for promotion trigger request
3601 * 3. Read from primary server via walreceiver (XLOG_FROM_STREAM)
3602 * 4. Rescan timelines
3603 * 5. Sleep wal_retrieve_retry_interval milliseconds, and loop back to 1.
3604 *
3605 * Failure to read from the current source advances the state machine to
3606 * the next state.
3607 *
3608 * 'currentSource' indicates the current state. There are no currentSource
3609 * values for "check trigger", "rescan timelines", and "sleep" states,
3610 * those actions are taken when reading from the previous source fails, as
3611 * part of advancing to the next state.
3612 *
3613 * If standby mode is turned off while reading WAL from stream, we move
3614 * to XLOG_FROM_ARCHIVE and reset lastSourceFailed, to force fetching
3615 * the files (which would be required at end of recovery, e.g., timeline
3616 * history file) from archive or pg_wal. We don't need to kill WAL receiver
3617 * here because it's already stopped when standby mode is turned off at
3618 * the end of recovery.
3619 *-------
3620 */
3621 if (!InArchiveRecovery)
3622 currentSource = XLOG_FROM_PG_WAL;
3623 else if (currentSource == XLOG_FROM_ANY ||
3624 (!StandbyMode && currentSource == XLOG_FROM_STREAM))
3625 {
3626 lastSourceFailed = false;
3627 currentSource = XLOG_FROM_ARCHIVE;
3628 }
3629
3630 for (;;)
3631 {
3632 XLogSource oldSource = currentSource;
3633 bool startWalReceiver = false;
3634
3635 /*
3636 * First check if we failed to read from the current source, and
3637 * advance the state machine if so. The failure to read might've
3638 * happened outside this function, e.g when a CRC check fails on a
3639 * record, or within this loop.
3640 */
3641 if (lastSourceFailed)
3642 {
3643 /*
3644 * Don't allow any retry loops to occur during nonblocking
3645 * readahead. Let the caller process everything that has been
3646 * decoded already first.
3647 */
3648 if (nonblocking)
3649 return XLREAD_WOULDBLOCK;
3650
3651 switch (currentSource)
3652 {
3653 case XLOG_FROM_ARCHIVE:
3654 case XLOG_FROM_PG_WAL:
3655
3656 /*
3657 * Check to see if promotion is requested. Note that we do
3658 * this only after failure, so when you promote, we still
3659 * finish replaying as much as we can from archive and
3660 * pg_wal before failover.
3661 */
3662 if (StandbyMode && CheckForStandbyTrigger())
3663 {
3664 XLogShutdownWalRcv();
3665 return XLREAD_FAIL;
3666 }
3667
3668 /*
3669 * Not in standby mode, and we've now tried the archive
3670 * and pg_wal.
3671 */
3672 if (!StandbyMode)
3673 return XLREAD_FAIL;
3674
3675 /*
3676 * Move to XLOG_FROM_STREAM state, and set to start a
3677 * walreceiver if necessary.
3678 */
3679 currentSource = XLOG_FROM_STREAM;
3680 startWalReceiver = true;
3681 break;
3682
3683 case XLOG_FROM_STREAM:
3684
3685 /*
3686 * Failure while streaming. Most likely, we got here
3687 * because streaming replication was terminated, or
3688 * promotion was triggered. But we also get here if we
3689 * find an invalid record in the WAL streamed from the
3690 * primary, in which case something is seriously wrong.
3691 * There's little chance that the problem will just go
3692 * away, but PANIC is not good for availability either,
3693 * especially in hot standby mode. So, we treat that the
3694 * same as disconnection, and retry from archive/pg_wal
3695 * again. The WAL in the archive should be identical to
3696 * what was streamed, so it's unlikely that it helps, but
3697 * one can hope...
3698 */
3699
3700 /*
3701 * We should be able to move to XLOG_FROM_STREAM only in
3702 * standby mode.
3703 */
3704 Assert(StandbyMode);
3705
3706 /*
3707 * Before we leave XLOG_FROM_STREAM state, make sure that
3708 * walreceiver is not active, so that it won't overwrite
3709 * WAL that we restore from archive.
3710 *
3711 * If walreceiver is actively streaming (or attempting to
3712 * connect), we must shut it down. However, if it's
3713 * already in WAITING state (e.g., due to timeline
3714 * divergence), we only need to reset the install flag to
3715 * allow archive restoration.
3716 */
3717 if (WalRcvStreaming())
3718 XLogShutdownWalRcv();
3719 else
3720 {
3721 ResetInstallXLogFileSegmentActive();
3722 }
3723
3724 /*
3725 * Before we sleep, re-scan for possible new timelines if
3726 * we were requested to recover to the latest timeline.
3727 */
3728 if (recoveryTargetTimeLineGoal == RECOVERY_TARGET_TIMELINE_LATEST)
3729 {
3730 if (rescanLatestTimeLine(replayTLI, replayLSN))
3731 {
3732 currentSource = XLOG_FROM_ARCHIVE;
3733 break;
3734 }
3735 }
3736
3737 /*
3738 * XLOG_FROM_STREAM is the last state in our state
3739 * machine, so we've exhausted all the options for
3740 * obtaining the requested WAL. We're going to loop back
3741 * and retry from the archive, but if it hasn't been long
3742 * since last attempt, sleep wal_retrieve_retry_interval
3743 * milliseconds to avoid busy-waiting.
3744 */
3745 now = GetCurrentTimestamp();
3746 if (!TimestampDifferenceExceeds(last_fail_time, now,
3747 wal_retrieve_retry_interval))
3748 {
3749 long wait_time;
3750
3751 wait_time = wal_retrieve_retry_interval -
3752 TimestampDifferenceMilliseconds(last_fail_time, now);
3753
3754 elog(LOG, "waiting for WAL to become available at %X/%X",
3755 LSN_FORMAT_ARGS(RecPtr));
3756
3757 /* Do background tasks that might benefit us later. */
3758 KnownAssignedTransactionIdsIdleMaintenance();
3759
3760 (void) WaitLatch(&XLogRecoveryCtl->recoveryWakeupLatch,
3761 WL_LATCH_SET | WL_TIMEOUT |
3762 WL_EXIT_ON_PM_DEATH,
3763 wait_time,
3764 WAIT_EVENT_RECOVERY_RETRIEVE_RETRY_INTERVAL);
3765 ResetLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
3766 now = GetCurrentTimestamp();
3767
3768 /* Handle interrupt signals of startup process */
3769 ProcessStartupProcInterrupts();
3770 }
3771 last_fail_time = now;
3772 currentSource = XLOG_FROM_ARCHIVE;
3773 break;
3774
3775 default:
3776 elog(ERROR, "unexpected WAL source %d", currentSource);
3777 }
3778 }
3779 else if (currentSource == XLOG_FROM_PG_WAL)
3780 {
3781 /*
3782 * We just successfully read a file in pg_wal. We prefer files in
3783 * the archive over ones in pg_wal, so try the next file again
3784 * from the archive first.
3785 */
3786 if (InArchiveRecovery)
3787 currentSource = XLOG_FROM_ARCHIVE;
3788 }
3789
3790 if (currentSource != oldSource)
3791 elog(DEBUG2, "switched WAL source from %s to %s after %s",
3792 xlogSourceNames[oldSource], xlogSourceNames[currentSource],
3793 lastSourceFailed ? "failure" : "success");
3794
3795 /*
3796 * We've now handled possible failure. Try to read from the chosen
3797 * source.
3798 */
3799 lastSourceFailed = false;
3800
3801 switch (currentSource)
3802 {
3803 case XLOG_FROM_ARCHIVE:
3804 case XLOG_FROM_PG_WAL:
3805
3806 /*
3807 * WAL receiver must not be running when reading WAL from
3808 * archive or pg_wal.
3809 */
3810 Assert(!WalRcvStreaming());
3811
3812 /* Close any old file we might have open. */
3813 if (readFile >= 0)
3814 {
3815 close(readFile);
3816 readFile = -1;
3817 }
3818 /* Reset curFileTLI if random fetch. */
3819 if (randAccess)
3820 curFileTLI = 0;
3821
3822 /*
3823 * Try to restore the file from archive, or read an existing
3824 * file from pg_wal.
3825 */
3826 readFile = XLogFileReadAnyTLI(readSegNo,
3827 currentSource == XLOG_FROM_ARCHIVE ? XLOG_FROM_ANY :
3828 currentSource);
3829 if (readFile >= 0)
3830 return XLREAD_SUCCESS; /* success! */
3831
3832 /*
3833 * Nope, not found in archive or pg_wal.
3834 */
3835 lastSourceFailed = true;
3836 break;
3837
3838 case XLOG_FROM_STREAM:
3839 {
3840 bool havedata;
3841
3842 /*
3843 * We should be able to move to XLOG_FROM_STREAM only in
3844 * standby mode.
3845 */
3846 Assert(StandbyMode);
3847
3848 /*
3849 * First, shutdown walreceiver if its restart has been
3850 * requested -- but no point if we're already slated for
3851 * starting it.
3852 */
3853 if (pendingWalRcvRestart && !startWalReceiver)
3854 {
3855 XLogShutdownWalRcv();
3856
3857 /*
3858 * Re-scan for possible new timelines if we were
3859 * requested to recover to the latest timeline.
3860 */
3861 if (recoveryTargetTimeLineGoal ==
3862 RECOVERY_TARGET_TIMELINE_LATEST)
3863 rescanLatestTimeLine(replayTLI, replayLSN);
3864
3865 startWalReceiver = true;
3866 }
3867 pendingWalRcvRestart = false;
3868
3869 /*
3870 * Launch walreceiver if needed.
3871 *
3872 * If fetching_ckpt is true, RecPtr points to the initial
3873 * checkpoint location. In that case, we use RedoStartLSN
3874 * as the streaming start position instead of RecPtr, so
3875 * that when we later jump backwards to start redo at
3876 * RedoStartLSN, we will have the logs streamed already.
3877 */
3878 if (startWalReceiver &&
3879 PrimaryConnInfo && strcmp(PrimaryConnInfo, "") != 0)
3880 {
3881 XLogRecPtr ptr;
3882 TimeLineID tli;
3883
3884 if (fetching_ckpt)
3885 {
3886 ptr = RedoStartLSN;
3887 tli = RedoStartTLI;
3888 }
3889 else
3890 {
3891 ptr = RecPtr;
3892
3893 /*
3894 * Use the record begin position to determine the
3895 * TLI, rather than the position we're reading.
3896 */
3897 tli = tliOfPointInHistory(tliRecPtr, expectedTLEs);
3898
3899 if (curFileTLI > 0 && tli < curFileTLI)
3900 elog(ERROR, "according to history file, WAL location %X/%X belongs to timeline %u, but previous recovered WAL file came from timeline %u",
3901 LSN_FORMAT_ARGS(tliRecPtr),
3902 tli, curFileTLI);
3903 }
3904 curFileTLI = tli;
3905 SetInstallXLogFileSegmentActive();
3906 RequestXLogStreaming(tli, ptr, PrimaryConnInfo,
3907 PrimarySlotName,
3908 wal_receiver_create_temp_slot);
3909 flushedUpto = 0;
3910 }
3911
3912 /*
3913 * Check if WAL receiver is active or wait to start up.
3914 */
3915 if (!WalRcvStreaming())
3916 {
3917 lastSourceFailed = true;
3918 break;
3919 }
3920
3921 /*
3922 * Walreceiver is active, so see if new data has arrived.
3923 *
3924 * We only advance XLogReceiptTime when we obtain fresh
3925 * WAL from walreceiver and observe that we had already
3926 * processed everything before the most recent "chunk"
3927 * that it flushed to disk. In steady state where we are
3928 * keeping up with the incoming data, XLogReceiptTime will
3929 * be updated on each cycle. When we are behind,
3930 * XLogReceiptTime will not advance, so the grace time
3931 * allotted to conflicting queries will decrease.
3932 */
3933 if (RecPtr < flushedUpto)
3934 havedata = true;
3935 else
3936 {
3937 XLogRecPtr latestChunkStart;
3938
3939 flushedUpto = GetWalRcvFlushRecPtr(&latestChunkStart, &receiveTLI);
3940 if (RecPtr < flushedUpto && receiveTLI == curFileTLI)
3941 {
3942 havedata = true;
3943 if (latestChunkStart <= RecPtr)
3944 {
3945 XLogReceiptTime = GetCurrentTimestamp();
3946 SetCurrentChunkStartTime(XLogReceiptTime);
3947 }
3948 }
3949 else
3950 havedata = false;
3951 }
3952 if (havedata)
3953 {
3954 /*
3955 * Great, streamed far enough. Open the file if it's
3956 * not open already. Also read the timeline history
3957 * file if we haven't initialized timeline history
3958 * yet; it should be streamed over and present in
3959 * pg_wal by now. Use XLOG_FROM_STREAM so that source
3960 * info is set correctly and XLogReceiptTime isn't
3961 * changed.
3962 *
3963 * NB: We must set readTimeLineHistory based on
3964 * recoveryTargetTLI, not receiveTLI. Normally they'll
3965 * be the same, but if recovery_target_timeline is
3966 * 'latest' and archiving is configured, then it's
3967 * possible that we managed to retrieve one or more
3968 * new timeline history files from the archive,
3969 * updating recoveryTargetTLI.
3970 */
3971 if (readFile < 0)
3972 {
3973 if (!expectedTLEs)
3974 expectedTLEs = readTimeLineHistory(recoveryTargetTLI);
3975 readFile = XLogFileRead(readSegNo, receiveTLI,
3976 XLOG_FROM_STREAM, false);
3977 Assert(readFile >= 0);
3978 }
3979 else
3980 {
3981 /* just make sure source info is correct... */
3982 readSource = XLOG_FROM_STREAM;
3983 XLogReceiptSource = XLOG_FROM_STREAM;
3984 return XLREAD_SUCCESS;
3985 }
3986 break;
3987 }
3988
3989 /* In nonblocking mode, return rather than sleeping. */
3990 if (nonblocking)
3991 return XLREAD_WOULDBLOCK;
3992
3993 /*
3994 * Data not here yet. Check for trigger, then wait for
3995 * walreceiver to wake us up when new WAL arrives.
3996 */
3997 if (CheckForStandbyTrigger())
3998 {
3999 /*
4000 * Note that we don't return XLREAD_FAIL immediately
4001 * here. After being triggered, we still want to
4002 * replay all the WAL that was already streamed. It's
4003 * in pg_wal now, so we just treat this as a failure,
4004 * and the state machine will move on to replay the
4005 * streamed WAL from pg_wal, and then recheck the
4006 * trigger and exit replay.
4007 */
4008 lastSourceFailed = true;
4009 break;
4010 }
4011
4012 /*
4013 * Since we have replayed everything we have received so
4014 * far and are about to start waiting for more WAL, let's
4015 * tell the upstream server our replay location now so
4016 * that pg_stat_replication doesn't show stale
4017 * information.
4018 */
4019 if (!streaming_reply_sent)
4020 {
4021 WalRcvForceReply();
4022 streaming_reply_sent = true;
4023 }
4024
4025 /* Do any background tasks that might benefit us later. */
4026 KnownAssignedTransactionIdsIdleMaintenance();
4027
4028 /* Update pg_stat_recovery_prefetch before sleeping. */
4029 XLogPrefetcherComputeStats(xlogprefetcher);
4030
4031 /*
4032 * Wait for more WAL to arrive, when we will be woken
4033 * immediately by the WAL receiver.
4034 */
4035 (void) WaitLatch(&XLogRecoveryCtl->recoveryWakeupLatch,
4036 WL_LATCH_SET | WL_EXIT_ON_PM_DEATH,
4037 -1L,
4038 WAIT_EVENT_RECOVERY_WAL_STREAM);
4039 ResetLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
4040 break;
4041 }
4042
4043 default:
4044 elog(ERROR, "unexpected WAL source %d", currentSource);
4045 }
4046
4047 /*
4048 * Check for recovery pause here so that we can confirm more quickly
4049 * that a requested pause has actually taken effect.
4050 */
4051 if (((volatile XLogRecoveryCtlData *) XLogRecoveryCtl)->recoveryPauseState !=
4052 RECOVERY_NOT_PAUSED)
4053 recoveryPausesHere(false);
4054
4055 /*
4056 * This possibly-long loop needs to handle interrupts of startup
4057 * process.
4058 */
4059 ProcessStartupProcInterrupts();
4060 }
4061
4062 return XLREAD_FAIL; /* not reached */
4063 }
4064
4065
4066 /*
4067 * Determine what log level should be used to report a corrupt WAL record
4068 * in the current WAL page, previously read by XLogPageRead().
4069 *
4070 * 'emode' is the error mode that would be used to report a file-not-found
4071 * or legitimate end-of-WAL situation. Generally, we use it as-is, but if
4072 * we're retrying the exact same record that we've tried previously, only
4073 * complain the first time to keep the noise down. However, we only do when
4074 * reading from pg_wal, because we don't expect any invalid records in archive
4075 * or in records streamed from the primary. Files in the archive should be complete,
4076 * and we should never hit the end of WAL because we stop and wait for more WAL
4077 * to arrive before replaying it.
4078 *
4079 * NOTE: This function remembers the RecPtr value it was last called with,
4080 * to suppress repeated messages about the same record. Only call this when
4081 * you are about to ereport(), or you might cause a later message to be
4082 * erroneously suppressed.
4083 */
4084 static int
4085 emode_for_corrupt_record(int emode, XLogRecPtr RecPtr)
4086 {
4087 static XLogRecPtr lastComplaint = 0;
4088
4089 if (readSource == XLOG_FROM_PG_WAL && emode == LOG)
4090 {
4091 if (RecPtr == lastComplaint)
4092 emode = DEBUG1;
4093 else
4094 lastComplaint = RecPtr;
4095 }
4096 return emode;
4097 }
4098
4099
4100 /*
4101 * Subroutine to try to fetch and validate a prior checkpoint record.
4102 */
4103 static XLogRecord *
4104 ReadCheckpointRecord(XLogPrefetcher *xlogprefetcher, XLogRecPtr RecPtr,
4105 TimeLineID replayTLI)
4106 {
4107 XLogRecord *record;
4108 uint8 info;
4109
4110 Assert(xlogreader != NULL);
4111
4112 if (!XRecOffIsValid(RecPtr))
4113 {
4114 ereport(LOG,
4115 (errmsg("invalid checkpoint location")));
4116 return NULL;
4117 }
4118
4119 XLogPrefetcherBeginRead(xlogprefetcher, RecPtr);
4120 record = ReadRecord(xlogprefetcher, LOG, true, replayTLI);
4121
4122 if (record == NULL)
4123 {
4124 ereport(LOG,
4125 (errmsg("invalid checkpoint record")));
4126 return NULL;
4127 }
4128 if (record->xl_rmid != RM_XLOG_ID)
4129 {
4130 ereport(LOG,
4131 (errmsg("invalid resource manager ID in checkpoint record")));
4132 return NULL;
4133 }
4134 info = record->xl_info & ~XLR_INFO_MASK;
4135 if (info != XLOG_CHECKPOINT_SHUTDOWN &&
4136 info != XLOG_CHECKPOINT_ONLINE)
4137 {
4138 ereport(LOG,
4139 (errmsg("invalid xl_info in checkpoint record")));
4140 return NULL;
4141 }
4142 if (record->xl_tot_len != SizeOfXLogRecord + SizeOfXLogRecordDataHeaderShort + sizeof(CheckPoint))
4143 {
4144 ereport(LOG,
4145 (errmsg("invalid length of checkpoint record")));
4146 return NULL;
4147 }
4148 return record;
4149 }
4150
4151 /*
4152 * Scan for new timelines that might have appeared in the archive since we
4153 * started recovery.
4154 *
4155 * If there are any, the function changes recovery target TLI to the latest
4156 * one and returns 'true'.
4157 */
4158 static bool
4159 rescanLatestTimeLine(TimeLineID replayTLI, XLogRecPtr replayLSN)
4160 {
4161 List *newExpectedTLEs;
4162 bool found;
4163 ListCell *cell;
4164 TimeLineID newtarget;
4165 TimeLineID oldtarget = recoveryTargetTLI;
4166 TimeLineHistoryEntry *currentTle = NULL;
4167
4168 newtarget = findNewestTimeLine(recoveryTargetTLI);
4169 if (newtarget == recoveryTargetTLI)
4170 {
4171 /* No new timelines found */
4172 return false;
4173 }
4174
4175 /*
4176 * Determine the list of expected TLIs for the new TLI
4177 */
4178
4179 newExpectedTLEs = readTimeLineHistory(newtarget);
4180
4181 /*
4182 * If the current timeline is not part of the history of the new timeline,
4183 * we cannot proceed to it.
4184 */
4185 found = false;
4186 foreach(cell, newExpectedTLEs)
4187 {
4188 currentTle = (TimeLineHistoryEntry *) lfirst(cell);
4189
4190 if (currentTle->tli == recoveryTargetTLI)
4191 {
4192 found = true;
4193 break;
4194 }
4195 }
4196 if (!found)
4197 {
4198 ereport(LOG,
4199 (errmsg("new timeline %u is not a child of database system timeline %u",
4200 newtarget,
4201 replayTLI)));
4202 return false;
4203 }
4204
4205 /*
4206 * The current timeline was found in the history file, but check that the
4207 * next timeline was forked off from it *after* the current recovery
4208 * location.
4209 */
4210 if (currentTle->end < replayLSN)
4211 {
4212 ereport(LOG,
4213 (errmsg("new timeline %u forked off current database system timeline %u before current recovery point %X/%X",
4214 newtarget,
4215 replayTLI,
4216 LSN_FORMAT_ARGS(replayLSN))));
4217 return false;
4218 }
4219
4220 /* The new timeline history seems valid. Switch target */
4221 recoveryTargetTLI = newtarget;
4222 list_free_deep(expectedTLEs);
4223 expectedTLEs = newExpectedTLEs;
4224
4225 /*
4226 * As in StartupXLOG(), try to ensure we have all the history files
4227 * between the old target and new target in pg_wal.
4228 */
4229 restoreTimeLineHistoryFiles(oldtarget + 1, newtarget);
4230
4231 ereport(LOG,
4232 (errmsg("new target timeline is %u",
4233 recoveryTargetTLI)));
4234
4235 return true;
4236 }
4237
4238
4239 /*
4240 * Open a logfile segment for reading (during recovery).
4241 *
4242 * If source == XLOG_FROM_ARCHIVE, the segment is retrieved from archive.
4243 * Otherwise, it's assumed to be already available in pg_wal.
4244 */
4245 static int
4246 XLogFileRead(XLogSegNo segno, TimeLineID tli,
4247 XLogSource source, bool notfoundOk)
4248 {
4249 char xlogfname[MAXFNAMELEN];
4250 char activitymsg[MAXFNAMELEN + 16];
4251 char path[MAXPGPATH];
4252 int fd;
4253
4254 XLogFileName(xlogfname, tli, segno, wal_segment_size);
4255
4256 switch (source)
4257 {
4258 case XLOG_FROM_ARCHIVE:
4259 /* Report recovery progress in PS display */
4260 snprintf(activitymsg, sizeof(activitymsg), "waiting for %s",
4261 xlogfname);
4262 set_ps_display(activitymsg);
4263
4264 if (!RestoreArchivedFile(path, xlogfname,
4265 "RECOVERYXLOG",
4266 wal_segment_size,
4267 InRedo))
4268 return -1;
4269 break;
4270
4271 case XLOG_FROM_PG_WAL:
4272 case XLOG_FROM_STREAM:
4273 XLogFilePath(path, tli, segno, wal_segment_size);
4274 break;
4275
4276 default:
4277 elog(ERROR, "invalid XLogFileRead source %d", source);
4278 }
4279
4280 /*
4281 * If the segment was fetched from archival storage, replace the existing
4282 * xlog segment (if any) with the archival version.
4283 */
4284 if (source == XLOG_FROM_ARCHIVE)
4285 {
4286 Assert(!IsInstallXLogFileSegmentActive());
4287 KeepFileRestoredFromArchive(path, xlogfname);
4288
4289 /*
4290 * Set path to point at the new file in pg_wal.
4291 */
4292 snprintf(path, MAXPGPATH, XLOGDIR "/%s", xlogfname);
4293 }
4294
4295 fd = BasicOpenFile(path, O_RDONLY | PG_BINARY);
4296 if (fd >= 0)
4297 {
4298 /* Success! */
4299 curFileTLI = tli;
4300
4301 /* Report recovery progress in PS display */
4302 snprintf(activitymsg, sizeof(activitymsg), "recovering %s",
4303 xlogfname);
4304 set_ps_display(activitymsg);
4305
4306 /* Track source of data in assorted state variables */
4307 readSource = source;
4308 XLogReceiptSource = source;
4309 /* In FROM_STREAM case, caller tracks receipt time, not me */
4310 if (source != XLOG_FROM_STREAM)
4311 XLogReceiptTime = GetCurrentTimestamp();
4312
4313 return fd;
4314 }
4315 if (errno != ENOENT || !notfoundOk) /* unexpected failure? */
4316 ereport(PANIC,
4317 (errcode_for_file_access(),
4318 errmsg("could not open file \"%s\": %m", path)));
4319 return -1;
4320 }
4321
4322 /*
4323 * Open a logfile segment for reading (during recovery).
4324 *
4325 * This version searches for the segment with any TLI listed in expectedTLEs.
4326 */
4327 static int
4328 XLogFileReadAnyTLI(XLogSegNo segno, XLogSource source)
4329 {
4330 char path[MAXPGPATH];
4331 ListCell *cell;
4332 int fd;
4333 List *tles;
4334
4335 /*
4336 * Loop looking for a suitable timeline ID: we might need to read any of
4337 * the timelines listed in expectedTLEs.
4338 *
4339 * We expect curFileTLI on entry to be the TLI of the preceding file in
4340 * sequence, or 0 if there was no predecessor. We do not allow curFileTLI
4341 * to go backwards; this prevents us from picking up the wrong file when a
4342 * parent timeline extends to higher segment numbers than the child we
4343 * want to read.
4344 *
4345 * If we haven't read the timeline history file yet, read it now, so that
4346 * we know which TLIs to scan. We don't save the list in expectedTLEs,
4347 * however, unless we actually find a valid segment. That way if there is
4348 * neither a timeline history file nor a WAL segment in the archive, and
4349 * streaming replication is set up, we'll read the timeline history file
4350 * streamed from the primary when we start streaming, instead of
4351 * recovering with a dummy history generated here.
4352 */
4353 if (expectedTLEs)
4354 tles = expectedTLEs;
4355 else
4356 tles = readTimeLineHistory(recoveryTargetTLI);
4357
4358 foreach(cell, tles)
4359 {
4360 TimeLineHistoryEntry *hent = (TimeLineHistoryEntry *) lfirst(cell);
4361 TimeLineID tli = hent->tli;
4362
4363 if (tli < curFileTLI)
4364 break; /* don't bother looking at too-old TLIs */
4365
4366 /*
4367 * Skip scanning the timeline ID that the logfile segment to read
4368 * doesn't belong to
4369 */
4370 if (hent->begin != InvalidXLogRecPtr)
4371 {
4372 XLogSegNo beginseg = 0;
4373
4374 XLByteToSeg(hent->begin, beginseg, wal_segment_size);
4375
4376 /*
4377 * The logfile segment that doesn't belong to the timeline is
4378 * older or newer than the segment that the timeline started or
4379 * ended at, respectively. It's sufficient to check only the
4380 * starting segment of the timeline here. Since the timelines are
4381 * scanned in descending order in this loop, any segments newer
4382 * than the ending segment should belong to newer timeline and
4383 * have already been read before. So it's not necessary to check
4384 * the ending segment of the timeline here.
4385 */
4386 if (segno < beginseg)
4387 continue;
4388 }
4389
4390 if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE)
4391 {
4392 fd = XLogFileRead(segno, tli, XLOG_FROM_ARCHIVE, true);
4393 if (fd != -1)
4394 {
4395 elog(DEBUG1, "got WAL segment from archive");
4396 if (!expectedTLEs)
4397 expectedTLEs = tles;
4398 return fd;
4399 }
4400 }
4401
4402 if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_WAL)
4403 {
4404 fd = XLogFileRead(segno, tli, XLOG_FROM_PG_WAL, true);
4405 if (fd != -1)
4406 {
4407 if (!expectedTLEs)
4408 expectedTLEs = tles;
4409 return fd;
4410 }
4411 }
4412 }
4413
4414 /* Couldn't find it. For simplicity, complain about front timeline */
4415 XLogFilePath(path, recoveryTargetTLI, segno, wal_segment_size);
4416 errno = ENOENT;
4417 ereport(DEBUG2,
4418 (errcode_for_file_access(),
4419 errmsg("could not open file \"%s\": %m", path)));
4420 return -1;
4421 }
4422
4423 /*
4424 * Set flag to signal the walreceiver to restart. (The startup process calls
4425 * this on noticing a relevant configuration change.)
4426 */
4427 void
4428 StartupRequestWalReceiverRestart(void)
4429 {
4430 if (currentSource == XLOG_FROM_STREAM && WalRcvRunning())
4431 {
4432 ereport(LOG,
4433 (errmsg("WAL receiver process shutdown requested")));
4434
4435 pendingWalRcvRestart = true;
4436 }
4437 }
4438
4439
4440 /*
4441 * Has a standby promotion already been triggered?
4442 *
4443 * Unlike CheckForStandbyTrigger(), this works in any process
4444 * that's connected to shared memory.
4445 */
4446 bool
4447 PromoteIsTriggered(void)
4448 {
4449 /*
4450 * We check shared state each time only until a standby promotion is
4451 * triggered. We can't trigger a promotion again, so there's no need to
4452 * keep checking after the shared variable has once been seen true.
4453 */
4454 if (LocalPromoteIsTriggered)
4455 return true;
4456
4457 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4458 LocalPromoteIsTriggered = XLogRecoveryCtl->SharedPromoteIsTriggered;
4459 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4460
4461 return LocalPromoteIsTriggered;
4462 }
4463
4464 static void
4465 SetPromoteIsTriggered(void)
4466 {
4467 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4468 XLogRecoveryCtl->SharedPromoteIsTriggered = true;
4469 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4470
4471 /*
4472 * Mark the recovery pause state as 'not paused' because the paused state
4473 * ends and promotion continues if a promotion is triggered while recovery
4474 * is paused. Otherwise pg_get_wal_replay_pause_state() can mistakenly
4475 * return 'paused' while a promotion is ongoing.
4476 */
4477 SetRecoveryPause(false);
4478
4479 LocalPromoteIsTriggered = true;
4480 }
4481
4482 /*
4483 * Check whether a promote request has arrived.
4484 */
4485 static bool
4486 CheckForStandbyTrigger(void)
4487 {
4488 if (LocalPromoteIsTriggered)
4489 return true;
4490
4491 if (IsPromoteSignaled() && CheckPromoteSignal())
4492 {
4493 ereport(LOG, (errmsg("received promote request")));
4494 RemovePromoteSignalFiles();
4495 ResetPromoteSignaled();
4496 SetPromoteIsTriggered();
4497 return true;
4498 }
4499
4500 return false;
4501 }
4502
4503 /*
4504 * Remove the files signaling a standby promotion request.
4505 */
4506 void
4507 RemovePromoteSignalFiles(void)
4508 {
4509 unlink(PROMOTE_SIGNAL_FILE);
4510 }
4511
4512 /*
4513 * Check to see if a promote request has arrived.
4514 */
4515 bool
4516 CheckPromoteSignal(void)
4517 {
4518 struct stat stat_buf;
4519
4520 if (stat(PROMOTE_SIGNAL_FILE, &stat_buf) == 0)
4521 return true;
4522
4523 return false;
4524 }
4525
4526 /*
4527 * Wake up startup process to replay newly arrived WAL, or to notice that
4528 * failover has been requested.
4529 */
4530 void
4531 WakeupRecovery(void)
4532 {
4533 SetLatch(&XLogRecoveryCtl->recoveryWakeupLatch);
4534 }
4535
4536 /*
4537 * Schedule a walreceiver wakeup in the main recovery loop.
4538 */
4539 void
4540 XLogRequestWalReceiverReply(void)
4541 {
4542 doRequestWalReceiverReply = true;
4543 }
4544
4545 /*
4546 * Is HotStandby active yet? This is only important in special backends
4547 * since normal backends won't ever be able to connect until this returns
4548 * true. Postmaster knows this by way of signal, not via shared memory.
4549 *
4550 * Unlike testing standbyState, this works in any process that's connected to
4551 * shared memory. (And note that standbyState alone doesn't tell the truth
4552 * anyway.)
4553 */
4554 bool
4555 HotStandbyActive(void)
4556 {
4557 /*
4558 * We check shared state each time only until Hot Standby is active. We
4559 * can't de-activate Hot Standby, so there's no need to keep checking
4560 * after the shared variable has once been seen true.
4561 */
4562 if (LocalHotStandbyActive)
4563 return true;
4564 else
4565 {
4566 /* spinlock is essential on machines with weak memory ordering! */
4567 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4568 LocalHotStandbyActive = XLogRecoveryCtl->SharedHotStandbyActive;
4569 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4570
4571 return LocalHotStandbyActive;
4572 }
4573 }
4574
4575 /*
4576 * Like HotStandbyActive(), but to be used only in WAL replay code,
4577 * where we don't need to ask any other process what the state is.
4578 */
4579 static bool
4580 HotStandbyActiveInReplay(void)
4581 {
4582 Assert(AmStartupProcess() || !IsPostmasterEnvironment);
4583 return LocalHotStandbyActive;
4584 }
4585
4586 /*
4587 * Get latest redo apply position.
4588 *
4589 * Exported to allow WALReceiver to read the pointer directly.
4590 */
4591 XLogRecPtr
4592 GetXLogReplayRecPtr(TimeLineID *replayTLI)
4593 {
4594 XLogRecPtr recptr;
4595 TimeLineID tli;
4596
4597 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4598 recptr = XLogRecoveryCtl->lastReplayedEndRecPtr;
4599 tli = XLogRecoveryCtl->lastReplayedTLI;
4600 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4601
4602 if (replayTLI)
4603 *replayTLI = tli;
4604 return recptr;
4605 }
4606
4607
4608 /*
4609 * Get position of last applied, or the record being applied.
4610 *
4611 * This is different from GetXLogReplayRecPtr() in that if a WAL
4612 * record is currently being applied, this includes that record.
4613 */
4614 XLogRecPtr
4615 GetCurrentReplayRecPtr(TimeLineID *replayEndTLI)
4616 {
4617 XLogRecPtr recptr;
4618 TimeLineID tli;
4619
4620 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4621 recptr = XLogRecoveryCtl->replayEndRecPtr;
4622 tli = XLogRecoveryCtl->replayEndTLI;
4623 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4624
4625 if (replayEndTLI)
4626 *replayEndTLI = tli;
4627 return recptr;
4628 }
4629
4630 /*
4631 * Save timestamp of latest processed commit/abort record.
4632 *
4633 * We keep this in XLogRecoveryCtl, not a simple static variable, so that it can be
4634 * seen by processes other than the startup process. Note in particular
4635 * that CreateRestartPoint is executed in the checkpointer.
4636 */
4637 static void
4638 SetLatestXTime(TimestampTz xtime)
4639 {
4640 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4641 XLogRecoveryCtl->recoveryLastXTime = xtime;
4642 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4643 }
4644
4645 /*
4646 * Fetch timestamp of latest processed commit/abort record.
4647 */
4648 TimestampTz
4649 GetLatestXTime(void)
4650 {
4651 TimestampTz xtime;
4652
4653 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4654 xtime = XLogRecoveryCtl->recoveryLastXTime;
4655 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4656
4657 return xtime;
4658 }
4659
4660 /*
4661 * Save timestamp of the next chunk of WAL records to apply.
4662 *
4663 * We keep this in XLogRecoveryCtl, not a simple static variable, so that it can be
4664 * seen by all backends.
4665 */
4666 static void
4667 SetCurrentChunkStartTime(TimestampTz xtime)
4668 {
4669 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4670 XLogRecoveryCtl->currentChunkStartTime = xtime;
4671 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4672 }
4673
4674 /*
4675 * Fetch timestamp of latest processed commit/abort record.
4676 * Startup process maintains an accurate local copy in XLogReceiptTime
4677 */
4678 TimestampTz
4679 GetCurrentChunkReplayStartTime(void)
4680 {
4681 TimestampTz xtime;
4682
4683 SpinLockAcquire(&XLogRecoveryCtl->info_lck);
4684 xtime = XLogRecoveryCtl->currentChunkStartTime;
4685 SpinLockRelease(&XLogRecoveryCtl->info_lck);
4686
4687 return xtime;
4688 }
4689
4690 /*
4691 * Returns time of receipt of current chunk of XLOG data, as well as
4692 * whether it was received from streaming replication or from archives.
4693 */
4694 void
4695 GetXLogReceiptTime(TimestampTz *rtime, bool *fromStream)
4696 {
4697 /*
4698 * This must be executed in the startup process, since we don't export the
4699 * relevant state to shared memory.
4700 */
4701 Assert(InRecovery);
4702
4703 *rtime = XLogReceiptTime;
4704 *fromStream = (XLogReceiptSource == XLOG_FROM_STREAM);
4705 }
4706
4707 /*
4708 * Note that text field supplied is a parameter name and does not require
4709 * translation
4710 */
4711 void
4712 RecoveryRequiresIntParameter(const char *param_name, int currValue, int minValue)
4713 {
4714 if (currValue < minValue)
4715 {
4716 if (HotStandbyActiveInReplay())
4717 {
4718 bool warned_for_promote = false;
4719
4720 ereport(WARNING,
4721 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
4722 errmsg("hot standby is not possible because of insufficient parameter settings"),
4723 errdetail("%s = %d is a lower setting than on the primary server, where its value was %d.",
4724 param_name,
4725 currValue,
4726 minValue)));
4727
4728 SetRecoveryPause(true);
4729
4730 ereport(LOG,
4731 (errmsg("recovery has paused"),
4732 errdetail("If recovery is unpaused, the server will shut down."),
4733 errhint("You can then restart the server after making the necessary configuration changes.")));
4734
4735 while (GetRecoveryPauseState() != RECOVERY_NOT_PAUSED)
4736 {
4737 ProcessStartupProcInterrupts();
4738
4739 if (CheckForStandbyTrigger())
4740 {
4741 if (!warned_for_promote)
4742 ereport(WARNING,
4743 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
4744 errmsg("promotion is not possible because of insufficient parameter settings"),
4745
4746 /*
4747 * Repeat the detail from above so it's easy to find
4748 * in the log.
4749 */
4750 errdetail("%s = %d is a lower setting than on the primary server, where its value was %d.",
4751 param_name,
4752 currValue,
4753 minValue),
4754 errhint("Restart the server after making the necessary configuration changes.")));
4755 warned_for_promote = true;
4756 }
4757
4758 /*
4759 * If recovery pause is requested then set it paused. While
4760 * we are in the loop, user might resume and pause again so
4761 * set this every time.
4762 */
4763 ConfirmRecoveryPaused();
4764
4765 /*
4766 * We wait on a condition variable that will wake us as soon
4767 * as the pause ends, but we use a timeout so we can check the
4768 * above conditions periodically too.
4769 */
4770 ConditionVariableTimedSleep(&XLogRecoveryCtl->recoveryNotPausedCV, 1000,
4771 WAIT_EVENT_RECOVERY_PAUSE);
4772 }
4773 ConditionVariableCancelSleep();
4774 }
4775
4776 ereport(FATAL,
4777 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
4778 errmsg("recovery aborted because of insufficient parameter settings"),
4779 /* Repeat the detail from above so it's easy to find in the log. */
4780 errdetail("%s = %d is a lower setting than on the primary server, where its value was %d.",
4781 param_name,
4782 currValue,
4783 minValue),
4784 errhint("You can restart the server after making the necessary configuration changes.")));
4785 }
4786 }
4787
4788
4789 /*
4790 * GUC check_hook for primary_slot_name
4791 */
4792 bool
4793 check_primary_slot_name(char **newval, void **extra, GucSource source)
4794 {
4795 int err_code;
4796 char *err_msg = NULL;
4797 char *err_hint = NULL;
4798
4799 if (*newval && strcmp(*newval, "") != 0 &&
4800 !ReplicationSlotValidateNameInternal(*newval, &err_code, &err_msg,
4801 &err_hint))
4802 {
4803 GUC_check_errcode(err_code);
4804 GUC_check_errdetail("%s", err_msg);
4805 if (err_hint != NULL)
4806 GUC_check_errhint("%s", err_hint);
4807 return false;
4808 }
4809
4810 return true;
4811 }
4812
4813 /*
4814 * Recovery target settings: Only one of the several recovery_target* settings
4815 * may be set. Setting a second one results in an error. The global variable
4816 * recoveryTarget tracks which kind of recovery target was chosen. Other
4817 * variables store the actual target value (for example a string or a xid).
4818 * The assign functions of the parameters check whether a competing parameter
4819 * was already set. But we want to allow setting the same parameter multiple
4820 * times. We also want to allow unsetting a parameter and setting a different
4821 * one, so we unset recoveryTarget when the parameter is set to an empty
4822 * string.
4823 *
4824 * XXX this code is broken by design. Throwing an error from a GUC assign
4825 * hook breaks fundamental assumptions of guc.c. So long as all the variables
4826 * for which this can happen are PGC_POSTMASTER, the consequences are limited,
4827 * since we'd just abort postmaster startup anyway. Nonetheless it's likely
4828 * that we have odd behaviors such as unexpected GUC ordering dependencies.
4829 */
4830
4831 pg_noreturn static void
4832 error_multiple_recovery_targets(void)
4833 {
4834 ereport(ERROR,
4835 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
4836 errmsg("multiple recovery targets specified"),
4837 errdetail("At most one of \"recovery_target\", \"recovery_target_lsn\", \"recovery_target_name\", \"recovery_target_time\", \"recovery_target_xid\" may be set.")));
4838 }
4839
4840 /*
4841 * GUC check_hook for recovery_target
4842 */
4843 bool
4844 check_recovery_target(char **newval, void **extra, GucSource source)
4845 {
4846 if (strcmp(*newval, "immediate") != 0 && strcmp(*newval, "") != 0)
4847 {
4848 GUC_check_errdetail("The only allowed value is \"immediate\".");
4849 return false;
4850 }
4851 return true;
4852 }
4853
4854 /*
4855 * GUC assign_hook for recovery_target
4856 */
4857 void
4858 assign_recovery_target(const char *newval, void *extra)
4859 {
4860 if (recoveryTarget != RECOVERY_TARGET_UNSET &&
4861 recoveryTarget != RECOVERY_TARGET_IMMEDIATE)
4862 error_multiple_recovery_targets();
4863
4864 if (newval && strcmp(newval, "") != 0)
4865 recoveryTarget = RECOVERY_TARGET_IMMEDIATE;
4866 else
4867 recoveryTarget = RECOVERY_TARGET_UNSET;
4868 }
4869
4870 /*
4871 * GUC check_hook for recovery_target_lsn
4872 */
4873 bool
4874 check_recovery_target_lsn(char **newval, void **extra, GucSource source)
4875 {
4876 if (strcmp(*newval, "") != 0)
4877 {
4878 XLogRecPtr lsn;
4879 XLogRecPtr *myextra;
4880 bool have_error = false;
4881
4882 lsn = pg_lsn_in_internal(*newval, &have_error);
4883 if (have_error)
4884 return false;
4885
4886 myextra = (XLogRecPtr *) guc_malloc(LOG, sizeof(XLogRecPtr));
4887 if (!myextra)
4888 return false;
4889 *myextra = lsn;
4890 *extra = myextra;
4891 }
4892 return true;
4893 }
4894
4895 /*
4896 * GUC assign_hook for recovery_target_lsn
4897 */
4898 void
4899 assign_recovery_target_lsn(const char *newval, void *extra)
4900 {
4901 if (recoveryTarget != RECOVERY_TARGET_UNSET &&
4902 recoveryTarget != RECOVERY_TARGET_LSN)
4903 error_multiple_recovery_targets();
4904
4905 if (newval && strcmp(newval, "") != 0)
4906 {
4907 recoveryTarget = RECOVERY_TARGET_LSN;
4908 recoveryTargetLSN = *((XLogRecPtr *) extra);
4909 }
4910 else
4911 recoveryTarget = RECOVERY_TARGET_UNSET;
4912 }
4913
4914 /*
4915 * GUC check_hook for recovery_target_name
4916 */
4917 bool
4918 check_recovery_target_name(char **newval, void **extra, GucSource source)
4919 {
4920 /* Use the value of newval directly */
4921 if (strlen(*newval) >= MAXFNAMELEN)
4922 {
4923 GUC_check_errdetail("\"%s\" is too long (maximum %d characters).",
4924 "recovery_target_name", MAXFNAMELEN - 1);
4925 return false;
4926 }
4927 return true;
4928 }
4929
4930 /*
4931 * GUC assign_hook for recovery_target_name
4932 */
4933 void
4934 assign_recovery_target_name(const char *newval, void *extra)
4935 {
4936 if (recoveryTarget != RECOVERY_TARGET_UNSET &&
4937 recoveryTarget != RECOVERY_TARGET_NAME)
4938 error_multiple_recovery_targets();
4939
4940 if (newval && strcmp(newval, "") != 0)
4941 {
4942 recoveryTarget = RECOVERY_TARGET_NAME;
4943 recoveryTargetName = newval;
4944 }
4945 else
4946 recoveryTarget = RECOVERY_TARGET_UNSET;
4947 }
4948
4949 /*
4950 * GUC check_hook for recovery_target_time
4951 *
4952 * The interpretation of the recovery_target_time string can depend on the
4953 * time zone setting, so we need to wait until after all GUC processing is
4954 * done before we can do the final parsing of the string. This check function
4955 * only does a parsing pass to catch syntax errors, but we store the string
4956 * and parse it again when we need to use it.
4957 */
4958 bool
4959 check_recovery_target_time(char **newval, void **extra, GucSource source)
4960 {
4961 if (strcmp(*newval, "") != 0)
4962 {
4963 /* reject some special values */
4964 if (strcmp(*newval, "now") == 0 ||
4965 strcmp(*newval, "today") == 0 ||
4966 strcmp(*newval, "tomorrow") == 0 ||
4967 strcmp(*newval, "yesterday") == 0)
4968 {
4969 return false;
4970 }
4971
4972 /*
4973 * parse timestamp value (see also timestamptz_in())
4974 */
4975 {
4976 char *str = *newval;
4977 fsec_t fsec;
4978 struct pg_tm tt,
4979 *tm = &tt;
4980 int tz;
4981 int dtype;
4982 int nf;
4983 int dterr;
4984 char *field[MAXDATEFIELDS];
4985 int ftype[MAXDATEFIELDS];
4986 char workbuf[MAXDATELEN + MAXDATEFIELDS];
4987 DateTimeErrorExtra dtextra;
4988 TimestampTz timestamp;
4989
4990 dterr = ParseDateTime(str, workbuf, sizeof(workbuf),
4991 field, ftype, MAXDATEFIELDS, &nf);
4992 if (dterr == 0)
4993 dterr = DecodeDateTime(field, ftype, nf,
4994 &dtype, tm, &fsec, &tz, &dtextra);
4995 if (dterr != 0)
4996 return false;
4997 if (dtype != DTK_DATE)
4998 return false;
4999
5000 if (tm2timestamp(tm, fsec, &tz, &timestamp) != 0)
5001 {
5002 GUC_check_errdetail("Timestamp out of range: \"%s\".", str);
5003 return false;
5004 }
5005 }
5006 }
5007 return true;
5008 }
5009
5010 /*
5011 * GUC assign_hook for recovery_target_time
5012 */
5013 void
5014 assign_recovery_target_time(const char *newval, void *extra)
5015 {
5016 if (recoveryTarget != RECOVERY_TARGET_UNSET &&
5017 recoveryTarget != RECOVERY_TARGET_TIME)
5018 error_multiple_recovery_targets();
5019
5020 if (newval && strcmp(newval, "") != 0)
5021 recoveryTarget = RECOVERY_TARGET_TIME;
5022 else
5023 recoveryTarget = RECOVERY_TARGET_UNSET;
5024 }
5025
5026 /*
5027 * GUC check_hook for recovery_target_timeline
5028 */
5029 bool
5030 check_recovery_target_timeline(char **newval, void **extra, GucSource source)
5031 {
5032 RecoveryTargetTimeLineGoal rttg;
5033 RecoveryTargetTimeLineGoal *myextra;
5034
5035 if (strcmp(*newval, "current") == 0)
5036 rttg = RECOVERY_TARGET_TIMELINE_CONTROLFILE;
5037 else if (strcmp(*newval, "latest") == 0)
5038 rttg = RECOVERY_TARGET_TIMELINE_LATEST;
5039 else
5040 {
5041 rttg = RECOVERY_TARGET_TIMELINE_NUMERIC;
5042
5043 errno = 0;
5044 strtoul(*newval, NULL, 0);
5045 if (errno == EINVAL || errno == ERANGE)
5046 {
5047 GUC_check_errdetail("\"recovery_target_timeline\" is not a valid number.");
5048 return false;
5049 }
5050 }
5051
5052 myextra = (RecoveryTargetTimeLineGoal *) guc_malloc(LOG, sizeof(RecoveryTargetTimeLineGoal));
5053 if (!myextra)
5054 return false;
5055 *myextra = rttg;
5056 *extra = myextra;
5057
5058 return true;
5059 }
5060
5061 /*
5062 * GUC assign_hook for recovery_target_timeline
5063 */
5064 void
5065 assign_recovery_target_timeline(const char *newval, void *extra)
5066 {
5067 recoveryTargetTimeLineGoal = *((RecoveryTargetTimeLineGoal *) extra);
5068 if (recoveryTargetTimeLineGoal == RECOVERY_TARGET_TIMELINE_NUMERIC)
5069 recoveryTargetTLIRequested = (TimeLineID) strtoul(newval, NULL, 0);
5070 else
5071 recoveryTargetTLIRequested = 0;
5072 }
5073
5074 /*
5075 * GUC check_hook for recovery_target_xid
5076 */
5077 bool
5078 check_recovery_target_xid(char **newval, void **extra, GucSource source)
5079 {
5080 if (strcmp(*newval, "") != 0)
5081 {
5082 TransactionId xid;
5083 TransactionId *myextra;
5084
5085 errno = 0;
5086 xid = (TransactionId) strtou64(*newval, NULL, 0);
5087 if (errno == EINVAL || errno == ERANGE)
5088 return false;
5089
5090 myextra = (TransactionId *) guc_malloc(LOG, sizeof(TransactionId));
5091 if (!myextra)
5092 return false;
5093 *myextra = xid;
5094 *extra = myextra;
5095 }
5096 return true;
5097 }
5098
5099 /*
5100 * GUC assign_hook for recovery_target_xid
5101 */
5102 void
5103 assign_recovery_target_xid(const char *newval, void *extra)
5104 {
5105 if (recoveryTarget != RECOVERY_TARGET_UNSET &&
5106 recoveryTarget != RECOVERY_TARGET_XID)
5107 error_multiple_recovery_targets();
5108
5109 if (newval && strcmp(newval, "") != 0)
5110 {
5111 recoveryTarget = RECOVERY_TARGET_XID;
5112 recoveryTargetXid = *((TransactionId *) extra);
5113 }
5114 else
5115 recoveryTarget = RECOVERY_TARGET_UNSET;
5116 }