| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* pgarch.c |
| 4 |
* |
| 5 |
* PostgreSQL WAL archiver |
| 6 |
* |
| 7 |
* All functions relating to archiver are included here |
| 8 |
* |
| 9 |
* - All functions executed by archiver process |
| 10 |
* |
| 11 |
* - archiver is forked from postmaster, and the two |
| 12 |
* processes then communicate using signals. All functions |
| 13 |
* executed by postmaster are included in this file. |
| 14 |
* |
| 15 |
* Initial author: Simon Riggs [email protected] |
| 16 |
* |
| 17 |
* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group |
| 18 |
* Portions Copyright (c) 1994, Regents of the University of California |
| 19 |
* |
| 20 |
* |
| 21 |
* IDENTIFICATION |
| 22 |
* src/backend/postmaster/pgarch.c |
| 23 |
* |
| 24 |
*------------------------------------------------------------------------- |
| 25 |
*/ |
| 26 |
#include "postgres.h" |
| 27 |
|
| 28 |
#include <time.h> |
| 29 |
#include <sys/stat.h> |
| 30 |
#include <unistd.h> |
| 31 |
|
| 32 |
#include "access/xlog.h" |
| 33 |
#include "access/xlog_internal.h" |
| 34 |
#include "archive/archive_module.h" |
| 35 |
#include "archive/shell_archive.h" |
| 36 |
#include "lib/binaryheap.h" |
| 37 |
#include "libpq/pqsignal.h" |
| 38 |
#include "pgstat.h" |
| 39 |
#include "postmaster/auxprocess.h" |
| 40 |
#include "postmaster/interrupt.h" |
| 41 |
#include "postmaster/pgarch.h" |
| 42 |
#include "storage/condition_variable.h" |
| 43 |
#include "storage/aio_subsys.h" |
| 44 |
#include "storage/fd.h" |
| 45 |
#include "storage/ipc.h" |
| 46 |
#include "storage/latch.h" |
| 47 |
#include "storage/pmsignal.h" |
| 48 |
#include "storage/proc.h" |
| 49 |
#include "storage/procsignal.h" |
| 50 |
#include "storage/shmem.h" |
| 51 |
#include "utils/guc.h" |
| 52 |
#include "utils/memutils.h" |
| 53 |
#include "utils/ps_status.h" |
| 54 |
#include "utils/resowner.h" |
| 55 |
#include "utils/timeout.h" |
| 56 |
|
| 57 |
|
| 58 |
/* ---------- |
| 59 |
* Timer definitions. |
| 60 |
* ---------- |
| 61 |
*/ |
| 62 |
#define PGARCH_AUTOWAKE_INTERVAL 60 /* How often to force a poll of the |
| 63 |
* archive status directory; in seconds. */ |
| 64 |
#define PGARCH_RESTART_INTERVAL 10 /* How often to attempt to restart a |
| 65 |
* failed archiver; in seconds. */ |
| 66 |
|
| 67 |
/* |
| 68 |
* Maximum number of retries allowed when attempting to archive a WAL |
| 69 |
* file. |
| 70 |
*/ |
| 71 |
#define NUM_ARCHIVE_RETRIES 3 |
| 72 |
|
| 73 |
/* |
| 74 |
* Maximum number of retries allowed when attempting to remove an |
| 75 |
* orphan archive status file. |
| 76 |
*/ |
| 77 |
#define NUM_ORPHAN_CLEANUP_RETRIES 3 |
| 78 |
|
| 79 |
/* |
| 80 |
* Maximum number of .ready files to gather per directory scan. |
| 81 |
*/ |
| 82 |
#define NUM_FILES_PER_DIRECTORY_SCAN 64 |
| 83 |
|
| 84 |
/* Shared memory area for archiver process */ |
| 85 |
typedef struct PgArchData |
| 86 |
{ |
| 87 |
int pgprocno; /* proc number of archiver process */ |
| 88 |
|
| 89 |
/* |
| 90 |
* Forces a directory scan in pgarch_readyXlog(). |
| 91 |
*/ |
| 92 |
pg_atomic_uint32 force_dir_scan; |
| 93 |
} PgArchData; |
| 94 |
|
| 95 |
char *XLogArchiveLibrary = ""; |
| 96 |
char *arch_module_check_errdetail_string; |
| 97 |
|
| 98 |
|
| 99 |
/* ---------- |
| 100 |
* Local data |
| 101 |
* ---------- |
| 102 |
*/ |
| 103 |
static time_t last_sigterm_time = 0; |
| 104 |
static PgArchData *PgArch = NULL; |
| 105 |
static const ArchiveModuleCallbacks *ArchiveCallbacks; |
| 106 |
static ArchiveModuleState *archive_module_state; |
| 107 |
static MemoryContext archive_context; |
| 108 |
|
| 109 |
|
| 110 |
/* |
| 111 |
* Stuff for tracking multiple files to archive from each scan of |
| 112 |
* archive_status. Minimizing the number of directory scans when there are |
| 113 |
* many files to archive can significantly improve archival rate. |
| 114 |
* |
| 115 |
* arch_heap is a max-heap that is used during the directory scan to track |
| 116 |
* the highest-priority files to archive. After the directory scan |
| 117 |
* completes, the file names are stored in ascending order of priority in |
| 118 |
* arch_files. pgarch_readyXlog() returns files from arch_files until it |
| 119 |
* is empty, at which point another directory scan must be performed. |
| 120 |
* |
| 121 |
* We only need this data in the archiver process, so make it a palloc'd |
| 122 |
* struct rather than a bunch of static arrays. |
| 123 |
*/ |
| 124 |
struct arch_files_state |
| 125 |
{ |
| 126 |
binaryheap *arch_heap; |
| 127 |
int arch_files_size; /* number of live entries in arch_files[] */ |
| 128 |
char *arch_files[NUM_FILES_PER_DIRECTORY_SCAN]; |
| 129 |
/* buffers underlying heap, and later arch_files[], entries: */ |
| 130 |
char arch_filenames[NUM_FILES_PER_DIRECTORY_SCAN][MAX_XFN_CHARS + 1]; |
| 131 |
}; |
| 132 |
|
| 133 |
static struct arch_files_state *arch_files = NULL; |
| 134 |
|
| 135 |
/* |
| 136 |
* Flags set by interrupt handlers for later service in the main loop. |
| 137 |
*/ |
| 138 |
static volatile sig_atomic_t ready_to_stop = false; |
| 139 |
|
| 140 |
/* ---------- |
| 141 |
* Local function forward declarations |
| 142 |
* ---------- |
| 143 |
*/ |
| 144 |
static void pgarch_waken_stop(SIGNAL_ARGS); |
| 145 |
static void pgarch_MainLoop(void); |
| 146 |
static void pgarch_ArchiverCopyLoop(void); |
| 147 |
static bool pgarch_archiveXlog(char *xlog); |
| 148 |
static bool pgarch_readyXlog(char *xlog); |
| 149 |
static void pgarch_archiveDone(char *xlog); |
| 150 |
static void pgarch_die(int code, Datum arg); |
| 151 |
static void ProcessPgArchInterrupts(void); |
| 152 |
static int ready_file_comparator(Datum a, Datum b, void *arg); |
| 153 |
static void LoadArchiveLibrary(void); |
| 154 |
static void pgarch_call_module_shutdown_cb(int code, Datum arg); |
| 155 |
|
| 156 |
/* Report shared memory space needed by PgArchShmemInit */ |
| 157 |
Size |
| 158 |
PgArchShmemSize(void) |
| 159 |
{ |
| 160 |
Size size = 0; |
| 161 |
|
| 162 |
size = add_size(size, sizeof(PgArchData)); |
| 163 |
|
| 164 |
return size; |
| 165 |
} |
| 166 |
|
| 167 |
/* Allocate and initialize archiver-related shared memory */ |
| 168 |
void |
| 169 |
PgArchShmemInit(void) |
| 170 |
{ |
| 171 |
bool found; |
| 172 |
|
| 173 |
PgArch = (PgArchData *) |
| 174 |
ShmemInitStruct("Archiver Data", PgArchShmemSize(), &found); |
| 175 |
|
| 176 |
if (!found) |
| 177 |
{ |
| 178 |
/* First time through, so initialize */ |
| 179 |
MemSet(PgArch, 0, PgArchShmemSize()); |
| 180 |
PgArch->pgprocno = INVALID_PROC_NUMBER; |
| 181 |
pg_atomic_init_u32(&PgArch->force_dir_scan, 0); |
| 182 |
} |
| 183 |
} |
| 184 |
|
| 185 |
/* |
| 186 |
* PgArchCanRestart |
| 187 |
* |
| 188 |
* Return true and archiver is allowed to restart if enough time has |
| 189 |
* passed since it was launched last to reach PGARCH_RESTART_INTERVAL. |
| 190 |
* Otherwise return false. |
| 191 |
* |
| 192 |
* This is a safety valve to protect against continuous respawn attempts if the |
| 193 |
* archiver is dying immediately at launch. Note that since we will retry to |
| 194 |
* launch the archiver from the postmaster main loop, we will get another |
| 195 |
* chance later. |
| 196 |
*/ |
| 197 |
bool |
| 198 |
PgArchCanRestart(void) |
| 199 |
{ |
| 200 |
static time_t last_pgarch_start_time = 0; |
| 201 |
time_t curtime = time(NULL); |
| 202 |
|
| 203 |
/* |
| 204 |
* Return false and don't restart archiver if too soon since last archiver |
| 205 |
* start. |
| 206 |
*/ |
| 207 |
if ((unsigned int) (curtime - last_pgarch_start_time) < |
| 208 |
(unsigned int) PGARCH_RESTART_INTERVAL) |
| 209 |
return false; |
| 210 |
|
| 211 |
last_pgarch_start_time = curtime; |
| 212 |
return true; |
| 213 |
} |
| 214 |
|
| 215 |
|
| 216 |
/* Main entry point for archiver process */ |
| 217 |
void |
| 218 |
PgArchiverMain(const void *startup_data, size_t startup_data_len) |
| 219 |
{ |
| 220 |
Assert(startup_data_len == 0); |
| 221 |
|
| 222 |
MyBackendType = B_ARCHIVER; |
| 223 |
AuxiliaryProcessMainCommon(); |
| 224 |
|
| 225 |
/* |
| 226 |
* Ignore all signals usually bound to some action in the postmaster, |
| 227 |
* except for SIGHUP, SIGTERM, SIGUSR1, SIGUSR2, and SIGQUIT. |
| 228 |
*/ |
| 229 |
pqsignal(SIGHUP, SignalHandlerForConfigReload); |
| 230 |
pqsignal(SIGINT, SIG_IGN); |
| 231 |
pqsignal(SIGTERM, SignalHandlerForShutdownRequest); |
| 232 |
/* SIGQUIT handler was already set up by InitPostmasterChild */ |
| 233 |
pqsignal(SIGALRM, SIG_IGN); |
| 234 |
pqsignal(SIGPIPE, SIG_IGN); |
| 235 |
pqsignal(SIGUSR1, procsignal_sigusr1_handler); |
| 236 |
pqsignal(SIGUSR2, pgarch_waken_stop); |
| 237 |
|
| 238 |
/* Reset some signals that are accepted by postmaster but not here */ |
| 239 |
pqsignal(SIGCHLD, SIG_DFL); |
| 240 |
|
| 241 |
/* Unblock signals (they were blocked when the postmaster forked us) */ |
| 242 |
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL); |
| 243 |
|
| 244 |
/* We shouldn't be launched unnecessarily. */ |
| 245 |
Assert(XLogArchivingActive()); |
| 246 |
|
| 247 |
/* Arrange to clean up at archiver exit */ |
| 248 |
on_shmem_exit(pgarch_die, 0); |
| 249 |
|
| 250 |
/* |
| 251 |
* Advertise our proc number so that backends can use our latch to wake us |
| 252 |
* up while we're sleeping. |
| 253 |
*/ |
| 254 |
PgArch->pgprocno = MyProcNumber; |
| 255 |
|
| 256 |
/* Create workspace for pgarch_readyXlog() */ |
| 257 |
arch_files = palloc(sizeof(struct arch_files_state)); |
| 258 |
arch_files->arch_files_size = 0; |
| 259 |
|
| 260 |
/* Initialize our max-heap for prioritizing files to archive. */ |
| 261 |
arch_files->arch_heap = binaryheap_allocate(NUM_FILES_PER_DIRECTORY_SCAN, |
| 262 |
ready_file_comparator, NULL); |
| 263 |
|
| 264 |
/* Initialize our memory context. */ |
| 265 |
archive_context = AllocSetContextCreate(TopMemoryContext, |
| 266 |
"archiver", |
| 267 |
ALLOCSET_DEFAULT_SIZES); |
| 268 |
|
| 269 |
/* Load the archive_library. */ |
| 270 |
LoadArchiveLibrary(); |
| 271 |
|
| 272 |
pgarch_MainLoop(); |
| 273 |
|
| 274 |
proc_exit(0); |
| 275 |
} |
| 276 |
|
| 277 |
/* |
| 278 |
* Wake up the archiver |
| 279 |
*/ |
| 280 |
void |
| 281 |
PgArchWakeup(void) |
| 282 |
{ |
| 283 |
int arch_pgprocno = PgArch->pgprocno; |
| 284 |
|
| 285 |
/* |
| 286 |
* We don't acquire ProcArrayLock here. It's actually fine because |
| 287 |
* procLatch isn't ever freed, so we just can potentially set the wrong |
| 288 |
* process' (or no process') latch. Even in that case the archiver will |
| 289 |
* be relaunched shortly and will start archiving. |
| 290 |
*/ |
| 291 |
if (arch_pgprocno != INVALID_PROC_NUMBER) |
| 292 |
SetLatch(&ProcGlobal->allProcs[arch_pgprocno].procLatch); |
| 293 |
} |
| 294 |
|
| 295 |
|
| 296 |
/* SIGUSR2 signal handler for archiver process */ |
| 297 |
static void |
| 298 |
pgarch_waken_stop(SIGNAL_ARGS) |
| 299 |
{ |
| 300 |
/* set flag to do a final cycle and shut down afterwards */ |
| 301 |
ready_to_stop = true; |
| 302 |
SetLatch(MyLatch); |
| 303 |
} |
| 304 |
|
| 305 |
/* |
| 306 |
* pgarch_MainLoop |
| 307 |
* |
| 308 |
* Main loop for archiver |
| 309 |
*/ |
| 310 |
static void |
| 311 |
pgarch_MainLoop(void) |
| 312 |
{ |
| 313 |
bool time_to_stop; |
| 314 |
|
| 315 |
/* |
| 316 |
* There shouldn't be anything for the archiver to do except to wait for a |
| 317 |
* signal ... however, the archiver exists to protect our data, so it |
| 318 |
* wakes up occasionally to allow itself to be proactive. |
| 319 |
*/ |
| 320 |
do |
| 321 |
{ |
| 322 |
ResetLatch(MyLatch); |
| 323 |
|
| 324 |
/* When we get SIGUSR2, we do one more archive cycle, then exit */ |
| 325 |
time_to_stop = ready_to_stop; |
| 326 |
|
| 327 |
/* Check for barrier events and config update */ |
| 328 |
ProcessPgArchInterrupts(); |
| 329 |
|
| 330 |
/* |
| 331 |
* If we've gotten SIGTERM, we normally just sit and do nothing until |
| 332 |
* SIGUSR2 arrives. However, that means a random SIGTERM would |
| 333 |
* disable archiving indefinitely, which doesn't seem like a good |
| 334 |
* idea. If more than 60 seconds pass since SIGTERM, exit anyway, so |
| 335 |
* that the postmaster can start a new archiver if needed. |
| 336 |
*/ |
| 337 |
if (ShutdownRequestPending) |
| 338 |
{ |
| 339 |
time_t curtime = time(NULL); |
| 340 |
|
| 341 |
if (last_sigterm_time == 0) |
| 342 |
last_sigterm_time = curtime; |
| 343 |
else if ((unsigned int) (curtime - last_sigterm_time) >= |
| 344 |
(unsigned int) 60) |
| 345 |
break; |
| 346 |
} |
| 347 |
|
| 348 |
/* Do what we're here for */ |
| 349 |
pgarch_ArchiverCopyLoop(); |
| 350 |
|
| 351 |
/* |
| 352 |
* Sleep until a signal is received, or until a poll is forced by |
| 353 |
* PGARCH_AUTOWAKE_INTERVAL, or until postmaster dies. |
| 354 |
*/ |
| 355 |
if (!time_to_stop) /* Don't wait during last iteration */ |
| 356 |
{ |
| 357 |
int rc; |
| 358 |
|
| 359 |
rc = WaitLatch(MyLatch, |
| 360 |
WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, |
| 361 |
PGARCH_AUTOWAKE_INTERVAL * 1000L, |
| 362 |
WAIT_EVENT_ARCHIVER_MAIN); |
| 363 |
if (rc & WL_POSTMASTER_DEATH) |
| 364 |
time_to_stop = true; |
| 365 |
} |
| 366 |
|
| 367 |
/* |
| 368 |
* The archiver quits either when the postmaster dies (not expected) |
| 369 |
* or after completing one more archiving cycle after receiving |
| 370 |
* SIGUSR2. |
| 371 |
*/ |
| 372 |
} while (!time_to_stop); |
| 373 |
} |
| 374 |
|
| 375 |
/* |
| 376 |
* pgarch_ArchiverCopyLoop |
| 377 |
* |
| 378 |
* Archives all outstanding xlogs then returns |
| 379 |
*/ |
| 380 |
static void |
| 381 |
pgarch_ArchiverCopyLoop(void) |
| 382 |
{ |
| 383 |
char xlog[MAX_XFN_CHARS + 1]; |
| 384 |
|
| 385 |
/* force directory scan in the first call to pgarch_readyXlog() */ |
| 386 |
arch_files->arch_files_size = 0; |
| 387 |
|
| 388 |
/* |
| 389 |
* loop through all xlogs with archive_status of .ready and archive |
| 390 |
* them...mostly we expect this to be a single file, though it is possible |
| 391 |
* some backend will add files onto the list of those that need archiving |
| 392 |
* while we are still copying earlier archives |
| 393 |
*/ |
| 394 |
while (pgarch_readyXlog(xlog)) |
| 395 |
{ |
| 396 |
int failures = 0; |
| 397 |
int failures_orphan = 0; |
| 398 |
|
| 399 |
for (;;) |
| 400 |
{ |
| 401 |
struct stat stat_buf; |
| 402 |
char pathname[MAXPGPATH]; |
| 403 |
|
| 404 |
/* |
| 405 |
* Do not initiate any more archive commands after receiving |
| 406 |
* SIGTERM, nor after the postmaster has died unexpectedly. The |
| 407 |
* first condition is to try to keep from having init SIGKILL the |
| 408 |
* command, and the second is to avoid conflicts with another |
| 409 |
* archiver spawned by a newer postmaster. |
| 410 |
*/ |
| 411 |
if (ShutdownRequestPending || !PostmasterIsAlive()) |
| 412 |
return; |
| 413 |
|
| 414 |
/* |
| 415 |
* Check for barrier events and config update. This is so that |
| 416 |
* we'll adopt a new setting for archive_command as soon as |
| 417 |
* possible, even if there is a backlog of files to be archived. |
| 418 |
*/ |
| 419 |
ProcessPgArchInterrupts(); |
| 420 |
|
| 421 |
/* Reset variables that might be set by the callback */ |
| 422 |
arch_module_check_errdetail_string = NULL; |
| 423 |
|
| 424 |
/* can't do anything if not configured ... */ |
| 425 |
if (ArchiveCallbacks->check_configured_cb != NULL && |
| 426 |
!ArchiveCallbacks->check_configured_cb(archive_module_state)) |
| 427 |
{ |
| 428 |
ereport(WARNING, |
| 429 |
(errmsg("\"archive_mode\" enabled, yet archiving is not configured"), |
| 430 |
arch_module_check_errdetail_string ? |
| 431 |
errdetail_internal("%s", arch_module_check_errdetail_string) : 0)); |
| 432 |
return; |
| 433 |
} |
| 434 |
|
| 435 |
/* |
| 436 |
* Since archive status files are not removed in a durable manner, |
| 437 |
* a system crash could leave behind .ready files for WAL segments |
| 438 |
* that have already been recycled or removed. In this case, |
| 439 |
* simply remove the orphan status file and move on. unlink() is |
| 440 |
* used here as even on subsequent crashes the same orphan files |
| 441 |
* would get removed, so there is no need to worry about |
| 442 |
* durability. |
| 443 |
*/ |
| 444 |
snprintf(pathname, MAXPGPATH, XLOGDIR "/%s", xlog); |
| 445 |
if (stat(pathname, &stat_buf) != 0 && errno == ENOENT) |
| 446 |
{ |
| 447 |
char xlogready[MAXPGPATH]; |
| 448 |
|
| 449 |
StatusFilePath(xlogready, xlog, ".ready"); |
| 450 |
if (unlink(xlogready) == 0) |
| 451 |
{ |
| 452 |
ereport(WARNING, |
| 453 |
(errmsg("removed orphan archive status file \"%s\"", |
| 454 |
xlogready))); |
| 455 |
|
| 456 |
/* leave loop and move to the next status file */ |
| 457 |
break; |
| 458 |
} |
| 459 |
|
| 460 |
if (++failures_orphan >= NUM_ORPHAN_CLEANUP_RETRIES) |
| 461 |
{ |
| 462 |
ereport(WARNING, |
| 463 |
(errmsg("removal of orphan archive status file \"%s\" failed too many times, will try again later", |
| 464 |
xlogready))); |
| 465 |
|
| 466 |
/* give up cleanup of orphan status files */ |
| 467 |
return; |
| 468 |
} |
| 469 |
|
| 470 |
/* wait a bit before retrying */ |
| 471 |
pg_usleep(1000000L); |
| 472 |
continue; |
| 473 |
} |
| 474 |
|
| 475 |
if (pgarch_archiveXlog(xlog)) |
| 476 |
{ |
| 477 |
/* successful */ |
| 478 |
pgarch_archiveDone(xlog); |
| 479 |
|
| 480 |
/* |
| 481 |
* Tell the cumulative stats system about the WAL file that we |
| 482 |
* successfully archived |
| 483 |
*/ |
| 484 |
pgstat_report_archiver(xlog, false); |
| 485 |
|
| 486 |
break; /* out of inner retry loop */ |
| 487 |
} |
| 488 |
else |
| 489 |
{ |
| 490 |
/* |
| 491 |
* Tell the cumulative stats system about the WAL file that we |
| 492 |
* failed to archive |
| 493 |
*/ |
| 494 |
pgstat_report_archiver(xlog, true); |
| 495 |
|
| 496 |
if (++failures >= NUM_ARCHIVE_RETRIES) |
| 497 |
{ |
| 498 |
ereport(WARNING, |
| 499 |
(errmsg("archiving write-ahead log file \"%s\" failed too many times, will try again later", |
| 500 |
xlog))); |
| 501 |
return; /* give up archiving for now */ |
| 502 |
} |
| 503 |
pg_usleep(1000000L); /* wait a bit before retrying */ |
| 504 |
} |
| 505 |
} |
| 506 |
} |
| 507 |
} |
| 508 |
|
| 509 |
/* |
| 510 |
* pgarch_archiveXlog |
| 511 |
* |
| 512 |
* Invokes archive_file_cb to copy one archive file to wherever it should go |
| 513 |
* |
| 514 |
* Returns true if successful |
| 515 |
*/ |
| 516 |
static bool |
| 517 |
pgarch_archiveXlog(char *xlog) |
| 518 |
{ |
| 519 |
sigjmp_buf local_sigjmp_buf; |
| 520 |
MemoryContext oldcontext; |
| 521 |
char pathname[MAXPGPATH]; |
| 522 |
char activitymsg[MAXFNAMELEN + 16]; |
| 523 |
bool ret; |
| 524 |
|
| 525 |
snprintf(pathname, MAXPGPATH, XLOGDIR "/%s", xlog); |
| 526 |
|
| 527 |
/* Report archive activity in PS display */ |
| 528 |
snprintf(activitymsg, sizeof(activitymsg), "archiving %s", xlog); |
| 529 |
set_ps_display(activitymsg); |
| 530 |
|
| 531 |
oldcontext = MemoryContextSwitchTo(archive_context); |
| 532 |
|
| 533 |
/* |
| 534 |
* Since the archiver operates at the bottom of the exception stack, |
| 535 |
* ERRORs turn into FATALs and cause the archiver process to restart. |
| 536 |
* However, using ereport(ERROR, ...) when there are problems is easy to |
| 537 |
* code and maintain. Therefore, we create our own exception handler to |
| 538 |
* catch ERRORs and return false instead of restarting the archiver |
| 539 |
* whenever there is a failure. |
| 540 |
* |
| 541 |
* We assume ERRORs from the archiving callback are the most common |
| 542 |
* exceptions experienced by the archiver, so we opt to handle exceptions |
| 543 |
* here instead of PgArchiverMain() to avoid reinitializing the archiver |
| 544 |
* too frequently. We could instead add a sigsetjmp() block to |
| 545 |
* PgArchiverMain() and use PG_TRY/PG_CATCH here, but the extra code to |
| 546 |
* avoid the odd archiver restart doesn't seem worth it. |
| 547 |
*/ |
| 548 |
if (sigsetjmp(local_sigjmp_buf, 1) != 0) |
| 549 |
{ |
| 550 |
/* Since not using PG_TRY, must reset error stack by hand */ |
| 551 |
error_context_stack = NULL; |
| 552 |
|
| 553 |
/* Prevent interrupts while cleaning up */ |
| 554 |
HOLD_INTERRUPTS(); |
| 555 |
|
| 556 |
/* Report the error to the server log. */ |
| 557 |
EmitErrorReport(); |
| 558 |
|
| 559 |
/* |
| 560 |
* Try to clean up anything the archive module left behind. We try to |
| 561 |
* cover anything that an archive module could conceivably have left |
| 562 |
* behind, but it is of course possible that modules could be doing |
| 563 |
* unexpected things that require additional cleanup. Module authors |
| 564 |
* should be sure to do any extra required cleanup in a PG_CATCH block |
| 565 |
* within the archiving callback, and they are encouraged to notify |
| 566 |
* the pgsql-hackers mailing list so that we can add it here. |
| 567 |
*/ |
| 568 |
disable_all_timeouts(false); |
| 569 |
LWLockReleaseAll(); |
| 570 |
ConditionVariableCancelSleep(); |
| 571 |
pgstat_report_wait_end(); |
| 572 |
pgaio_error_cleanup(); |
| 573 |
ReleaseAuxProcessResources(false); |
| 574 |
AtEOXact_Files(false); |
| 575 |
AtEOXact_HashTables(false); |
| 576 |
|
| 577 |
/* |
| 578 |
* Return to the original memory context and clear ErrorContext for |
| 579 |
* next time. |
| 580 |
*/ |
| 581 |
MemoryContextSwitchTo(oldcontext); |
| 582 |
FlushErrorState(); |
| 583 |
|
| 584 |
/* Flush any leaked data */ |
| 585 |
MemoryContextReset(archive_context); |
| 586 |
|
| 587 |
/* Remove our exception handler */ |
| 588 |
PG_exception_stack = NULL; |
| 589 |
|
| 590 |
/* Now we can allow interrupts again */ |
| 591 |
RESUME_INTERRUPTS(); |
| 592 |
|
| 593 |
/* Report failure so that the archiver retries this file */ |
| 594 |
ret = false; |
| 595 |
} |
| 596 |
else |
| 597 |
{ |
| 598 |
/* Enable our exception handler */ |
| 599 |
PG_exception_stack = &local_sigjmp_buf; |
| 600 |
|
| 601 |
/* Archive the file! */ |
| 602 |
ret = ArchiveCallbacks->archive_file_cb(archive_module_state, |
| 603 |
xlog, pathname); |
| 604 |
|
| 605 |
/* Remove our exception handler */ |
| 606 |
PG_exception_stack = NULL; |
| 607 |
|
| 608 |
/* Reset our memory context and switch back to the original one */ |
| 609 |
MemoryContextSwitchTo(oldcontext); |
| 610 |
MemoryContextReset(archive_context); |
| 611 |
} |
| 612 |
|
| 613 |
if (ret) |
| 614 |
snprintf(activitymsg, sizeof(activitymsg), "last was %s", xlog); |
| 615 |
else |
| 616 |
snprintf(activitymsg, sizeof(activitymsg), "failed on %s", xlog); |
| 617 |
set_ps_display(activitymsg); |
| 618 |
|
| 619 |
return ret; |
| 620 |
} |
| 621 |
|
| 622 |
/* |
| 623 |
* pgarch_readyXlog |
| 624 |
* |
| 625 |
* Return name of the oldest xlog file that has not yet been archived. |
| 626 |
* No notification is set that file archiving is now in progress, so |
| 627 |
* this would need to be extended if multiple concurrent archival |
| 628 |
* tasks were created. If a failure occurs, we will completely |
| 629 |
* re-copy the file at the next available opportunity. |
| 630 |
* |
| 631 |
* It is important that we return the oldest, so that we archive xlogs |
| 632 |
* in order that they were written, for two reasons: |
| 633 |
* 1) to maintain the sequential chain of xlogs required for recovery |
| 634 |
* 2) because the oldest ones will sooner become candidates for |
| 635 |
* recycling at time of checkpoint |
| 636 |
* |
| 637 |
* NOTE: the "oldest" comparison will consider any .history file to be older |
| 638 |
* than any other file except another .history file. Segments on a timeline |
| 639 |
* with a smaller ID will be older than all segments on a timeline with a |
| 640 |
* larger ID; the net result being that past timelines are given higher |
| 641 |
* priority for archiving. This seems okay, or at least not obviously worth |
| 642 |
* changing. |
| 643 |
*/ |
| 644 |
static bool |
| 645 |
pgarch_readyXlog(char *xlog) |
| 646 |
{ |
| 647 |
char XLogArchiveStatusDir[MAXPGPATH]; |
| 648 |
DIR *rldir; |
| 649 |
struct dirent *rlde; |
| 650 |
|
| 651 |
/* |
| 652 |
* If a directory scan was requested, clear the stored file names and |
| 653 |
* proceed. |
| 654 |
*/ |
| 655 |
if (pg_atomic_exchange_u32(&PgArch->force_dir_scan, 0) == 1) |
| 656 |
arch_files->arch_files_size = 0; |
| 657 |
|
| 658 |
/* |
| 659 |
* If we still have stored file names from the previous directory scan, |
| 660 |
* try to return one of those. We check to make sure the status file is |
| 661 |
* still present, as the archive_command for a previous file may have |
| 662 |
* already marked it done. |
| 663 |
*/ |
| 664 |
while (arch_files->arch_files_size > 0) |
| 665 |
{ |
| 666 |
struct stat st; |
| 667 |
char status_file[MAXPGPATH]; |
| 668 |
char *arch_file; |
| 669 |
|
| 670 |
arch_files->arch_files_size--; |
| 671 |
arch_file = arch_files->arch_files[arch_files->arch_files_size]; |
| 672 |
StatusFilePath(status_file, arch_file, ".ready"); |
| 673 |
|
| 674 |
if (stat(status_file, &st) == 0) |
| 675 |
{ |
| 676 |
strcpy(xlog, arch_file); |
| 677 |
return true; |
| 678 |
} |
| 679 |
else if (errno != ENOENT) |
| 680 |
ereport(ERROR, |
| 681 |
(errcode_for_file_access(), |
| 682 |
errmsg("could not stat file \"%s\": %m", status_file))); |
| 683 |
} |
| 684 |
|
| 685 |
/* arch_heap is probably empty, but let's make sure */ |
| 686 |
binaryheap_reset(arch_files->arch_heap); |
| 687 |
|
| 688 |
/* |
| 689 |
* Open the archive status directory and read through the list of files |
| 690 |
* with the .ready suffix, looking for the earliest files. |
| 691 |
*/ |
| 692 |
snprintf(XLogArchiveStatusDir, MAXPGPATH, XLOGDIR "/archive_status"); |
| 693 |
rldir = AllocateDir(XLogArchiveStatusDir); |
| 694 |
|
| 695 |
while ((rlde = ReadDir(rldir, XLogArchiveStatusDir)) != NULL) |
| 696 |
{ |
| 697 |
int basenamelen = (int) strlen(rlde->d_name) - 6; |
| 698 |
char basename[MAX_XFN_CHARS + 1]; |
| 699 |
char *arch_file; |
| 700 |
|
| 701 |
/* Ignore entries with unexpected number of characters */ |
| 702 |
if (basenamelen < MIN_XFN_CHARS || |
| 703 |
basenamelen > MAX_XFN_CHARS) |
| 704 |
continue; |
| 705 |
|
| 706 |
/* Ignore entries with unexpected characters */ |
| 707 |
if (strspn(rlde->d_name, VALID_XFN_CHARS) < basenamelen) |
| 708 |
continue; |
| 709 |
|
| 710 |
/* Ignore anything not suffixed with .ready */ |
| 711 |
if (strcmp(rlde->d_name + basenamelen, ".ready") != 0) |
| 712 |
continue; |
| 713 |
|
| 714 |
/* Truncate off the .ready */ |
| 715 |
memcpy(basename, rlde->d_name, basenamelen); |
| 716 |
basename[basenamelen] = '\0'; |
| 717 |
|
| 718 |
/* |
| 719 |
* Store the file in our max-heap if it has a high enough priority. |
| 720 |
*/ |
| 721 |
if (arch_files->arch_heap->bh_size < NUM_FILES_PER_DIRECTORY_SCAN) |
| 722 |
{ |
| 723 |
/* If the heap isn't full yet, quickly add it. */ |
| 724 |
arch_file = arch_files->arch_filenames[arch_files->arch_heap->bh_size]; |
| 725 |
strcpy(arch_file, basename); |
| 726 |
binaryheap_add_unordered(arch_files->arch_heap, CStringGetDatum(arch_file)); |
| 727 |
|
| 728 |
/* If we just filled the heap, make it a valid one. */ |
| 729 |
if (arch_files->arch_heap->bh_size == NUM_FILES_PER_DIRECTORY_SCAN) |
| 730 |
binaryheap_build(arch_files->arch_heap); |
| 731 |
} |
| 732 |
else if (ready_file_comparator(binaryheap_first(arch_files->arch_heap), |
| 733 |
CStringGetDatum(basename), NULL) > 0) |
| 734 |
{ |
| 735 |
/* |
| 736 |
* Remove the lowest priority file and add the current one to the |
| 737 |
* heap. |
| 738 |
*/ |
| 739 |
arch_file = DatumGetCString(binaryheap_remove_first(arch_files->arch_heap)); |
| 740 |
strcpy(arch_file, basename); |
| 741 |
binaryheap_add(arch_files->arch_heap, CStringGetDatum(arch_file)); |
| 742 |
} |
| 743 |
} |
| 744 |
FreeDir(rldir); |
| 745 |
|
| 746 |
/* If no files were found, simply return. */ |
| 747 |
if (arch_files->arch_heap->bh_size == 0) |
| 748 |
return false; |
| 749 |
|
| 750 |
/* |
| 751 |
* If we didn't fill the heap, we didn't make it a valid one. Do that |
| 752 |
* now. |
| 753 |
*/ |
| 754 |
if (arch_files->arch_heap->bh_size < NUM_FILES_PER_DIRECTORY_SCAN) |
| 755 |
binaryheap_build(arch_files->arch_heap); |
| 756 |
|
| 757 |
/* |
| 758 |
* Fill arch_files array with the files to archive in ascending order of |
| 759 |
* priority. |
| 760 |
*/ |
| 761 |
arch_files->arch_files_size = arch_files->arch_heap->bh_size; |
| 762 |
for (int i = 0; i < arch_files->arch_files_size; i++) |
| 763 |
arch_files->arch_files[i] = DatumGetCString(binaryheap_remove_first(arch_files->arch_heap)); |
| 764 |
|
| 765 |
/* Return the highest priority file. */ |
| 766 |
arch_files->arch_files_size--; |
| 767 |
strcpy(xlog, arch_files->arch_files[arch_files->arch_files_size]); |
| 768 |
|
| 769 |
return true; |
| 770 |
} |
| 771 |
|
| 772 |
/* |
| 773 |
* ready_file_comparator |
| 774 |
* |
| 775 |
* Compares the archival priority of the given files to archive. If "a" |
| 776 |
* has a higher priority than "b", a negative value will be returned. If |
| 777 |
* "b" has a higher priority than "a", a positive value will be returned. |
| 778 |
* If "a" and "b" have equivalent values, 0 will be returned. |
| 779 |
*/ |
| 780 |
static int |
| 781 |
ready_file_comparator(Datum a, Datum b, void *arg) |
| 782 |
{ |
| 783 |
char *a_str = DatumGetCString(a); |
| 784 |
char *b_str = DatumGetCString(b); |
| 785 |
bool a_history = IsTLHistoryFileName(a_str); |
| 786 |
bool b_history = IsTLHistoryFileName(b_str); |
| 787 |
|
| 788 |
/* Timeline history files always have the highest priority. */ |
| 789 |
if (a_history != b_history) |
| 790 |
return a_history ? -1 : 1; |
| 791 |
|
| 792 |
/* Priority is given to older files. */ |
| 793 |
return strcmp(a_str, b_str); |
| 794 |
} |
| 795 |
|
| 796 |
/* |
| 797 |
* PgArchForceDirScan |
| 798 |
* |
| 799 |
* When called, the next call to pgarch_readyXlog() will perform a |
| 800 |
* directory scan. This is useful for ensuring that important files such |
| 801 |
* as timeline history files are archived as quickly as possible. |
| 802 |
*/ |
| 803 |
void |
| 804 |
PgArchForceDirScan(void) |
| 805 |
{ |
| 806 |
pg_atomic_write_membarrier_u32(&PgArch->force_dir_scan, 1); |
| 807 |
} |
| 808 |
|
| 809 |
/* |
| 810 |
* pgarch_archiveDone |
| 811 |
* |
| 812 |
* Emit notification that an xlog file has been successfully archived. |
| 813 |
* We do this by renaming the status file from NNN.ready to NNN.done. |
| 814 |
* Eventually, a checkpoint process will notice this and delete both the |
| 815 |
* NNN.done file and the xlog file itself. |
| 816 |
*/ |
| 817 |
static void |
| 818 |
pgarch_archiveDone(char *xlog) |
| 819 |
{ |
| 820 |
char rlogready[MAXPGPATH]; |
| 821 |
char rlogdone[MAXPGPATH]; |
| 822 |
|
| 823 |
StatusFilePath(rlogready, xlog, ".ready"); |
| 824 |
StatusFilePath(rlogdone, xlog, ".done"); |
| 825 |
|
| 826 |
/* |
| 827 |
* To avoid extra overhead, we don't durably rename the .ready file to |
| 828 |
* .done. Archive commands and libraries must gracefully handle attempts |
| 829 |
* to re-archive files (e.g., if the server crashes just before this |
| 830 |
* function is called), so it should be okay if the .ready file reappears |
| 831 |
* after a crash. |
| 832 |
*/ |
| 833 |
if (rename(rlogready, rlogdone) < 0) |
| 834 |
ereport(WARNING, |
| 835 |
(errcode_for_file_access(), |
| 836 |
errmsg("could not rename file \"%s\" to \"%s\": %m", |
| 837 |
rlogready, rlogdone))); |
| 838 |
} |
| 839 |
|
| 840 |
|
| 841 |
/* |
| 842 |
* pgarch_die |
| 843 |
* |
| 844 |
* Exit-time cleanup handler |
| 845 |
*/ |
| 846 |
static void |
| 847 |
pgarch_die(int code, Datum arg) |
| 848 |
{ |
| 849 |
PgArch->pgprocno = INVALID_PROC_NUMBER; |
| 850 |
} |
| 851 |
|
| 852 |
/* |
| 853 |
* Interrupt handler for WAL archiver process. |
| 854 |
* |
| 855 |
* This is called in the loops pgarch_MainLoop and pgarch_ArchiverCopyLoop. |
| 856 |
* It checks for barrier events, config update and request for logging of |
| 857 |
* memory contexts, but not shutdown request because how to handle |
| 858 |
* shutdown request is different between those loops. |
| 859 |
*/ |
| 860 |
static void |
| 861 |
ProcessPgArchInterrupts(void) |
| 862 |
{ |
| 863 |
if (ProcSignalBarrierPending) |
| 864 |
ProcessProcSignalBarrier(); |
| 865 |
|
| 866 |
/* Perform logging of memory contexts of this process */ |
| 867 |
if (LogMemoryContextPending) |
| 868 |
ProcessLogMemoryContextInterrupt(); |
| 869 |
|
| 870 |
if (ConfigReloadPending) |
| 871 |
{ |
| 872 |
char *archiveLib = pstrdup(XLogArchiveLibrary); |
| 873 |
bool archiveLibChanged; |
| 874 |
|
| 875 |
ConfigReloadPending = false; |
| 876 |
ProcessConfigFile(PGC_SIGHUP); |
| 877 |
|
| 878 |
if (XLogArchiveLibrary[0] != '\0' && XLogArchiveCommand[0] != '\0') |
| 879 |
ereport(ERROR, |
| 880 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 881 |
errmsg("both \"archive_command\" and \"archive_library\" set"), |
| 882 |
errdetail("Only one of \"archive_command\", \"archive_library\" may be set."))); |
| 883 |
|
| 884 |
archiveLibChanged = strcmp(XLogArchiveLibrary, archiveLib) != 0; |
| 885 |
pfree(archiveLib); |
| 886 |
|
| 887 |
if (archiveLibChanged) |
| 888 |
{ |
| 889 |
/* |
| 890 |
* Ideally, we would simply unload the previous archive module and |
| 891 |
* load the new one, but there is presently no mechanism for |
| 892 |
* unloading a library (see the comment above |
| 893 |
* internal_load_library()). To deal with this, we simply restart |
| 894 |
* the archiver. The new archive module will be loaded when the |
| 895 |
* new archiver process starts up. Note that this triggers the |
| 896 |
* module's shutdown callback, if defined. |
| 897 |
*/ |
| 898 |
ereport(LOG, |
| 899 |
(errmsg("restarting archiver process because value of " |
| 900 |
"\"archive_library\" was changed"))); |
| 901 |
|
| 902 |
proc_exit(0); |
| 903 |
} |
| 904 |
} |
| 905 |
} |
| 906 |
|
| 907 |
/* |
| 908 |
* LoadArchiveLibrary |
| 909 |
* |
| 910 |
* Loads the archiving callbacks into our local ArchiveCallbacks. |
| 911 |
*/ |
| 912 |
static void |
| 913 |
LoadArchiveLibrary(void) |
| 914 |
{ |
| 915 |
ArchiveModuleInit archive_init; |
| 916 |
|
| 917 |
if (XLogArchiveLibrary[0] != '\0' && XLogArchiveCommand[0] != '\0') |
| 918 |
ereport(ERROR, |
| 919 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 920 |
errmsg("both \"archive_command\" and \"archive_library\" set"), |
| 921 |
errdetail("Only one of \"archive_command\", \"archive_library\" may be set."))); |
| 922 |
|
| 923 |
/* |
| 924 |
* If shell archiving is enabled, use our special initialization function. |
| 925 |
* Otherwise, load the library and call its _PG_archive_module_init(). |
| 926 |
*/ |
| 927 |
if (XLogArchiveLibrary[0] == '\0') |
| 928 |
archive_init = shell_archive_init; |
| 929 |
else |
| 930 |
archive_init = (ArchiveModuleInit) |
| 931 |
load_external_function(XLogArchiveLibrary, |
| 932 |
"_PG_archive_module_init", false, NULL); |
| 933 |
|
| 934 |
if (archive_init == NULL) |
| 935 |
ereport(ERROR, |
| 936 |
(errmsg("archive modules have to define the symbol %s", "_PG_archive_module_init"))); |
| 937 |
|
| 938 |
ArchiveCallbacks = (*archive_init) (); |
| 939 |
|
| 940 |
if (ArchiveCallbacks->archive_file_cb == NULL) |
| 941 |
ereport(ERROR, |
| 942 |
(errmsg("archive modules must register an archive callback"))); |
| 943 |
|
| 944 |
archive_module_state = (ArchiveModuleState *) palloc0(sizeof(ArchiveModuleState)); |
| 945 |
if (ArchiveCallbacks->startup_cb != NULL) |
| 946 |
ArchiveCallbacks->startup_cb(archive_module_state); |
| 947 |
|
| 948 |
before_shmem_exit(pgarch_call_module_shutdown_cb, 0); |
| 949 |
} |
| 950 |
|
| 951 |
/* |
| 952 |
* Call the shutdown callback of the loaded archive module, if defined. |
| 953 |
*/ |
| 954 |
static void |
| 955 |
pgarch_call_module_shutdown_cb(int code, Datum arg) |
| 956 |
{ |
| 957 |
if (ArchiveCallbacks->shutdown_cb != NULL) |
| 958 |
ArchiveCallbacks->shutdown_cb(archive_module_state); |
| 959 |
} |