Debian Codegraph @

src/backend/replication/logical/slotsync.c

raw
1 /*-------------------------------------------------------------------------
2 * slotsync.c
3 * Functionality for synchronizing slots to a standby server from the
4 * primary server.
5 *
6 * Copyright (c) 2024-2025, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/replication/logical/slotsync.c
10 *
11 * This file contains the code for slot synchronization on a physical standby
12 * to fetch logical failover slots information from the primary server, create
13 * the slots on the standby and synchronize them periodically.
14 *
15 * Slot synchronization can be performed either automatically by enabling slot
16 * sync worker or manually by calling SQL function pg_sync_replication_slots().
17 *
18 * If the WAL corresponding to the remote's restart_lsn is not available on the
19 * physical standby or the remote's catalog_xmin precedes the oldest xid for
20 * which it is guaranteed that rows wouldn't have been removed then we cannot
21 * create the local standby slot because that would mean moving the local slot
22 * backward and decoding won't be possible via such a slot. In this case, the
23 * slot will be marked as RS_TEMPORARY. Once the primary server catches up,
24 * the slot will be marked as RS_PERSISTENT (which means sync-ready) after
25 * which slot sync worker can perform the sync periodically or user can call
26 * pg_sync_replication_slots() periodically to perform the syncs.
27 *
28 * If synchronized slots fail to build a consistent snapshot from the
29 * restart_lsn before reaching confirmed_flush_lsn, they would become
30 * unreliable after promotion due to potential data loss from changes
31 * before reaching a consistent point. This can happen because the slots can
32 * be synced at some random time and we may not reach the consistent point
33 * at the same WAL location as the primary. So, we mark such slots as
34 * RS_TEMPORARY. Once the decoding from corresponding LSNs can reach a
35 * consistent point, they will be marked as RS_PERSISTENT.
36 *
37 * The slot sync worker waits for some time before the next synchronization,
38 * with the duration varying based on whether any slots were updated during
39 * the last cycle. Refer to the comments above wait_for_slot_activity() for
40 * more details.
41 *
42 * Any standby synchronized slots will be dropped if they no longer need
43 * to be synchronized. See comment atop drop_local_obsolete_slots() for more
44 * details.
45 *---------------------------------------------------------------------------
46 */
47
48 #include "postgres.h"
49
50 #include <time.h>
51
52 #include "access/xlog_internal.h"
53 #include "access/xlogrecovery.h"
54 #include "catalog/pg_database.h"
55 #include "commands/dbcommands.h"
56 #include "libpq/pqsignal.h"
57 #include "pgstat.h"
58 #include "postmaster/interrupt.h"
59 #include "replication/logical.h"
60 #include "replication/slotsync.h"
61 #include "replication/snapbuild.h"
62 #include "storage/ipc.h"
63 #include "storage/lmgr.h"
64 #include "storage/proc.h"
65 #include "storage/procarray.h"
66 #include "tcop/tcopprot.h"
67 #include "utils/builtins.h"
68 #include "utils/pg_lsn.h"
69 #include "utils/ps_status.h"
70 #include "utils/timeout.h"
71
72 /*
73 * Struct for sharing information to control slot synchronization.
74 *
75 * The 'pid' is either the slot sync worker's pid or the backend's pid running
76 * the SQL function pg_sync_replication_slots(). On promotion, the startup
77 * process sets 'stopSignaled' and uses this 'pid' to signal the synchronizing
78 * process with PROCSIG_SLOTSYNC_MESSAGE and also to wake it up so that the
79 * process can immediately stop its synchronizing work.
80 * Setting 'stopSignaled' on the other hand is used to handle the race
81 * condition when the postmaster has not noticed the promotion yet and thus may
82 * end up restarting the slot sync worker. If 'stopSignaled' is set, the worker
83 * will exit in such a case. The SQL function pg_sync_replication_slots() will
84 * also error out if this flag is set. Note that we don't need to reset this
85 * variable as after promotion the slot sync worker won't be restarted because
86 * the pmState changes to PM_RUN from PM_HOT_STANDBY and we don't support
87 * demoting primary without restarting the server.
88 * See LaunchMissingBackgroundProcesses.
89 *
90 * The 'syncing' flag is needed to prevent concurrent slot syncs to avoid slot
91 * overwrites.
92 *
93 * The 'last_start_time' is needed by postmaster to start the slot sync worker
94 * once per SLOTSYNC_RESTART_INTERVAL_SEC. In cases where an immediate restart
95 * is expected (e.g., slot sync GUCs change), slot sync worker will reset
96 * last_start_time before exiting, so that postmaster can start the worker
97 * without waiting for SLOTSYNC_RESTART_INTERVAL_SEC.
98 */
99 typedef struct SlotSyncCtxStruct
100 {
101 pid_t pid;
102 bool stopSignaled;
103 bool syncing;
104 time_t last_start_time;
105 slock_t mutex;
106 } SlotSyncCtxStruct;
107
108 static SlotSyncCtxStruct *SlotSyncCtx = NULL;
109
110 /* GUC variable */
111 bool sync_replication_slots = false;
112
113 /*
114 * The sleep time (ms) between slot-sync cycles varies dynamically
115 * (within a MIN/MAX range) according to slot activity. See
116 * wait_for_slot_activity() for details.
117 */
118 #define MIN_SLOTSYNC_WORKER_NAPTIME_MS 200
119 #define MAX_SLOTSYNC_WORKER_NAPTIME_MS 30000 /* 30s */
120
121 static long sleep_ms = MIN_SLOTSYNC_WORKER_NAPTIME_MS;
122
123 /* The restart interval for slot sync work used by postmaster */
124 #define SLOTSYNC_RESTART_INTERVAL_SEC 10
125
126 /*
127 * Flag to tell if we are syncing replication slots. Unlike the 'syncing' flag
128 * in SlotSyncCtxStruct, this flag is true only if the current process is
129 * performing slot synchronization.
130 */
131 static bool syncing_slots = false;
132
133 /*
134 * Interrupt flag set when PROCSIG_SLOTSYNC_MESSAGE is received, asking the
135 * slotsync worker or pg_sync_replication_slots() to stop because
136 * standby promotion has been triggered.
137 */
138 volatile sig_atomic_t SlotSyncShutdownPending = false;
139
140 /*
141 * Structure to hold information fetched from the primary server about a logical
142 * replication slot.
143 */
144 typedef struct RemoteSlot
145 {
146 char *name;
147 char *plugin;
148 char *database;
149 bool two_phase;
150 bool failover;
151 XLogRecPtr restart_lsn;
152 XLogRecPtr confirmed_lsn;
153 XLogRecPtr two_phase_at;
154 TransactionId catalog_xmin;
155
156 /* RS_INVAL_NONE if valid, or the reason of invalidation */
157 ReplicationSlotInvalidationCause invalidated;
158 } RemoteSlot;
159
160 static void slotsync_failure_callback(int code, Datum arg);
161 static void update_synced_slots_inactive_since(void);
162
163 /*
164 * If necessary, update the local synced slot's metadata based on the data
165 * from the remote slot.
166 *
167 * If no update was needed (the data of the remote slot is the same as the
168 * local slot) return false, otherwise true.
169 *
170 * *found_consistent_snapshot will be true iff the remote slot's LSN or xmin is
171 * modified, and decoding from the corresponding LSN's can reach a
172 * consistent snapshot.
173 *
174 * *remote_slot_precedes will be true if the remote slot's LSN or xmin
175 * precedes locally reserved position.
176 */
177 static bool
178 update_local_synced_slot(RemoteSlot *remote_slot, Oid remote_dbid,
179 bool *found_consistent_snapshot,
180 bool *remote_slot_precedes)
181 {
182 ReplicationSlot *slot = MyReplicationSlot;
183 bool updated_xmin_or_lsn = false;
184 bool updated_config = false;
185
186 Assert(slot->data.invalidated == RS_INVAL_NONE);
187
188 if (found_consistent_snapshot)
189 *found_consistent_snapshot = false;
190
191 if (remote_slot_precedes)
192 *remote_slot_precedes = false;
193
194 /*
195 * Don't overwrite if we already have a newer catalog_xmin and
196 * restart_lsn.
197 */
198 if (remote_slot->restart_lsn < slot->data.restart_lsn ||
199 TransactionIdPrecedes(remote_slot->catalog_xmin,
200 slot->data.catalog_xmin))
201 {
202 /*
203 * This can happen in following situations:
204 *
205 * If the slot is temporary, it means either the initial WAL location
206 * reserved for the local slot is ahead of the remote slot's
207 * restart_lsn or the initial xmin_horizon computed for the local slot
208 * is ahead of the remote slot.
209 *
210 * If the slot is persistent, both restart_lsn and catalog_xmin of the
211 * synced slot could still be ahead of the remote slot. Since we use
212 * slot advance functionality to keep snapbuild/slot updated, it is
213 * possible that the restart_lsn and catalog_xmin are advanced to a
214 * later position than it has on the primary. This can happen when
215 * slot advancing machinery finds running xacts record after reaching
216 * the consistent state at a later point than the primary where it
217 * serializes the snapshot and updates the restart_lsn.
218 *
219 * We LOG the message if the slot is temporary as it can help the user
220 * to understand why the slot is not sync-ready. In the case of a
221 * persistent slot, it would be a more common case and won't directly
222 * impact the users, so we used DEBUG1 level to log the message.
223 */
224 ereport(slot->data.persistency == RS_TEMPORARY ? LOG : DEBUG1,
225 errmsg("could not synchronize replication slot \"%s\"",
226 remote_slot->name),
227 errdetail("Synchronization could lead to data loss, because the remote slot needs WAL at LSN %X/%X and catalog xmin %u, but the standby has LSN %X/%X and catalog xmin %u.",
228 LSN_FORMAT_ARGS(remote_slot->restart_lsn),
229 remote_slot->catalog_xmin,
230 LSN_FORMAT_ARGS(slot->data.restart_lsn),
231 slot->data.catalog_xmin));
232
233 if (remote_slot_precedes)
234 *remote_slot_precedes = true;
235
236 /*
237 * Skip updating the configuration. This is required to avoid syncing
238 * two_phase_at without syncing confirmed_lsn. Otherwise, the prepared
239 * transaction between old confirmed_lsn and two_phase_at will
240 * unexpectedly get decoded and sent to the downstream after
241 * promotion. See comments in ReorderBufferFinishPrepared.
242 */
243 return false;
244 }
245
246 /*
247 * Attempt to sync LSNs and xmins only if remote slot is ahead of local
248 * slot.
249 */
250 if (remote_slot->confirmed_lsn > slot->data.confirmed_flush ||
251 remote_slot->restart_lsn > slot->data.restart_lsn ||
252 TransactionIdFollows(remote_slot->catalog_xmin,
253 slot->data.catalog_xmin))
254 {
255 /*
256 * We can't directly copy the remote slot's LSN or xmin unless there
257 * exists a consistent snapshot at that point. Otherwise, after
258 * promotion, the slots may not reach a consistent point before the
259 * confirmed_flush_lsn which can lead to a data loss. To avoid data
260 * loss, we let slot machinery advance the slot which ensures that
261 * snapbuilder/slot statuses are updated properly.
262 */
263 if (SnapBuildSnapshotExists(remote_slot->restart_lsn))
264 {
265 /*
266 * Update the slot info directly if there is a serialized snapshot
267 * at the restart_lsn, as the slot can quickly reach consistency
268 * at restart_lsn by restoring the snapshot.
269 */
270 SpinLockAcquire(&slot->mutex);
271 slot->data.restart_lsn = remote_slot->restart_lsn;
272 slot->data.confirmed_flush = remote_slot->confirmed_lsn;
273 slot->data.catalog_xmin = remote_slot->catalog_xmin;
274 SpinLockRelease(&slot->mutex);
275
276 if (found_consistent_snapshot)
277 *found_consistent_snapshot = true;
278
279 updated_xmin_or_lsn = true;
280 }
281 else
282 {
283 XLogRecPtr old_confirmed_lsn = slot->data.confirmed_flush;
284 XLogRecPtr old_restart_lsn = slot->data.restart_lsn;
285 TransactionId old_catalog_xmin = slot->data.catalog_xmin;
286
287 LogicalSlotAdvanceAndCheckSnapState(remote_slot->confirmed_lsn,
288 found_consistent_snapshot);
289
290 /* Sanity check */
291 if (slot->data.confirmed_flush != remote_slot->confirmed_lsn)
292 ereport(ERROR,
293 errmsg_internal("synchronized confirmed_flush for slot \"%s\" differs from remote slot",
294 remote_slot->name),
295 errdetail_internal("Remote slot has LSN %X/%X but local slot has LSN %X/%X.",
296 LSN_FORMAT_ARGS(remote_slot->confirmed_lsn),
297 LSN_FORMAT_ARGS(slot->data.confirmed_flush)));
298
299 /*
300 * It is possible that the slot's xmin or LSNs are not updated,
301 * when the synced slot has reached consistent snapshot state or
302 * cannot build one at all.
303 */
304 updated_xmin_or_lsn = (old_confirmed_lsn != slot->data.confirmed_flush ||
305 old_restart_lsn != slot->data.restart_lsn ||
306 old_catalog_xmin != slot->data.catalog_xmin);
307 }
308 }
309
310 if (remote_dbid != slot->data.database ||
311 remote_slot->two_phase != slot->data.two_phase ||
312 remote_slot->failover != slot->data.failover ||
313 strcmp(remote_slot->plugin, NameStr(slot->data.plugin)) != 0 ||
314 remote_slot->two_phase_at != slot->data.two_phase_at)
315 {
316 NameData plugin_name;
317
318 /* Avoid expensive operations while holding a spinlock. */
319 namestrcpy(&plugin_name, remote_slot->plugin);
320
321 SpinLockAcquire(&slot->mutex);
322 slot->data.plugin = plugin_name;
323 slot->data.database = remote_dbid;
324 slot->data.two_phase = remote_slot->two_phase;
325 slot->data.two_phase_at = remote_slot->two_phase_at;
326 slot->data.failover = remote_slot->failover;
327 SpinLockRelease(&slot->mutex);
328
329 updated_config = true;
330
331 /*
332 * Ensure that there is no risk of sending prepared transactions
333 * unexpectedly after the promotion.
334 */
335 Assert(slot->data.two_phase_at <= slot->data.confirmed_flush);
336 }
337
338 /*
339 * We have to write the changed xmin to disk *before* we change the
340 * in-memory value, otherwise after a crash we wouldn't know that some
341 * catalog tuples might have been removed already.
342 */
343 if (updated_config || updated_xmin_or_lsn)
344 {
345 ReplicationSlotMarkDirty();
346 ReplicationSlotSave();
347 }
348
349 /*
350 * Now the new xmin is safely on disk, we can let the global value
351 * advance. We do not take ProcArrayLock or similar since we only advance
352 * xmin here and there's not much harm done by a concurrent computation
353 * missing that.
354 */
355 if (updated_xmin_or_lsn)
356 {
357 SpinLockAcquire(&slot->mutex);
358 slot->effective_catalog_xmin = remote_slot->catalog_xmin;
359 SpinLockRelease(&slot->mutex);
360
361 ReplicationSlotsComputeRequiredXmin(false);
362 ReplicationSlotsComputeRequiredLSN();
363 }
364
365 return updated_config || updated_xmin_or_lsn;
366 }
367
368 /*
369 * Get the list of local logical slots that are synchronized from the
370 * primary server.
371 */
372 static List *
373 get_local_synced_slots(void)
374 {
375 List *local_slots = NIL;
376
377 LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
378
379 for (int i = 0; i < max_replication_slots; i++)
380 {
381 ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
382
383 /* Check if it is a synchronized slot */
384 if (s->in_use && s->data.synced)
385 {
386 Assert(SlotIsLogical(s));
387 local_slots = lappend(local_slots, s);
388 }
389 }
390
391 LWLockRelease(ReplicationSlotControlLock);
392
393 return local_slots;
394 }
395
396 /*
397 * Helper function to check if local_slot is required to be retained.
398 *
399 * Return false either if local_slot does not exist in the remote_slots list
400 * or is invalidated while the corresponding remote slot is still valid,
401 * otherwise true.
402 */
403 static bool
404 local_sync_slot_required(ReplicationSlot *local_slot, List *remote_slots)
405 {
406 bool remote_exists = false;
407 bool locally_invalidated = false;
408
409 foreach_ptr(RemoteSlot, remote_slot, remote_slots)
410 {
411 if (strcmp(remote_slot->name, NameStr(local_slot->data.name)) == 0)
412 {
413 remote_exists = true;
414
415 /*
416 * If remote slot is not invalidated but local slot is marked as
417 * invalidated, then set locally_invalidated flag.
418 */
419 SpinLockAcquire(&local_slot->mutex);
420 locally_invalidated =
421 (remote_slot->invalidated == RS_INVAL_NONE) &&
422 (local_slot->data.invalidated != RS_INVAL_NONE);
423 SpinLockRelease(&local_slot->mutex);
424
425 break;
426 }
427 }
428
429 return (remote_exists && !locally_invalidated);
430 }
431
432 /*
433 * Drop local obsolete slots.
434 *
435 * Drop the local slots that no longer need to be synced i.e. these either do
436 * not exist on the primary or are no longer enabled for failover.
437 *
438 * Additionally, drop any slots that are valid on the primary but got
439 * invalidated on the standby. This situation may occur due to the following
440 * reasons:
441 * - The 'max_slot_wal_keep_size' on the standby is insufficient to retain WAL
442 * records from the restart_lsn of the slot.
443 * - 'primary_slot_name' is temporarily reset to null and the physical slot is
444 * removed.
445 * These dropped slots will get recreated in next sync-cycle and it is okay to
446 * drop and recreate such slots as long as these are not consumable on the
447 * standby (which is the case currently).
448 *
449 * Note: Change of 'wal_level' on the primary server to a level lower than
450 * logical may also result in slot invalidation and removal on the standby.
451 * This is because such 'wal_level' change is only possible if the logical
452 * slots are removed on the primary server, so it's expected to see the
453 * slots being invalidated and removed on the standby too (and re-created
454 * if they are re-created on the primary server).
455 */
456 static void
457 drop_local_obsolete_slots(List *remote_slot_list)
458 {
459 List *local_slots = get_local_synced_slots();
460
461 foreach_ptr(ReplicationSlot, local_slot, local_slots)
462 {
463 /* Drop the local slot if it is not required to be retained. */
464 if (!local_sync_slot_required(local_slot, remote_slot_list))
465 {
466 Oid slot_database = local_slot->data.database;
467 bool synced_slot;
468
469 /*
470 * Use shared lock to prevent a conflict with
471 * ReplicationSlotsDropDBSlots(), trying to drop the same slot
472 * during a drop-database operation.
473 */
474 LockSharedObject(DatabaseRelationId, slot_database, 0,
475 AccessShareLock);
476
477 /*
478 * In the small window between getting the slot to drop and
479 * locking the database, there is a possibility of a parallel
480 * database drop by the startup process and the creation of a new
481 * slot by the user. This new user-created slot may end up using
482 * the same shared memory as that of 'local_slot'. Thus check if
483 * local_slot is still the synced one before performing actual
484 * drop.
485 */
486 SpinLockAcquire(&local_slot->mutex);
487 synced_slot = local_slot->in_use && local_slot->data.synced;
488 SpinLockRelease(&local_slot->mutex);
489
490 if (synced_slot)
491 {
492 NameData slot_name = local_slot->data.name;
493
494 ReplicationSlotAcquire(NameStr(slot_name), true, false);
495 ReplicationSlotDropAcquired();
496
497 ereport(LOG,
498 errmsg("dropped replication slot \"%s\" of database with OID %u",
499 NameStr(slot_name),
500 slot_database));
501 }
502
503 UnlockSharedObject(DatabaseRelationId, slot_database, 0,
504 AccessShareLock);
505 }
506 }
507 }
508
509 /*
510 * Reserve WAL for the currently active local slot using the specified WAL
511 * location (restart_lsn).
512 *
513 * If the given WAL location has been removed or is at risk of removal,
514 * reserve WAL using the oldest segment that is non-removable.
515 */
516 static void
517 reserve_wal_for_local_slot(XLogRecPtr restart_lsn)
518 {
519 XLogRecPtr slot_min_lsn;
520 XLogRecPtr min_safe_lsn;
521 XLogSegNo segno;
522 ReplicationSlot *slot = MyReplicationSlot;
523
524 Assert(slot != NULL);
525 Assert(!XLogRecPtrIsValid(slot->data.restart_lsn));
526
527 /*
528 * Acquire an exclusive lock to prevent the checkpoint process from
529 * concurrently calculating the minimum slot LSN (see
530 * CheckPointReplicationSlots), ensuring that if WAL reservation occurs
531 * first, the checkpoint must wait for the restart_lsn update before
532 * calculating the minimum LSN.
533 *
534 * Note: Unlike ReplicationSlotReserveWal(), this lock does not protect a
535 * newly synced slot from being invalidated if a concurrent checkpoint has
536 * invoked CheckPointReplicationSlots() before the WAL reservation here.
537 * This can happen because the initial restart_lsn received from the
538 * remote server can precede the redo pointer. Therefore, when selecting
539 * the initial restart_lsn, we consider using the redo pointer or the
540 * minimum slot LSN (if those values are greater than the remote
541 * restart_lsn) instead of relying solely on the remote value.
542 */
543 LWLockAcquire(ReplicationSlotAllocationLock, LW_EXCLUSIVE);
544
545 /*
546 * Determine the minimum non-removable LSN by comparing the redo pointer
547 * with the minimum slot LSN.
548 *
549 * The minimum slot LSN is considered because the redo pointer advances at
550 * every checkpoint, even when replication slots are present on the
551 * standby. In such scenarios, the redo pointer can exceed the remote
552 * restart_lsn, while WALs preceding the remote restart_lsn remain
553 * protected by a local replication slot.
554 */
555 min_safe_lsn = GetRedoRecPtr();
556 slot_min_lsn = XLogGetReplicationSlotMinimumLSN();
557
558 if (XLogRecPtrIsValid(slot_min_lsn) && min_safe_lsn > slot_min_lsn)
559 min_safe_lsn = slot_min_lsn;
560
561 /*
562 * If the minimum safe LSN is greater than the given restart_lsn, use it
563 * as the initial restart_lsn for the newly synced slot. Otherwise, use
564 * the given remote restart_lsn.
565 */
566 SpinLockAcquire(&slot->mutex);
567 slot->data.restart_lsn = Max(restart_lsn, min_safe_lsn);
568 SpinLockRelease(&slot->mutex);
569
570 ReplicationSlotsComputeRequiredLSN();
571
572 XLByteToSeg(slot->data.restart_lsn, segno, wal_segment_size);
573 if (XLogGetLastRemovedSegno() >= segno)
574 elog(ERROR, "WAL required by replication slot %s has been removed concurrently",
575 NameStr(slot->data.name));
576
577 LWLockRelease(ReplicationSlotAllocationLock);
578 }
579
580 /*
581 * If the remote restart_lsn and catalog_xmin have caught up with the
582 * local ones, then update the LSNs and persist the local synced slot for
583 * future synchronization; otherwise, do nothing.
584 *
585 * Return true if the slot is marked as RS_PERSISTENT (sync-ready), otherwise
586 * false.
587 */
588 static bool
589 update_and_persist_local_synced_slot(RemoteSlot *remote_slot, Oid remote_dbid)
590 {
591 ReplicationSlot *slot = MyReplicationSlot;
592 bool found_consistent_snapshot = false;
593 bool remote_slot_precedes = false;
594
595 (void) update_local_synced_slot(remote_slot, remote_dbid,
596 &found_consistent_snapshot,
597 &remote_slot_precedes);
598
599 /*
600 * Check if the primary server has caught up. Refer to the comment atop
601 * the file for details on this check.
602 */
603 if (remote_slot_precedes)
604 {
605 /*
606 * The remote slot didn't catch up to locally reserved position.
607 *
608 * We do not drop the slot because the restart_lsn can be ahead of the
609 * current location when recreating the slot in the next cycle. It may
610 * take more time to create such a slot. Therefore, we keep this slot
611 * and attempt the synchronization in the next cycle.
612 */
613 return false;
614 }
615
616 /*
617 * Don't persist the slot if it cannot reach the consistent point from the
618 * restart_lsn. See comments atop this file.
619 */
620 if (!found_consistent_snapshot)
621 {
622 ereport(LOG,
623 errmsg("could not synchronize replication slot \"%s\"", remote_slot->name),
624 errdetail("Synchronization could lead to data loss, because the standby could not build a consistent snapshot to decode WALs at LSN %X/%X.",
625 LSN_FORMAT_ARGS(slot->data.restart_lsn)));
626
627 return false;
628 }
629
630 ReplicationSlotPersist();
631
632 ereport(LOG,
633 errmsg("newly created replication slot \"%s\" is sync-ready now",
634 remote_slot->name));
635
636 return true;
637 }
638
639 /*
640 * Synchronize a single slot to the given position.
641 *
642 * This creates a new slot if there is no existing one and updates the
643 * metadata of the slot as per the data received from the primary server.
644 *
645 * The slot is created as a temporary slot and stays in the same state until the
646 * remote_slot catches up with locally reserved position and local slot is
647 * updated. The slot is then persisted and is considered as sync-ready for
648 * periodic syncs.
649 *
650 * Returns TRUE if the local slot is updated.
651 */
652 static bool
653 synchronize_one_slot(RemoteSlot *remote_slot, Oid remote_dbid)
654 {
655 ReplicationSlot *slot;
656 XLogRecPtr latestFlushPtr;
657 bool slot_updated = false;
658
659 /*
660 * Make sure that concerned WAL is received and flushed before syncing
661 * slot to target lsn received from the primary server.
662 */
663 latestFlushPtr = GetStandbyFlushRecPtr(NULL);
664 if (remote_slot->confirmed_lsn > latestFlushPtr)
665 {
666 /*
667 * Can get here only if GUC 'synchronized_standby_slots' on the
668 * primary server was not configured correctly.
669 */
670 ereport(AmLogicalSlotSyncWorkerProcess() ? LOG : ERROR,
671 errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
672 errmsg("skipping slot synchronization because the received slot sync"
673 " LSN %X/%X for slot \"%s\" is ahead of the standby position %X/%X",
674 LSN_FORMAT_ARGS(remote_slot->confirmed_lsn),
675 remote_slot->name,
676 LSN_FORMAT_ARGS(latestFlushPtr)));
677
678 return false;
679 }
680
681 /* Search for the named slot */
682 if ((slot = SearchNamedReplicationSlot(remote_slot->name, true)))
683 {
684 bool synced;
685
686 SpinLockAcquire(&slot->mutex);
687 synced = slot->data.synced;
688 SpinLockRelease(&slot->mutex);
689
690 /* User-created slot with the same name exists, raise ERROR. */
691 if (!synced)
692 ereport(ERROR,
693 errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
694 errmsg("exiting from slot synchronization because same"
695 " name slot \"%s\" already exists on the standby",
696 remote_slot->name));
697
698 /*
699 * The slot has been synchronized before.
700 *
701 * It is important to acquire the slot here before checking
702 * invalidation. If we don't acquire the slot first, there could be a
703 * race condition that the local slot could be invalidated just after
704 * checking the 'invalidated' flag here and we could end up
705 * overwriting 'invalidated' flag to remote_slot's value. See
706 * InvalidatePossiblyObsoleteSlot() where it invalidates slot directly
707 * if the slot is not acquired by other processes.
708 *
709 * XXX: If it ever turns out that slot acquire/release is costly for
710 * cases when none of the slot properties is changed then we can do a
711 * pre-check to ensure that at least one of the slot properties is
712 * changed before acquiring the slot.
713 */
714 ReplicationSlotAcquire(remote_slot->name, true, false);
715
716 Assert(slot == MyReplicationSlot);
717
718 /*
719 * Copy the invalidation cause from remote only if local slot is not
720 * invalidated locally, we don't want to overwrite existing one.
721 */
722 if (slot->data.invalidated == RS_INVAL_NONE &&
723 remote_slot->invalidated != RS_INVAL_NONE)
724 {
725 SpinLockAcquire(&slot->mutex);
726 slot->data.invalidated = remote_slot->invalidated;
727 SpinLockRelease(&slot->mutex);
728
729 /* Make sure the invalidated state persists across server restart */
730 ReplicationSlotMarkDirty();
731 ReplicationSlotSave();
732
733 slot_updated = true;
734 }
735
736 /* Skip the sync of an invalidated slot */
737 if (slot->data.invalidated != RS_INVAL_NONE)
738 {
739 ReplicationSlotRelease();
740 return slot_updated;
741 }
742
743 /* Slot not ready yet, let's attempt to make it sync-ready now. */
744 if (slot->data.persistency == RS_TEMPORARY)
745 {
746 slot_updated = update_and_persist_local_synced_slot(remote_slot,
747 remote_dbid);
748 }
749
750 /* Slot ready for sync, so sync it. */
751 else
752 {
753 /*
754 * Sanity check: As long as the invalidations are handled
755 * appropriately as above, this should never happen.
756 *
757 * We don't need to check restart_lsn here. See the comments in
758 * update_local_synced_slot() for details.
759 */
760 if (remote_slot->confirmed_lsn < slot->data.confirmed_flush)
761 ereport(ERROR,
762 errmsg_internal("cannot synchronize local slot \"%s\"",
763 remote_slot->name),
764 errdetail_internal("Local slot's start streaming location LSN(%X/%X) is ahead of remote slot's LSN(%X/%X).",
765 LSN_FORMAT_ARGS(slot->data.confirmed_flush),
766 LSN_FORMAT_ARGS(remote_slot->confirmed_lsn)));
767
768 slot_updated = update_local_synced_slot(remote_slot, remote_dbid,
769 NULL, NULL);
770 }
771 }
772 /* Otherwise create the slot first. */
773 else
774 {
775 NameData plugin_name;
776 TransactionId xmin_horizon = InvalidTransactionId;
777
778 /* Skip creating the local slot if remote_slot is invalidated already */
779 if (remote_slot->invalidated != RS_INVAL_NONE)
780 return false;
781
782 /*
783 * We create temporary slots instead of ephemeral slots here because
784 * we want the slots to survive after releasing them. This is done to
785 * avoid dropping and re-creating the slots in each synchronization
786 * cycle if the restart_lsn or catalog_xmin of the remote slot has not
787 * caught up.
788 */
789 ReplicationSlotCreate(remote_slot->name, true, RS_TEMPORARY,
790 remote_slot->two_phase,
791 remote_slot->failover,
792 true);
793
794 /* For shorter lines. */
795 slot = MyReplicationSlot;
796
797 /* Avoid expensive operations while holding a spinlock. */
798 namestrcpy(&plugin_name, remote_slot->plugin);
799
800 SpinLockAcquire(&slot->mutex);
801 slot->data.database = remote_dbid;
802 slot->data.plugin = plugin_name;
803 SpinLockRelease(&slot->mutex);
804
805 reserve_wal_for_local_slot(remote_slot->restart_lsn);
806
807 LWLockAcquire(ReplicationSlotControlLock, LW_EXCLUSIVE);
808 LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
809 xmin_horizon = GetOldestSafeDecodingTransactionId(true);
810 SpinLockAcquire(&slot->mutex);
811 slot->effective_catalog_xmin = xmin_horizon;
812 slot->data.catalog_xmin = xmin_horizon;
813 SpinLockRelease(&slot->mutex);
814 ReplicationSlotsComputeRequiredXmin(true);
815 LWLockRelease(ProcArrayLock);
816 LWLockRelease(ReplicationSlotControlLock);
817
818 update_and_persist_local_synced_slot(remote_slot, remote_dbid);
819
820 slot_updated = true;
821 }
822
823 ReplicationSlotRelease();
824
825 return slot_updated;
826 }
827
828 /*
829 * Synchronize slots.
830 *
831 * Gets the failover logical slots info from the primary server and updates
832 * the slots locally. Creates the slots if not present on the standby.
833 *
834 * Returns TRUE if any of the slots gets updated in this sync-cycle.
835 */
836 static bool
837 synchronize_slots(WalReceiverConn *wrconn)
838 {
839 #define SLOTSYNC_COLUMN_COUNT 10
840 Oid slotRow[SLOTSYNC_COLUMN_COUNT] = {TEXTOID, TEXTOID, LSNOID,
841 LSNOID, XIDOID, BOOLOID, LSNOID, BOOLOID, TEXTOID, TEXTOID};
842
843 WalRcvExecResult *res;
844 TupleTableSlot *tupslot;
845 List *remote_slot_list = NIL;
846 bool some_slot_updated = false;
847 bool started_tx = false;
848 const char *query = "SELECT slot_name, plugin, confirmed_flush_lsn,"
849 " restart_lsn, catalog_xmin, two_phase, two_phase_at, failover,"
850 " database, invalidation_reason"
851 " FROM pg_catalog.pg_replication_slots"
852 " WHERE failover and NOT temporary";
853
854 /* The syscache access in walrcv_exec() needs a transaction env. */
855 if (!IsTransactionState())
856 {
857 StartTransactionCommand();
858 started_tx = true;
859 }
860
861 /* Execute the query */
862 res = walrcv_exec(wrconn, query, SLOTSYNC_COLUMN_COUNT, slotRow);
863 if (res->status != WALRCV_OK_TUPLES)
864 ereport(ERROR,
865 errmsg("could not fetch failover logical slots info from the primary server: %s",
866 res->err));
867
868 /* Construct the remote_slot tuple and synchronize each slot locally */
869 tupslot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
870 while (tuplestore_gettupleslot(res->tuplestore, true, false, tupslot))
871 {
872 bool isnull;
873 RemoteSlot *remote_slot = palloc0(sizeof(RemoteSlot));
874 Datum d;
875 int col = 0;
876
877 remote_slot->name = TextDatumGetCString(slot_getattr(tupslot, ++col,
878 &isnull));
879 Assert(!isnull);
880
881 remote_slot->plugin = TextDatumGetCString(slot_getattr(tupslot, ++col,
882 &isnull));
883 Assert(!isnull);
884
885 /*
886 * It is possible to get null values for LSN and Xmin if slot is
887 * invalidated on the primary server, so handle accordingly.
888 */
889 d = slot_getattr(tupslot, ++col, &isnull);
890 remote_slot->confirmed_lsn = isnull ? InvalidXLogRecPtr :
891 DatumGetLSN(d);
892
893 d = slot_getattr(tupslot, ++col, &isnull);
894 remote_slot->restart_lsn = isnull ? InvalidXLogRecPtr : DatumGetLSN(d);
895
896 d = slot_getattr(tupslot, ++col, &isnull);
897 remote_slot->catalog_xmin = isnull ? InvalidTransactionId :
898 DatumGetTransactionId(d);
899
900 remote_slot->two_phase = DatumGetBool(slot_getattr(tupslot, ++col,
901 &isnull));
902 Assert(!isnull);
903
904 d = slot_getattr(tupslot, ++col, &isnull);
905 remote_slot->two_phase_at = isnull ? InvalidXLogRecPtr : DatumGetLSN(d);
906
907 remote_slot->failover = DatumGetBool(slot_getattr(tupslot, ++col,
908 &isnull));
909 Assert(!isnull);
910
911 remote_slot->database = TextDatumGetCString(slot_getattr(tupslot,
912 ++col, &isnull));
913 Assert(!isnull);
914
915 d = slot_getattr(tupslot, ++col, &isnull);
916 remote_slot->invalidated = isnull ? RS_INVAL_NONE :
917 GetSlotInvalidationCause(TextDatumGetCString(d));
918
919 /* Sanity check */
920 Assert(col == SLOTSYNC_COLUMN_COUNT);
921
922 /*
923 * If restart_lsn, confirmed_lsn or catalog_xmin is invalid but the
924 * slot is valid, that means we have fetched the remote_slot in its
925 * RS_EPHEMERAL state. In such a case, don't sync it; we can always
926 * sync it in the next sync cycle when the remote_slot is persisted
927 * and has valid lsn(s) and xmin values.
928 *
929 * XXX: In future, if we plan to expose 'slot->data.persistency' in
930 * pg_replication_slots view, then we can avoid fetching RS_EPHEMERAL
931 * slots in the first place.
932 */
933 if ((XLogRecPtrIsInvalid(remote_slot->restart_lsn) ||
934 XLogRecPtrIsInvalid(remote_slot->confirmed_lsn) ||
935 !TransactionIdIsValid(remote_slot->catalog_xmin)) &&
936 remote_slot->invalidated == RS_INVAL_NONE)
937 pfree(remote_slot);
938 else
939 /* Create list of remote slots */
940 remote_slot_list = lappend(remote_slot_list, remote_slot);
941
942 ExecClearTuple(tupslot);
943 }
944
945 /* Drop local slots that no longer need to be synced. */
946 drop_local_obsolete_slots(remote_slot_list);
947
948 /* Now sync the slots locally */
949 foreach_ptr(RemoteSlot, remote_slot, remote_slot_list)
950 {
951 Oid remote_dbid = get_database_oid(remote_slot->database, false);
952
953 /*
954 * Use shared lock to prevent a conflict with
955 * ReplicationSlotsDropDBSlots(), trying to drop the same slot during
956 * a drop-database operation.
957 */
958 LockSharedObject(DatabaseRelationId, remote_dbid, 0, AccessShareLock);
959
960 some_slot_updated |= synchronize_one_slot(remote_slot, remote_dbid);
961
962 UnlockSharedObject(DatabaseRelationId, remote_dbid, 0, AccessShareLock);
963 }
964
965 /* We are done, free remote_slot_list elements */
966 list_free_deep(remote_slot_list);
967
968 walrcv_clear_result(res);
969
970 if (started_tx)
971 CommitTransactionCommand();
972
973 return some_slot_updated;
974 }
975
976 /*
977 * Checks the remote server info.
978 *
979 * We ensure that the 'primary_slot_name' exists on the remote server and the
980 * remote server is not a standby node.
981 */
982 static void
983 validate_remote_info(WalReceiverConn *wrconn)
984 {
985 #define PRIMARY_INFO_OUTPUT_COL_COUNT 2
986 WalRcvExecResult *res;
987 Oid slotRow[PRIMARY_INFO_OUTPUT_COL_COUNT] = {BOOLOID, BOOLOID};
988 StringInfoData cmd;
989 bool isnull;
990 TupleTableSlot *tupslot;
991 bool remote_in_recovery;
992 bool primary_slot_valid;
993 bool started_tx = false;
994
995 initStringInfo(&cmd);
996 appendStringInfo(&cmd,
997 "SELECT pg_is_in_recovery(), count(*) = 1"
998 " FROM pg_catalog.pg_replication_slots"
999 " WHERE slot_type='physical' AND slot_name=%s",
1000 quote_literal_cstr(PrimarySlotName));
1001
1002 /* The syscache access in walrcv_exec() needs a transaction env. */
1003 if (!IsTransactionState())
1004 {
1005 StartTransactionCommand();
1006 started_tx = true;
1007 }
1008
1009 res = walrcv_exec(wrconn, cmd.data, PRIMARY_INFO_OUTPUT_COL_COUNT, slotRow);
1010 pfree(cmd.data);
1011
1012 if (res->status != WALRCV_OK_TUPLES)
1013 ereport(ERROR,
1014 errmsg("could not fetch primary slot name \"%s\" info from the primary server: %s",
1015 PrimarySlotName, res->err),
1016 errhint("Check if \"primary_slot_name\" is configured correctly."));
1017
1018 tupslot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
1019 if (!tuplestore_gettupleslot(res->tuplestore, true, false, tupslot))
1020 elog(ERROR,
1021 "failed to fetch tuple for the primary server slot specified by \"primary_slot_name\"");
1022
1023 remote_in_recovery = DatumGetBool(slot_getattr(tupslot, 1, &isnull));
1024 Assert(!isnull);
1025
1026 /*
1027 * Slot sync is currently not supported on a cascading standby. This is
1028 * because if we allow it, the primary server needs to wait for all the
1029 * cascading standbys, otherwise, logical subscribers can still be ahead
1030 * of one of the cascading standbys which we plan to promote. Thus, to
1031 * avoid this additional complexity, we restrict it for the time being.
1032 */
1033 if (remote_in_recovery)
1034 ereport(ERROR,
1035 errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1036 errmsg("cannot synchronize replication slots from a standby server"));
1037
1038 primary_slot_valid = DatumGetBool(slot_getattr(tupslot, 2, &isnull));
1039 Assert(!isnull);
1040
1041 if (!primary_slot_valid)
1042 ereport(ERROR,
1043 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1044 /* translator: second %s is a GUC variable name */
1045 errmsg("replication slot \"%s\" specified by \"%s\" does not exist on primary server",
1046 PrimarySlotName, "primary_slot_name"));
1047
1048 ExecClearTuple(tupslot);
1049 walrcv_clear_result(res);
1050
1051 if (started_tx)
1052 CommitTransactionCommand();
1053 }
1054
1055 /*
1056 * Checks if dbname is specified in 'primary_conninfo'.
1057 *
1058 * Error out if not specified otherwise return it.
1059 */
1060 char *
1061 CheckAndGetDbnameFromConninfo(void)
1062 {
1063 char *dbname;
1064
1065 /*
1066 * The slot synchronization needs a database connection for walrcv_exec to
1067 * work.
1068 */
1069 dbname = walrcv_get_dbname_from_conninfo(PrimaryConnInfo);
1070 if (dbname == NULL)
1071 ereport(ERROR,
1072 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1073
1074 /*
1075 * translator: first %s is a connection option; second %s is a GUC
1076 * variable name
1077 */
1078 errmsg("replication slot synchronization requires \"%s\" to be specified in \"%s\"",
1079 "dbname", "primary_conninfo"));
1080 return dbname;
1081 }
1082
1083 /*
1084 * Return true if all necessary GUCs for slot synchronization are set
1085 * appropriately, otherwise, return false.
1086 */
1087 bool
1088 ValidateSlotSyncParams(int elevel)
1089 {
1090 /*
1091 * Logical slot sync/creation requires wal_level >= logical.
1092 */
1093 if (wal_level < WAL_LEVEL_LOGICAL)
1094 {
1095 ereport(elevel,
1096 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1097 errmsg("replication slot synchronization requires \"wal_level\" >= \"logical\""));
1098 return false;
1099 }
1100
1101 /*
1102 * A physical replication slot(primary_slot_name) is required on the
1103 * primary to ensure that the rows needed by the standby are not removed
1104 * after restarting, so that the synchronized slot on the standby will not
1105 * be invalidated.
1106 */
1107 if (PrimarySlotName == NULL || *PrimarySlotName == '\0')
1108 {
1109 ereport(elevel,
1110 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1111 /* translator: %s is a GUC variable name */
1112 errmsg("replication slot synchronization requires \"%s\" to be set", "primary_slot_name"));
1113 return false;
1114 }
1115
1116 /*
1117 * hot_standby_feedback must be enabled to cooperate with the physical
1118 * replication slot, which allows informing the primary about the xmin and
1119 * catalog_xmin values on the standby.
1120 */
1121 if (!hot_standby_feedback)
1122 {
1123 ereport(elevel,
1124 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1125 /* translator: %s is a GUC variable name */
1126 errmsg("replication slot synchronization requires \"%s\" to be enabled",
1127 "hot_standby_feedback"));
1128 return false;
1129 }
1130
1131 /*
1132 * The primary_conninfo is required to make connection to primary for
1133 * getting slots information.
1134 */
1135 if (PrimaryConnInfo == NULL || *PrimaryConnInfo == '\0')
1136 {
1137 ereport(elevel,
1138 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1139 /* translator: %s is a GUC variable name */
1140 errmsg("replication slot synchronization requires \"%s\" to be set",
1141 "primary_conninfo"));
1142 return false;
1143 }
1144
1145 return true;
1146 }
1147
1148 /*
1149 * Re-read the config file for slot synchronization.
1150 *
1151 * Exit or throw error if relevant GUCs have changed depending on whether
1152 * called from slot sync worker or from the SQL function pg_sync_replication_slots()
1153 */
1154 static void
1155 slotsync_reread_config(void)
1156 {
1157 char *old_primary_conninfo = pstrdup(PrimaryConnInfo);
1158 char *old_primary_slotname = pstrdup(PrimarySlotName);
1159 bool old_sync_replication_slots = sync_replication_slots;
1160 bool old_hot_standby_feedback = hot_standby_feedback;
1161 bool conninfo_changed;
1162 bool primary_slotname_changed;
1163 bool is_slotsync_worker = AmLogicalSlotSyncWorkerProcess();
1164 bool parameter_changed = false;
1165
1166 if (is_slotsync_worker)
1167 Assert(sync_replication_slots);
1168
1169 ConfigReloadPending = false;
1170 ProcessConfigFile(PGC_SIGHUP);
1171
1172 conninfo_changed = strcmp(old_primary_conninfo, PrimaryConnInfo) != 0;
1173 primary_slotname_changed = strcmp(old_primary_slotname, PrimarySlotName) != 0;
1174 pfree(old_primary_conninfo);
1175 pfree(old_primary_slotname);
1176
1177 if (old_sync_replication_slots != sync_replication_slots)
1178 {
1179 if (is_slotsync_worker)
1180 {
1181 ereport(LOG,
1182 /* translator: %s is a GUC variable name */
1183 errmsg("replication slot synchronization worker will stop because \"%s\" is disabled",
1184 "sync_replication_slots"));
1185
1186 proc_exit(0);
1187 }
1188
1189 parameter_changed = true;
1190 }
1191 else
1192 {
1193 if (conninfo_changed ||
1194 primary_slotname_changed ||
1195 (old_hot_standby_feedback != hot_standby_feedback))
1196 {
1197
1198 if (is_slotsync_worker)
1199 {
1200 ereport(LOG,
1201 errmsg("replication slot synchronization worker will restart because of a parameter change"));
1202
1203 /*
1204 * Reset the last-start time for this worker so that the
1205 * postmaster can restart it without waiting for
1206 * SLOTSYNC_RESTART_INTERVAL_SEC.
1207 */
1208 SlotSyncCtx->last_start_time = 0;
1209
1210 proc_exit(0);
1211 }
1212
1213 parameter_changed = true;
1214 }
1215 }
1216
1217 /*
1218 * If we have reached here with a parameter change, we must be running in
1219 * SQL function, emit error in such a case.
1220 */
1221 if (parameter_changed)
1222 {
1223 Assert(!is_slotsync_worker);
1224 ereport(ERROR,
1225 errcode(ERRCODE_INVALID_PARAMETER_VALUE),
1226 errmsg("replication slot synchronization will stop because of a parameter change"));
1227 }
1228
1229 }
1230
1231 /*
1232 * Handle receipt of an interrupt indicating a slotsync shutdown message.
1233 *
1234 * This is called within the SIGUSR1 handler. All we do here is set a flag
1235 * that will cause the next CHECK_FOR_INTERRUPTS() to invoke
1236 * ProcessSlotSyncMessage().
1237 */
1238 void
1239 HandleSlotSyncMessageInterrupt(void)
1240 {
1241 InterruptPending = true;
1242 SlotSyncShutdownPending = true;
1243 /* latch will be set by procsignal_sigusr1_handler */
1244 }
1245
1246 /*
1247 * Handle a PROCSIG_SLOTSYNC_MESSAGE signal, called from ProcessInterrupts().
1248 *
1249 * If the current process is the slotsync background worker, log a message
1250 * and exit cleanly. If it is a backend executing pg_sync_replication_slots(),
1251 * raise an error, unless the sync has already finished, in which case there
1252 * is no need to interrupt the caller.
1253 */
1254 void
1255 ProcessSlotSyncMessage(void)
1256 {
1257 SlotSyncShutdownPending = false;
1258
1259 if (AmLogicalSlotSyncWorkerProcess())
1260 {
1261 ereport(LOG,
1262 errmsg("replication slot synchronization worker will stop because promotion is triggered"));
1263 proc_exit(0);
1264 }
1265 else
1266 {
1267 /*
1268 * If sync has already completed, there is no need to interrupt the
1269 * caller with an error.
1270 */
1271 if (!IsSyncingReplicationSlots())
1272 return;
1273
1274 ereport(ERROR,
1275 errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1276 errmsg("replication slot synchronization will stop because promotion is triggered"));
1277 }
1278 }
1279
1280 /*
1281 * Connection cleanup function for slotsync worker.
1282 *
1283 * Called on slotsync worker exit.
1284 */
1285 static void
1286 slotsync_worker_disconnect(int code, Datum arg)
1287 {
1288 WalReceiverConn *wrconn = (WalReceiverConn *) DatumGetPointer(arg);
1289
1290 walrcv_disconnect(wrconn);
1291 }
1292
1293 /*
1294 * Cleanup function for slotsync worker.
1295 *
1296 * Called on slotsync worker exit.
1297 */
1298 static void
1299 slotsync_worker_onexit(int code, Datum arg)
1300 {
1301 /*
1302 * We need to do slots cleanup here just like WalSndErrorCleanup() does.
1303 *
1304 * The startup process during promotion invokes ShutDownSlotSync() which
1305 * waits for slot sync to finish and it does that by checking the
1306 * 'syncing' flag. Thus the slot sync worker must be done with slots'
1307 * release and cleanup to avoid any dangling temporary slots or active
1308 * slots before it marks itself as finished syncing.
1309 */
1310
1311 /* Make sure active replication slots are released */
1312 if (MyReplicationSlot != NULL)
1313 ReplicationSlotRelease();
1314
1315 /* Also cleanup the temporary slots. */
1316 ReplicationSlotCleanup(false);
1317
1318 SpinLockAcquire(&SlotSyncCtx->mutex);
1319
1320 SlotSyncCtx->pid = InvalidPid;
1321
1322 /*
1323 * If syncing_slots is true, it indicates that the process errored out
1324 * without resetting the flag. So, we need to clean up shared memory and
1325 * reset the flag here.
1326 */
1327 if (syncing_slots)
1328 {
1329 SlotSyncCtx->syncing = false;
1330 syncing_slots = false;
1331 }
1332
1333 SpinLockRelease(&SlotSyncCtx->mutex);
1334 }
1335
1336 /*
1337 * Sleep for long enough that we believe it's likely that the slots on primary
1338 * get updated.
1339 *
1340 * If there is no slot activity the wait time between sync-cycles will double
1341 * (to a maximum of 30s). If there is some slot activity the wait time between
1342 * sync-cycles is reset to the minimum (200ms).
1343 */
1344 static void
1345 wait_for_slot_activity(bool some_slot_updated)
1346 {
1347 int rc;
1348
1349 if (!some_slot_updated)
1350 {
1351 /*
1352 * No slots were updated, so double the sleep time, but not beyond the
1353 * maximum allowable value.
1354 */
1355 sleep_ms = Min(sleep_ms * 2, MAX_SLOTSYNC_WORKER_NAPTIME_MS);
1356 }
1357 else
1358 {
1359 /*
1360 * Some slots were updated since the last sleep, so reset the sleep
1361 * time.
1362 */
1363 sleep_ms = MIN_SLOTSYNC_WORKER_NAPTIME_MS;
1364 }
1365
1366 rc = WaitLatch(MyLatch,
1367 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
1368 sleep_ms,
1369 WAIT_EVENT_REPLICATION_SLOTSYNC_MAIN);
1370
1371 if (rc & WL_LATCH_SET)
1372 ResetLatch(MyLatch);
1373 }
1374
1375 /*
1376 * Emit an error if a concurrent sync call is in progress.
1377 * Otherwise, advertise that a sync is in progress.
1378 */
1379 static void
1380 check_and_set_sync_info(pid_t sync_process_pid)
1381 {
1382 SpinLockAcquire(&SlotSyncCtx->mutex);
1383
1384 /*
1385 * Exit immediately if promotion has been triggered. This guards against
1386 * a new worker (or a call to pg_sync_replication_slots()) that starts
1387 * after the old worker was stopped by ShutDownSlotSync().
1388 */
1389 if (SlotSyncCtx->stopSignaled)
1390 {
1391 SpinLockRelease(&SlotSyncCtx->mutex);
1392
1393 if (AmLogicalSlotSyncWorkerProcess())
1394 {
1395 ereport(DEBUG1,
1396 errmsg("replication slot synchronization worker will not start because promotion was triggered"));
1397
1398 proc_exit(0);
1399 }
1400 else
1401 {
1402 /*
1403 * For the backend executing SQL function
1404 * pg_sync_replication_slots().
1405 */
1406 ereport(ERROR,
1407 errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1408 errmsg("replication slot synchronization will not start because promotion was triggered"));
1409 }
1410 }
1411
1412 if (SlotSyncCtx->syncing)
1413 {
1414 SpinLockRelease(&SlotSyncCtx->mutex);
1415 ereport(ERROR,
1416 errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1417 errmsg("cannot synchronize replication slots concurrently"));
1418 }
1419
1420 /* The pid must not be already assigned in SlotSyncCtx */
1421 Assert(SlotSyncCtx->pid == InvalidPid);
1422
1423 SlotSyncCtx->syncing = true;
1424
1425 /*
1426 * Advertise the required PID so that the startup process can kill the
1427 * slot sync process on promotion.
1428 */
1429 SlotSyncCtx->pid = sync_process_pid;
1430
1431 SpinLockRelease(&SlotSyncCtx->mutex);
1432
1433 syncing_slots = true;
1434 }
1435
1436 /*
1437 * Reset syncing flag.
1438 */
1439 static void
1440 reset_syncing_flag()
1441 {
1442 SpinLockAcquire(&SlotSyncCtx->mutex);
1443 SlotSyncCtx->syncing = false;
1444 SlotSyncCtx->pid = InvalidPid;
1445 SpinLockRelease(&SlotSyncCtx->mutex);
1446
1447 syncing_slots = false;
1448 }
1449
1450 /*
1451 * The main loop of our worker process.
1452 *
1453 * It connects to the primary server, fetches logical failover slots
1454 * information periodically in order to create and sync the slots.
1455 */
1456 void
1457 ReplSlotSyncWorkerMain(const void *startup_data, size_t startup_data_len)
1458 {
1459 WalReceiverConn *wrconn = NULL;
1460 char *dbname;
1461 char *err;
1462 sigjmp_buf local_sigjmp_buf;
1463 StringInfoData app_name;
1464
1465 Assert(startup_data_len == 0);
1466
1467 MyBackendType = B_SLOTSYNC_WORKER;
1468
1469 init_ps_display(NULL);
1470
1471 Assert(GetProcessingMode() == InitProcessing);
1472
1473 /*
1474 * Create a per-backend PGPROC struct in shared memory. We must do this
1475 * before we access any shared memory.
1476 */
1477 InitProcess();
1478
1479 /*
1480 * Early initialization.
1481 */
1482 BaseInit();
1483
1484 Assert(SlotSyncCtx != NULL);
1485
1486 /*
1487 * If an exception is encountered, processing resumes here.
1488 *
1489 * We just need to clean up, report the error, and go away.
1490 *
1491 * If we do not have this handling here, then since this worker process
1492 * operates at the bottom of the exception stack, ERRORs turn into FATALs.
1493 * Therefore, we create our own exception handler to catch ERRORs.
1494 */
1495 if (sigsetjmp(local_sigjmp_buf, 1) != 0)
1496 {
1497 /* since not using PG_TRY, must reset error stack by hand */
1498 error_context_stack = NULL;
1499
1500 /* Prevents interrupts while cleaning up */
1501 HOLD_INTERRUPTS();
1502
1503 /* Report the error to the server log */
1504 EmitErrorReport();
1505
1506 /*
1507 * We can now go away. Note that because we called InitProcess, a
1508 * callback was registered to do ProcKill, which will clean up
1509 * necessary state.
1510 */
1511 proc_exit(0);
1512 }
1513
1514 /* We can now handle ereport(ERROR) */
1515 PG_exception_stack = &local_sigjmp_buf;
1516
1517 /* Setup signal handling */
1518 pqsignal(SIGHUP, SignalHandlerForConfigReload);
1519 pqsignal(SIGINT, StatementCancelHandler);
1520 pqsignal(SIGTERM, die);
1521 pqsignal(SIGFPE, FloatExceptionHandler);
1522 pqsignal(SIGUSR1, procsignal_sigusr1_handler);
1523 pqsignal(SIGUSR2, SIG_IGN);
1524 pqsignal(SIGPIPE, SIG_IGN);
1525 pqsignal(SIGCHLD, SIG_DFL);
1526
1527 check_and_set_sync_info(MyProcPid);
1528
1529 ereport(LOG, errmsg("slot sync worker started"));
1530
1531 /* Register it as soon as SlotSyncCtx->pid is initialized. */
1532 before_shmem_exit(slotsync_worker_onexit, (Datum) 0);
1533
1534 /*
1535 * Establishes SIGALRM handler and initialize timeout module. It is needed
1536 * by InitPostgres to register different timeouts.
1537 */
1538 InitializeTimeouts();
1539
1540 /* Load the libpq-specific functions */
1541 load_file("libpqwalreceiver", false);
1542
1543 /*
1544 * Unblock signals (they were blocked when the postmaster forked us)
1545 */
1546 sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
1547
1548 /*
1549 * Set always-secure search path, so malicious users can't redirect user
1550 * code (e.g. operators).
1551 *
1552 * It's not strictly necessary since we won't be scanning or writing to
1553 * any user table locally, but it's good to retain it here for added
1554 * precaution.
1555 */
1556 SetConfigOption("search_path", "", PGC_SUSET, PGC_S_OVERRIDE);
1557
1558 dbname = CheckAndGetDbnameFromConninfo();
1559
1560 /*
1561 * Connect to the database specified by the user in primary_conninfo. We
1562 * need a database connection for walrcv_exec to work which we use to
1563 * fetch slot information from the remote node. See comments atop
1564 * libpqrcv_exec.
1565 *
1566 * We do not specify a specific user here since the slot sync worker will
1567 * operate as a superuser. This is safe because the slot sync worker does
1568 * not interact with user tables, eliminating the risk of executing
1569 * arbitrary code within triggers.
1570 */
1571 InitPostgres(dbname, InvalidOid, NULL, InvalidOid, 0, NULL);
1572
1573 SetProcessingMode(NormalProcessing);
1574
1575 initStringInfo(&app_name);
1576 if (cluster_name[0])
1577 appendStringInfo(&app_name, "%s_%s", cluster_name, "slotsync worker");
1578 else
1579 appendStringInfoString(&app_name, "slotsync worker");
1580
1581 /*
1582 * Establish the connection to the primary server for slot
1583 * synchronization.
1584 */
1585 wrconn = walrcv_connect(PrimaryConnInfo, false, false, false,
1586 app_name.data, &err);
1587
1588 if (!wrconn)
1589 ereport(ERROR,
1590 errcode(ERRCODE_CONNECTION_FAILURE),
1591 errmsg("synchronization worker \"%s\" could not connect to the primary server: %s",
1592 app_name.data, err));
1593
1594 pfree(app_name.data);
1595
1596 /*
1597 * Register the disconnection callback.
1598 *
1599 * XXX: This can be combined with previous cleanup registration of
1600 * slotsync_worker_onexit() but that will need the connection to be made
1601 * global and we want to avoid introducing global for this purpose.
1602 */
1603 before_shmem_exit(slotsync_worker_disconnect, PointerGetDatum(wrconn));
1604
1605 /*
1606 * Using the specified primary server connection, check that we are not a
1607 * cascading standby and slot configured in 'primary_slot_name' exists on
1608 * the primary server.
1609 */
1610 validate_remote_info(wrconn);
1611
1612 /* Main loop to synchronize slots */
1613 for (;;)
1614 {
1615 bool some_slot_updated = false;
1616
1617 CHECK_FOR_INTERRUPTS();
1618
1619 if (ConfigReloadPending)
1620 slotsync_reread_config();
1621
1622 some_slot_updated = synchronize_slots(wrconn);
1623
1624 wait_for_slot_activity(some_slot_updated);
1625 }
1626
1627 /*
1628 * The slot sync worker can't get here because it will only stop when it
1629 * receives a stop request from the startup process, or when there is an
1630 * error.
1631 */
1632 Assert(false);
1633 }
1634
1635 /*
1636 * Update the inactive_since property for synced slots.
1637 *
1638 * Note that this function is currently called when we shutdown the slot
1639 * sync machinery.
1640 */
1641 static void
1642 update_synced_slots_inactive_since(void)
1643 {
1644 TimestampTz now = 0;
1645
1646 /*
1647 * We need to update inactive_since only when we are promoting standby to
1648 * correctly interpret the inactive_since if the standby gets promoted
1649 * without a restart. We don't want the slots to appear inactive for a
1650 * long time after promotion if they haven't been synchronized recently.
1651 * Whoever acquires the slot, i.e., makes the slot active, will reset it.
1652 */
1653 if (!StandbyMode)
1654 return;
1655
1656 /* The slot sync worker or the SQL function mustn't be running by now */
1657 Assert((SlotSyncCtx->pid == InvalidPid) && !SlotSyncCtx->syncing);
1658
1659 LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
1660
1661 for (int i = 0; i < max_replication_slots; i++)
1662 {
1663 ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
1664
1665 /* Check if it is a synchronized slot */
1666 if (s->in_use && s->data.synced)
1667 {
1668 Assert(SlotIsLogical(s));
1669
1670 /* The slot must not be acquired by any process */
1671 Assert(s->active_pid == 0);
1672
1673 /* Use the same inactive_since time for all the slots. */
1674 if (now == 0)
1675 now = GetCurrentTimestamp();
1676
1677 ReplicationSlotSetInactiveSince(s, now, true);
1678 }
1679 }
1680
1681 LWLockRelease(ReplicationSlotControlLock);
1682 }
1683
1684 /*
1685 * Shut down slot synchronization.
1686 *
1687 * This function sets stopSignaled=true and wakes up the slot sync process
1688 * (either worker or backend running the SQL function pg_sync_replication_slots())
1689 * so that worker can exit or the SQL function pg_sync_replication_slots() can
1690 * finish. It also waits till the slot sync worker has exited or
1691 * pg_sync_replication_slots() has finished.
1692 */
1693 void
1694 ShutDownSlotSync(void)
1695 {
1696 pid_t sync_process_pid;
1697
1698 SpinLockAcquire(&SlotSyncCtx->mutex);
1699
1700 SlotSyncCtx->stopSignaled = true;
1701
1702 /*
1703 * Return if neither the slot sync worker is running nor the function
1704 * pg_sync_replication_slots() is executing.
1705 */
1706 if (!SlotSyncCtx->syncing)
1707 {
1708 SpinLockRelease(&SlotSyncCtx->mutex);
1709 update_synced_slots_inactive_since();
1710 return;
1711 }
1712
1713 sync_process_pid = SlotSyncCtx->pid;
1714
1715 SpinLockRelease(&SlotSyncCtx->mutex);
1716
1717 /*
1718 * Signal process doing slotsync, if any, asking it to stop.
1719 */
1720 if (sync_process_pid != InvalidPid)
1721 SendProcSignal(sync_process_pid, PROCSIG_SLOTSYNC_MESSAGE,
1722 INVALID_PROC_NUMBER);
1723
1724 /* Wait for slot sync to end */
1725 for (;;)
1726 {
1727 int rc;
1728
1729 /* Wait a bit, we don't expect to have to wait long */
1730 rc = WaitLatch(MyLatch,
1731 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
1732 10L, WAIT_EVENT_REPLICATION_SLOTSYNC_SHUTDOWN);
1733
1734 if (rc & WL_LATCH_SET)
1735 {
1736 ResetLatch(MyLatch);
1737 CHECK_FOR_INTERRUPTS();
1738 }
1739
1740 SpinLockAcquire(&SlotSyncCtx->mutex);
1741
1742 /* Ensure that no process is syncing the slots. */
1743 if (!SlotSyncCtx->syncing)
1744 break;
1745
1746 SpinLockRelease(&SlotSyncCtx->mutex);
1747 }
1748
1749 SpinLockRelease(&SlotSyncCtx->mutex);
1750
1751 update_synced_slots_inactive_since();
1752 }
1753
1754 /*
1755 * SlotSyncWorkerCanRestart
1756 *
1757 * Returns true if enough time (SLOTSYNC_RESTART_INTERVAL_SEC) has passed
1758 * since it was launched last. Otherwise returns false.
1759 *
1760 * This is a safety valve to protect against continuous respawn attempts if the
1761 * worker is dying immediately at launch. Note that since we will retry to
1762 * launch the worker from the postmaster main loop, we will get another
1763 * chance later.
1764 */
1765 bool
1766 SlotSyncWorkerCanRestart(void)
1767 {
1768 time_t curtime = time(NULL);
1769
1770 /* Return false if too soon since last start. */
1771 if ((unsigned int) (curtime - SlotSyncCtx->last_start_time) <
1772 (unsigned int) SLOTSYNC_RESTART_INTERVAL_SEC)
1773 return false;
1774
1775 SlotSyncCtx->last_start_time = curtime;
1776
1777 return true;
1778 }
1779
1780 /*
1781 * Is current process syncing replication slots?
1782 *
1783 * Could be either backend executing SQL function or slot sync worker.
1784 */
1785 bool
1786 IsSyncingReplicationSlots(void)
1787 {
1788 return syncing_slots;
1789 }
1790
1791 /*
1792 * Amount of shared memory required for slot synchronization.
1793 */
1794 Size
1795 SlotSyncShmemSize(void)
1796 {
1797 return sizeof(SlotSyncCtxStruct);
1798 }
1799
1800 /*
1801 * Allocate and initialize the shared memory of slot synchronization.
1802 */
1803 void
1804 SlotSyncShmemInit(void)
1805 {
1806 Size size = SlotSyncShmemSize();
1807 bool found;
1808
1809 SlotSyncCtx = (SlotSyncCtxStruct *)
1810 ShmemInitStruct("Slot Sync Data", size, &found);
1811
1812 if (!found)
1813 {
1814 memset(SlotSyncCtx, 0, size);
1815 SlotSyncCtx->pid = InvalidPid;
1816 SpinLockInit(&SlotSyncCtx->mutex);
1817 }
1818 }
1819
1820 /*
1821 * Error cleanup callback for slot sync SQL function.
1822 */
1823 static void
1824 slotsync_failure_callback(int code, Datum arg)
1825 {
1826 WalReceiverConn *wrconn = (WalReceiverConn *) DatumGetPointer(arg);
1827
1828 /*
1829 * We need to do slots cleanup here just like WalSndErrorCleanup() does.
1830 *
1831 * The startup process during promotion invokes ShutDownSlotSync() which
1832 * waits for slot sync to finish and it does that by checking the
1833 * 'syncing' flag. Thus the SQL function must be done with slots' release
1834 * and cleanup to avoid any dangling temporary slots or active slots
1835 * before it marks itself as finished syncing.
1836 */
1837
1838 /* Make sure active replication slots are released */
1839 if (MyReplicationSlot != NULL)
1840 ReplicationSlotRelease();
1841
1842 /* Also cleanup the synced temporary slots. */
1843 ReplicationSlotCleanup(true);
1844
1845 /*
1846 * The set syncing_slots indicates that the process errored out without
1847 * resetting the flag. So, we need to clean up shared memory and reset the
1848 * flag here.
1849 */
1850 if (syncing_slots)
1851 reset_syncing_flag();
1852
1853 walrcv_disconnect(wrconn);
1854 }
1855
1856 /*
1857 * Synchronize the failover enabled replication slots using the specified
1858 * primary server connection.
1859 */
1860 void
1861 SyncReplicationSlots(WalReceiverConn *wrconn)
1862 {
1863 PG_ENSURE_ERROR_CLEANUP(slotsync_failure_callback, PointerGetDatum(wrconn));
1864 {
1865 check_and_set_sync_info(MyProcPid);
1866
1867 validate_remote_info(wrconn);
1868
1869 synchronize_slots(wrconn);
1870
1871 /* Cleanup the synced temporary slots */
1872 ReplicationSlotCleanup(true);
1873
1874 /* We are done with sync, so reset sync flag */
1875 reset_syncing_flag();
1876 }
1877 PG_END_ENSURE_ERROR_CLEANUP(slotsync_failure_callback, PointerGetDatum(wrconn));
1878 }