Debian Codegraph @

src/backend/replication/logical/tablesync.c

raw
1 /*-------------------------------------------------------------------------
2 * tablesync.c
3 * PostgreSQL logical replication: initial table data synchronization
4 *
5 * Copyright (c) 2012-2025, PostgreSQL Global Development Group
6 *
7 * IDENTIFICATION
8 * src/backend/replication/logical/tablesync.c
9 *
10 * NOTES
11 * This file contains code for initial table data synchronization for
12 * logical replication.
13 *
14 * The initial data synchronization is done separately for each table,
15 * in a separate apply worker that only fetches the initial snapshot data
16 * from the publisher and then synchronizes the position in the stream with
17 * the leader apply worker.
18 *
19 * There are several reasons for doing the synchronization this way:
20 * - It allows us to parallelize the initial data synchronization
21 * which lowers the time needed for it to happen.
22 * - The initial synchronization does not have to hold the xid and LSN
23 * for the time it takes to copy data of all tables, causing less
24 * bloat and lower disk consumption compared to doing the
25 * synchronization in a single process for the whole database.
26 * - It allows us to synchronize any tables added after the initial
27 * synchronization has finished.
28 *
29 * The stream position synchronization works in multiple steps:
30 * - Apply worker requests a tablesync worker to start, setting the new
31 * table state to INIT.
32 * - Tablesync worker starts; changes table state from INIT to DATASYNC while
33 * copying.
34 * - Tablesync worker does initial table copy; there is a FINISHEDCOPY (sync
35 * worker specific) state to indicate when the copy phase has completed, so
36 * if the worker crashes with this (non-memory) state then the copy will not
37 * be re-attempted.
38 * - Tablesync worker then sets table state to SYNCWAIT; waits for state change.
39 * - Apply worker periodically checks for tables in SYNCWAIT state. When
40 * any appear, it sets the table state to CATCHUP and starts loop-waiting
41 * until either the table state is set to SYNCDONE or the sync worker
42 * exits.
43 * - After the sync worker has seen the state change to CATCHUP, it will
44 * read the stream and apply changes (acting like an apply worker) until
45 * it catches up to the specified stream position. Then it sets the
46 * state to SYNCDONE. There might be zero changes applied between
47 * CATCHUP and SYNCDONE, because the sync worker might be ahead of the
48 * apply worker.
49 * - Once the state is set to SYNCDONE, the apply will continue tracking
50 * the table until it reaches the SYNCDONE stream position, at which
51 * point it sets state to READY and stops tracking. Again, there might
52 * be zero changes in between.
53 *
54 * So the state progression is always: INIT -> DATASYNC -> FINISHEDCOPY
55 * -> SYNCWAIT -> CATCHUP -> SYNCDONE -> READY.
56 *
57 * The catalog pg_subscription_rel is used to keep information about
58 * subscribed tables and their state. The catalog holds all states
59 * except SYNCWAIT and CATCHUP which are only in shared memory.
60 *
61 * Example flows look like this:
62 * - Apply is in front:
63 * sync:8
64 * -> set in catalog FINISHEDCOPY
65 * -> set in memory SYNCWAIT
66 * apply:10
67 * -> set in memory CATCHUP
68 * -> enter wait-loop
69 * sync:10
70 * -> set in catalog SYNCDONE
71 * -> exit
72 * apply:10
73 * -> exit wait-loop
74 * -> continue rep
75 * apply:11
76 * -> set in catalog READY
77 *
78 * - Sync is in front:
79 * sync:10
80 * -> set in catalog FINISHEDCOPY
81 * -> set in memory SYNCWAIT
82 * apply:8
83 * -> set in memory CATCHUP
84 * -> continue per-table filtering
85 * sync:10
86 * -> set in catalog SYNCDONE
87 * -> exit
88 * apply:10
89 * -> set in catalog READY
90 * -> stop per-table filtering
91 * -> continue rep
92 *-------------------------------------------------------------------------
93 */
94
95 #include "postgres.h"
96
97 #include "access/table.h"
98 #include "access/xact.h"
99 #include "catalog/indexing.h"
100 #include "catalog/pg_subscription_rel.h"
101 #include "catalog/pg_type.h"
102 #include "commands/copy.h"
103 #include "miscadmin.h"
104 #include "nodes/makefuncs.h"
105 #include "parser/parse_relation.h"
106 #include "pgstat.h"
107 #include "replication/logicallauncher.h"
108 #include "replication/logicalrelation.h"
109 #include "replication/logicalworker.h"
110 #include "replication/origin.h"
111 #include "replication/slot.h"
112 #include "replication/walreceiver.h"
113 #include "replication/worker_internal.h"
114 #include "storage/ipc.h"
115 #include "storage/lmgr.h"
116 #include "utils/acl.h"
117 #include "utils/array.h"
118 #include "utils/builtins.h"
119 #include "utils/lsyscache.h"
120 #include "utils/memutils.h"
121 #include "utils/rls.h"
122 #include "utils/snapmgr.h"
123 #include "utils/syscache.h"
124 #include "utils/usercontext.h"
125
126 typedef enum
127 {
128 SYNC_TABLE_STATE_NEEDS_REBUILD,
129 SYNC_TABLE_STATE_REBUILD_STARTED,
130 SYNC_TABLE_STATE_VALID,
131 } SyncingTablesState;
132
133 static SyncingTablesState table_states_validity = SYNC_TABLE_STATE_NEEDS_REBUILD;
134 static List *table_states_not_ready = NIL;
135 static bool FetchTableStates(bool *started_tx);
136
137 static StringInfo copybuf = NULL;
138
139 /*
140 * Exit routine for synchronization worker.
141 */
142 pg_noreturn static void
143 finish_sync_worker(void)
144 {
145 /*
146 * Commit any outstanding transaction. This is the usual case, unless
147 * there was nothing to do for the table.
148 */
149 if (IsTransactionState())
150 {
151 CommitTransactionCommand();
152 pgstat_report_stat(true);
153 }
154
155 /* And flush all writes. */
156 XLogFlush(GetXLogWriteRecPtr());
157
158 StartTransactionCommand();
159 ereport(LOG,
160 (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
161 MySubscription->name,
162 get_rel_name(MyLogicalRepWorker->relid))));
163 CommitTransactionCommand();
164
165 /* Find the leader apply worker and signal it. */
166 logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
167
168 /* Stop gracefully */
169 proc_exit(0);
170 }
171
172 /*
173 * Wait until the relation sync state is set in the catalog to the expected
174 * one; return true when it happens.
175 *
176 * Returns false if the table sync worker or the table itself have
177 * disappeared, or the table state has been reset.
178 *
179 * Currently, this is used in the apply worker when transitioning from
180 * CATCHUP state to SYNCDONE.
181 */
182 static bool
183 wait_for_relation_state_change(Oid relid, char expected_state)
184 {
185 char state;
186
187 for (;;)
188 {
189 LogicalRepWorker *worker;
190 XLogRecPtr statelsn;
191
192 CHECK_FOR_INTERRUPTS();
193
194 InvalidateCatalogSnapshot();
195 state = GetSubscriptionRelState(MyLogicalRepWorker->subid,
196 relid, &statelsn);
197
198 if (state == SUBREL_STATE_UNKNOWN)
199 break;
200
201 if (state == expected_state)
202 return true;
203
204 /* Check if the sync worker is still running and bail if not. */
205 LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
206 worker = logicalrep_worker_find(MyLogicalRepWorker->subid, relid,
207 false);
208 LWLockRelease(LogicalRepWorkerLock);
209 if (!worker)
210 break;
211
212 (void) WaitLatch(MyLatch,
213 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
214 1000L, WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE);
215
216 ResetLatch(MyLatch);
217 }
218
219 return false;
220 }
221
222 /*
223 * Wait until the apply worker changes the state of our synchronization
224 * worker to the expected one.
225 *
226 * Used when transitioning from SYNCWAIT state to CATCHUP.
227 *
228 * Returns false if the apply worker has disappeared.
229 */
230 static bool
231 wait_for_worker_state_change(char expected_state)
232 {
233 int rc;
234
235 for (;;)
236 {
237 LogicalRepWorker *worker;
238
239 CHECK_FOR_INTERRUPTS();
240
241 /*
242 * Done if already in correct state. (We assume this fetch is atomic
243 * enough to not give a misleading answer if we do it with no lock.)
244 */
245 if (MyLogicalRepWorker->relstate == expected_state)
246 return true;
247
248 /*
249 * Bail out if the apply worker has died, else signal it we're
250 * waiting.
251 */
252 LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
253 worker = logicalrep_worker_find(MyLogicalRepWorker->subid,
254 InvalidOid, false);
255 if (worker && worker->proc)
256 logicalrep_worker_wakeup_ptr(worker);
257 LWLockRelease(LogicalRepWorkerLock);
258 if (!worker)
259 break;
260
261 /*
262 * Wait. We expect to get a latch signal back from the apply worker,
263 * but use a timeout in case it dies without sending one.
264 */
265 rc = WaitLatch(MyLatch,
266 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
267 1000L, WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE);
268
269 if (rc & WL_LATCH_SET)
270 ResetLatch(MyLatch);
271 }
272
273 return false;
274 }
275
276 /*
277 * Callback from syscache invalidation.
278 */
279 void
280 invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
281 {
282 table_states_validity = SYNC_TABLE_STATE_NEEDS_REBUILD;
283 }
284
285 /*
286 * Handle table synchronization cooperation from the synchronization
287 * worker.
288 *
289 * If the sync worker is in CATCHUP state and reached (or passed) the
290 * predetermined synchronization point in the WAL stream, mark the table as
291 * SYNCDONE and finish.
292 */
293 static void
294 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
295 {
296 SpinLockAcquire(&MyLogicalRepWorker->relmutex);
297
298 if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
299 current_lsn >= MyLogicalRepWorker->relstate_lsn)
300 {
301 TimeLineID tli;
302 char syncslotname[NAMEDATALEN] = {0};
303 char originname[NAMEDATALEN] = {0};
304
305 MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
306 MyLogicalRepWorker->relstate_lsn = current_lsn;
307
308 SpinLockRelease(&MyLogicalRepWorker->relmutex);
309
310 /*
311 * UpdateSubscriptionRelState must be called within a transaction.
312 */
313 if (!IsTransactionState())
314 StartTransactionCommand();
315
316 UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
317 MyLogicalRepWorker->relid,
318 MyLogicalRepWorker->relstate,
319 MyLogicalRepWorker->relstate_lsn,
320 false);
321
322 /*
323 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
324 * the slot.
325 */
326 walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
327
328 /*
329 * Cleanup the tablesync slot.
330 *
331 * This has to be done after updating the state because otherwise if
332 * there is an error while doing the database operations we won't be
333 * able to rollback dropped slot.
334 */
335 ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
336 MyLogicalRepWorker->relid,
337 syncslotname,
338 sizeof(syncslotname));
339
340 /*
341 * It is important to give an error if we are unable to drop the slot,
342 * otherwise, it won't be dropped till the corresponding subscription
343 * is dropped. So passing missing_ok = false.
344 */
345 ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
346
347 CommitTransactionCommand();
348 pgstat_report_stat(false);
349
350 /*
351 * Start a new transaction to clean up the tablesync origin tracking.
352 * This transaction will be ended within the finish_sync_worker().
353 * Now, even, if we fail to remove this here, the apply worker will
354 * ensure to clean it up afterward.
355 *
356 * We need to do this after the table state is set to SYNCDONE.
357 * Otherwise, if an error occurs while performing the database
358 * operation, the worker will be restarted and the in-memory state of
359 * replication progress (remote_lsn) won't be rolled-back which would
360 * have been cleared before restart. So, the restarted worker will use
361 * invalid replication progress state resulting in replay of
362 * transactions that have already been applied.
363 */
364 StartTransactionCommand();
365
366 ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
367 MyLogicalRepWorker->relid,
368 originname,
369 sizeof(originname));
370
371 /*
372 * Resetting the origin session removes the ownership of the slot.
373 * This is needed to allow the origin to be dropped.
374 */
375 replorigin_session_reset();
376 replorigin_session_origin = InvalidRepOriginId;
377 replorigin_session_origin_lsn = InvalidXLogRecPtr;
378 replorigin_session_origin_timestamp = 0;
379
380 /*
381 * Drop the tablesync's origin tracking if exists.
382 *
383 * There is a chance that the user is concurrently performing refresh
384 * for the subscription where we remove the table state and its origin
385 * or the apply worker would have removed this origin. So passing
386 * missing_ok = true.
387 */
388 replorigin_drop_by_name(originname, true, false);
389
390 finish_sync_worker();
391 }
392 else
393 SpinLockRelease(&MyLogicalRepWorker->relmutex);
394 }
395
396 /*
397 * Handle table synchronization cooperation from the apply worker.
398 *
399 * Walk over all subscription tables that are individually tracked by the
400 * apply process (currently, all that have state other than
401 * SUBREL_STATE_READY) and manage synchronization for them.
402 *
403 * If there are tables that need synchronizing and are not being synchronized
404 * yet, start sync workers for them (if there are free slots for sync
405 * workers). To prevent starting the sync worker for the same relation at a
406 * high frequency after a failure, we store its last start time with each sync
407 * state info. We start the sync worker for the same relation after waiting
408 * at least wal_retrieve_retry_interval.
409 *
410 * For tables that are being synchronized already, check if sync workers
411 * either need action from the apply worker or have finished. This is the
412 * SYNCWAIT to CATCHUP transition.
413 *
414 * If the synchronization position is reached (SYNCDONE), then the table can
415 * be marked as READY and is no longer tracked.
416 */
417 static void
418 process_syncing_tables_for_apply(XLogRecPtr current_lsn)
419 {
420 struct tablesync_start_time_mapping
421 {
422 Oid relid;
423 TimestampTz last_start_time;
424 };
425 static HTAB *last_start_times = NULL;
426 ListCell *lc;
427 bool started_tx = false;
428 bool should_exit = false;
429 Relation rel = NULL;
430
431 Assert(!IsTransactionState());
432
433 /* We need up-to-date sync state info for subscription tables here. */
434 FetchTableStates(&started_tx);
435
436 /*
437 * Prepare a hash table for tracking last start times of workers, to avoid
438 * immediate restarts. We don't need it if there are no tables that need
439 * syncing.
440 */
441 if (table_states_not_ready != NIL && !last_start_times)
442 {
443 HASHCTL ctl;
444
445 ctl.keysize = sizeof(Oid);
446 ctl.entrysize = sizeof(struct tablesync_start_time_mapping);
447 last_start_times = hash_create("Logical replication table sync worker start times",
448 256, &ctl, HASH_ELEM | HASH_BLOBS);
449 }
450
451 /*
452 * Clean up the hash table when we're done with all tables (just to
453 * release the bit of memory).
454 */
455 else if (table_states_not_ready == NIL && last_start_times)
456 {
457 hash_destroy(last_start_times);
458 last_start_times = NULL;
459 }
460
461 /*
462 * Process all tables that are being synchronized.
463 */
464 foreach(lc, table_states_not_ready)
465 {
466 SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
467
468 if (rstate->state == SUBREL_STATE_SYNCDONE)
469 {
470 /*
471 * Apply has caught up to the position where the table sync has
472 * finished. Mark the table as ready so that the apply will just
473 * continue to replicate it normally.
474 */
475 if (current_lsn >= rstate->lsn)
476 {
477 char originname[NAMEDATALEN];
478
479 rstate->state = SUBREL_STATE_READY;
480 rstate->lsn = current_lsn;
481 if (!started_tx)
482 {
483 StartTransactionCommand();
484 started_tx = true;
485 }
486
487 /*
488 * Remove the tablesync origin tracking if exists.
489 *
490 * There is a chance that the user is concurrently performing
491 * refresh for the subscription where we remove the table
492 * state and its origin or the tablesync worker would have
493 * already removed this origin. We can't rely on tablesync
494 * worker to remove the origin tracking as if there is any
495 * error while dropping we won't restart it to drop the
496 * origin. So passing missing_ok = true.
497 *
498 * Lock the subscription and origin in the same order as we
499 * are doing during DDL commands to avoid deadlocks. See
500 * AlterSubscription_refresh.
501 */
502 LockSharedObject(SubscriptionRelationId, MyLogicalRepWorker->subid,
503 0, AccessShareLock);
504
505 if (!rel)
506 rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
507
508 ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
509 rstate->relid,
510 originname,
511 sizeof(originname));
512 replorigin_drop_by_name(originname, true, false);
513
514 /*
515 * Update the state to READY only after the origin cleanup.
516 */
517 UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
518 rstate->relid, rstate->state,
519 rstate->lsn, true);
520 }
521 }
522 else
523 {
524 LogicalRepWorker *syncworker;
525
526 /*
527 * Look for a sync worker for this relation.
528 */
529 LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
530
531 syncworker = logicalrep_worker_find(MyLogicalRepWorker->subid,
532 rstate->relid, false);
533
534 if (syncworker)
535 {
536 /* Found one, update our copy of its state */
537 SpinLockAcquire(&syncworker->relmutex);
538 rstate->state = syncworker->relstate;
539 rstate->lsn = syncworker->relstate_lsn;
540 if (rstate->state == SUBREL_STATE_SYNCWAIT)
541 {
542 /*
543 * Sync worker is waiting for apply. Tell sync worker it
544 * can catchup now.
545 */
546 syncworker->relstate = SUBREL_STATE_CATCHUP;
547 syncworker->relstate_lsn =
548 Max(syncworker->relstate_lsn, current_lsn);
549 }
550 SpinLockRelease(&syncworker->relmutex);
551
552 /* If we told worker to catch up, wait for it. */
553 if (rstate->state == SUBREL_STATE_SYNCWAIT)
554 {
555 /* Signal the sync worker, as it may be waiting for us. */
556 if (syncworker->proc)
557 logicalrep_worker_wakeup_ptr(syncworker);
558
559 /* Now safe to release the LWLock */
560 LWLockRelease(LogicalRepWorkerLock);
561
562 if (started_tx)
563 {
564 /*
565 * We must commit the existing transaction to release
566 * the existing locks before entering a busy loop.
567 * This is required to avoid any undetected deadlocks
568 * due to any existing lock as deadlock detector won't
569 * be able to detect the waits on the latch.
570 *
571 * Also close any tables prior to the commit.
572 */
573 if (rel)
574 {
575 table_close(rel, NoLock);
576 rel = NULL;
577 }
578 CommitTransactionCommand();
579 pgstat_report_stat(false);
580 }
581
582 /*
583 * Enter busy loop and wait for synchronization worker to
584 * reach expected state (or die trying).
585 */
586 StartTransactionCommand();
587 started_tx = true;
588
589 wait_for_relation_state_change(rstate->relid,
590 SUBREL_STATE_SYNCDONE);
591 }
592 else
593 LWLockRelease(LogicalRepWorkerLock);
594 }
595 else
596 {
597 /*
598 * If there is no sync worker for this table yet, count
599 * running sync workers for this subscription, while we have
600 * the lock.
601 */
602 int nsyncworkers =
603 logicalrep_sync_worker_count(MyLogicalRepWorker->subid);
604
605 /* Now safe to release the LWLock */
606 LWLockRelease(LogicalRepWorkerLock);
607
608 /*
609 * If there are free sync worker slot(s), start a new sync
610 * worker for the table.
611 */
612 if (nsyncworkers < max_sync_workers_per_subscription)
613 {
614 TimestampTz now = GetCurrentTimestamp();
615 struct tablesync_start_time_mapping *hentry;
616 bool found;
617
618 hentry = hash_search(last_start_times, &rstate->relid,
619 HASH_ENTER, &found);
620
621 if (!found ||
622 TimestampDifferenceExceeds(hentry->last_start_time, now,
623 wal_retrieve_retry_interval))
624 {
625 /*
626 * Set the last_start_time even if we fail to start
627 * the worker, so that we won't retry until
628 * wal_retrieve_retry_interval has elapsed.
629 */
630 hentry->last_start_time = now;
631 (void) logicalrep_worker_launch(WORKERTYPE_TABLESYNC,
632 MyLogicalRepWorker->dbid,
633 MySubscription->oid,
634 MySubscription->name,
635 MyLogicalRepWorker->userid,
636 rstate->relid,
637 DSM_HANDLE_INVALID);
638 }
639 }
640 }
641 }
642 }
643
644 /* Close table if opened */
645 if (rel)
646 table_close(rel, NoLock);
647
648
649 if (started_tx)
650 {
651 /*
652 * Even when the two_phase mode is requested by the user, it remains
653 * as 'pending' until all tablesyncs have reached READY state.
654 *
655 * When this happens, we restart the apply worker and (if the
656 * conditions are still ok) then the two_phase tri-state will become
657 * 'enabled' at that time.
658 *
659 * Note: If the subscription has no tables then leave the state as
660 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
661 * work.
662 */
663 if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING)
664 {
665 CommandCounterIncrement(); /* make updates visible */
666 if (AllTablesyncsReady())
667 {
668 ereport(LOG,
669 (errmsg("logical replication apply worker for subscription \"%s\" will restart so that two_phase can be enabled",
670 MySubscription->name)));
671 should_exit = true;
672 }
673 }
674
675 CommitTransactionCommand();
676 pgstat_report_stat(true);
677 }
678
679 if (should_exit)
680 {
681 /*
682 * Reset the last-start time for this worker so that the launcher will
683 * restart it without waiting for wal_retrieve_retry_interval.
684 */
685 ApplyLauncherForgetWorkerStartTime(MySubscription->oid);
686
687 proc_exit(0);
688 }
689 }
690
691 /*
692 * Process possible state change(s) of tables that are being synchronized.
693 */
694 void
695 process_syncing_tables(XLogRecPtr current_lsn)
696 {
697 switch (MyLogicalRepWorker->type)
698 {
699 case WORKERTYPE_PARALLEL_APPLY:
700
701 /*
702 * Skip for parallel apply workers because they only operate on
703 * tables that are in a READY state. See pa_can_start() and
704 * should_apply_changes_for_rel().
705 */
706 break;
707
708 case WORKERTYPE_TABLESYNC:
709 process_syncing_tables_for_sync(current_lsn);
710 break;
711
712 case WORKERTYPE_APPLY:
713 process_syncing_tables_for_apply(current_lsn);
714 break;
715
716 case WORKERTYPE_UNKNOWN:
717 /* Should never happen. */
718 elog(ERROR, "Unknown worker type");
719 }
720 }
721
722 /*
723 * Create list of columns for COPY based on logical relation mapping.
724 */
725 static List *
726 make_copy_attnamelist(LogicalRepRelMapEntry *rel)
727 {
728 List *attnamelist = NIL;
729 int i;
730
731 for (i = 0; i < rel->remoterel.natts; i++)
732 {
733 attnamelist = lappend(attnamelist,
734 makeString(rel->remoterel.attnames[i]));
735 }
736
737
738 return attnamelist;
739 }
740
741 /*
742 * Data source callback for the COPY FROM, which reads from the remote
743 * connection and passes the data back to our local COPY.
744 */
745 static int
746 copy_read_data(void *outbuf, int minread, int maxread)
747 {
748 int bytesread = 0;
749 int avail;
750
751 /* If there are some leftover data from previous read, use it. */
752 avail = copybuf->len - copybuf->cursor;
753 if (avail)
754 {
755 if (avail > maxread)
756 avail = maxread;
757 memcpy(outbuf, &copybuf->data[copybuf->cursor], avail);
758 copybuf->cursor += avail;
759 maxread -= avail;
760 bytesread += avail;
761 }
762
763 while (maxread > 0 && bytesread < minread)
764 {
765 pgsocket fd = PGINVALID_SOCKET;
766 int len;
767 char *buf = NULL;
768
769 for (;;)
770 {
771 /* Try read the data. */
772 len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
773
774 CHECK_FOR_INTERRUPTS();
775
776 if (len == 0)
777 break;
778 else if (len < 0)
779 return bytesread;
780 else
781 {
782 /* Process the data */
783 copybuf->data = buf;
784 copybuf->len = len;
785 copybuf->cursor = 0;
786
787 avail = copybuf->len - copybuf->cursor;
788 if (avail > maxread)
789 avail = maxread;
790 memcpy(outbuf, &copybuf->data[copybuf->cursor], avail);
791 outbuf = (char *) outbuf + avail;
792 copybuf->cursor += avail;
793 maxread -= avail;
794 bytesread += avail;
795 }
796
797 if (maxread <= 0 || bytesread >= minread)
798 return bytesread;
799 }
800
801 /*
802 * Wait for more data or latch.
803 */
804 (void) WaitLatchOrSocket(MyLatch,
805 WL_SOCKET_READABLE | WL_LATCH_SET |
806 WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
807 fd, 1000L, WAIT_EVENT_LOGICAL_SYNC_DATA);
808
809 ResetLatch(MyLatch);
810 }
811
812 return bytesread;
813 }
814
815
816 /*
817 * Get information about remote relation in similar fashion the RELATION
818 * message provides during replication.
819 *
820 * This function also returns (a) the relation qualifications to be used in
821 * the COPY command, and (b) whether the remote relation has published any
822 * generated column.
823 */
824 static void
825 fetch_remote_table_info(char *nspname, char *relname, LogicalRepRelation *lrel,
826 List **qual, bool *gencol_published)
827 {
828 WalRcvExecResult *res;
829 StringInfoData cmd;
830 TupleTableSlot *slot;
831 Oid tableRow[] = {OIDOID, CHAROID, CHAROID};
832 Oid attrRow[] = {INT2OID, TEXTOID, OIDOID, BOOLOID, BOOLOID};
833 Oid qualRow[] = {TEXTOID};
834 bool isnull;
835 int natt;
836 StringInfo pub_names = NULL;
837 Bitmapset *included_cols = NULL;
838 int server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
839
840 lrel->nspname = nspname;
841 lrel->relname = relname;
842
843 /* First fetch Oid and replica identity. */
844 initStringInfo(&cmd);
845 appendStringInfo(&cmd, "SELECT c.oid, c.relreplident, c.relkind"
846 " FROM pg_catalog.pg_class c"
847 " INNER JOIN pg_catalog.pg_namespace n"
848 " ON (c.relnamespace = n.oid)"
849 " WHERE n.nspname = %s"
850 " AND c.relname = %s",
851 quote_literal_cstr(nspname),
852 quote_literal_cstr(relname));
853 res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data,
854 lengthof(tableRow), tableRow);
855
856 if (res->status != WALRCV_OK_TUPLES)
857 ereport(ERROR,
858 (errcode(ERRCODE_CONNECTION_FAILURE),
859 errmsg("could not fetch table info for table \"%s.%s\" from publisher: %s",
860 nspname, relname, res->err)));
861
862 slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
863 if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot))
864 ereport(ERROR,
865 (errcode(ERRCODE_UNDEFINED_OBJECT),
866 errmsg("table \"%s.%s\" not found on publisher",
867 nspname, relname)));
868
869 lrel->remoteid = DatumGetObjectId(slot_getattr(slot, 1, &isnull));
870 Assert(!isnull);
871 lrel->replident = DatumGetChar(slot_getattr(slot, 2, &isnull));
872 Assert(!isnull);
873 lrel->relkind = DatumGetChar(slot_getattr(slot, 3, &isnull));
874 Assert(!isnull);
875
876 ExecDropSingleTupleTableSlot(slot);
877 walrcv_clear_result(res);
878
879
880 /*
881 * Get column lists for each relation.
882 *
883 * We need to do this before fetching info about column names and types,
884 * so that we can skip columns that should not be replicated.
885 */
886 if (server_version >= 150000)
887 {
888 WalRcvExecResult *pubres;
889 TupleTableSlot *tslot;
890 Oid attrsRow[] = {INT2VECTOROID};
891
892 /* Build the pub_names comma-separated string. */
893 pub_names = makeStringInfo();
894 GetPublicationsStr(MySubscription->publications, pub_names, true);
895
896 /*
897 * Fetch info about column lists for the relation (from all the
898 * publications).
899 */
900 resetStringInfo(&cmd);
901 appendStringInfo(&cmd,
902 "SELECT DISTINCT"
903 " (CASE WHEN (array_length(gpt.attrs, 1) = c.relnatts)"
904 " THEN NULL ELSE gpt.attrs END)"
905 " FROM pg_publication p,"
906 " LATERAL pg_get_publication_tables(p.pubname) gpt,"
907 " pg_class c"
908 " WHERE gpt.relid = %u AND c.oid = gpt.relid"
909 " AND p.pubname IN ( %s )",
910 lrel->remoteid,
911 pub_names->data);
912
913 pubres = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data,
914 lengthof(attrsRow), attrsRow);
915
916 if (pubres->status != WALRCV_OK_TUPLES)
917 ereport(ERROR,
918 (errcode(ERRCODE_CONNECTION_FAILURE),
919 errmsg("could not fetch column list info for table \"%s.%s\" from publisher: %s",
920 nspname, relname, pubres->err)));
921
922 /*
923 * We don't support the case where the column list is different for
924 * the same table when combining publications. See comments atop
925 * fetch_table_list. So there should be only one row returned.
926 * Although we already checked this when creating the subscription, we
927 * still need to check here in case the column list was changed after
928 * creating the subscription and before the sync worker is started.
929 */
930 if (tuplestore_tuple_count(pubres->tuplestore) > 1)
931 ereport(ERROR,
932 errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
933 errmsg("cannot use different column lists for table \"%s.%s\" in different publications",
934 nspname, relname));
935
936 /*
937 * Get the column list and build a single bitmap with the attnums.
938 *
939 * If we find a NULL value, it means all the columns should be
940 * replicated.
941 */
942 tslot = MakeSingleTupleTableSlot(pubres->tupledesc, &TTSOpsMinimalTuple);
943 if (tuplestore_gettupleslot(pubres->tuplestore, true, false, tslot))
944 {
945 Datum cfval = slot_getattr(tslot, 1, &isnull);
946
947 if (!isnull)
948 {
949 ArrayType *arr;
950 int nelems;
951 int16 *elems;
952
953 arr = DatumGetArrayTypeP(cfval);
954 nelems = ARR_DIMS(arr)[0];
955 elems = (int16 *) ARR_DATA_PTR(arr);
956
957 for (natt = 0; natt < nelems; natt++)
958 included_cols = bms_add_member(included_cols, elems[natt]);
959 }
960
961 ExecClearTuple(tslot);
962 }
963 ExecDropSingleTupleTableSlot(tslot);
964
965 walrcv_clear_result(pubres);
966 }
967
968 /*
969 * Now fetch column names and types.
970 */
971 resetStringInfo(&cmd);
972 appendStringInfoString(&cmd,
973 "SELECT a.attnum,"
974 " a.attname,"
975 " a.atttypid,"
976 " a.attnum = ANY(i.indkey)");
977
978 /* Generated columns can be replicated since version 18. */
979 if (server_version >= 180000)
980 appendStringInfoString(&cmd, ", a.attgenerated != ''");
981
982 appendStringInfo(&cmd,
983 " FROM pg_catalog.pg_attribute a"
984 " LEFT JOIN pg_catalog.pg_index i"
985 " ON (i.indexrelid = pg_get_replica_identity_index(%u))"
986 " WHERE a.attnum > 0::pg_catalog.int2"
987 " AND NOT a.attisdropped %s"
988 " AND a.attrelid = %u"
989 " ORDER BY a.attnum",
990 lrel->remoteid,
991 (server_version >= 120000 && server_version < 180000 ?
992 "AND a.attgenerated = ''" : ""),
993 lrel->remoteid);
994 res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data,
995 server_version >= 180000 ? lengthof(attrRow) : lengthof(attrRow) - 1, attrRow);
996
997 if (res->status != WALRCV_OK_TUPLES)
998 ereport(ERROR,
999 (errcode(ERRCODE_CONNECTION_FAILURE),
1000 errmsg("could not fetch table info for table \"%s.%s\" from publisher: %s",
1001 nspname, relname, res->err)));
1002
1003 /* We don't know the number of rows coming, so allocate enough space. */
1004 lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *));
1005 lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid));
1006 lrel->attkeys = NULL;
1007
1008 /*
1009 * Store the columns as a list of names. Ignore those that are not
1010 * present in the column list, if there is one.
1011 */
1012 natt = 0;
1013 slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
1014 while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
1015 {
1016 char *rel_colname;
1017 AttrNumber attnum;
1018
1019 attnum = DatumGetInt16(slot_getattr(slot, 1, &isnull));
1020 Assert(!isnull);
1021
1022 /* If the column is not in the column list, skip it. */
1023 if (included_cols != NULL && !bms_is_member(attnum, included_cols))
1024 {
1025 ExecClearTuple(slot);
1026 continue;
1027 }
1028
1029 rel_colname = TextDatumGetCString(slot_getattr(slot, 2, &isnull));
1030 Assert(!isnull);
1031
1032 lrel->attnames[natt] = rel_colname;
1033 lrel->atttyps[natt] = DatumGetObjectId(slot_getattr(slot, 3, &isnull));
1034 Assert(!isnull);
1035
1036 if (DatumGetBool(slot_getattr(slot, 4, &isnull)))
1037 lrel->attkeys = bms_add_member(lrel->attkeys, natt);
1038
1039 /* Remember if the remote table has published any generated column. */
1040 if (server_version >= 180000 && !(*gencol_published))
1041 {
1042 *gencol_published = DatumGetBool(slot_getattr(slot, 5, &isnull));
1043 Assert(!isnull);
1044 }
1045
1046 /* Should never happen. */
1047 if (++natt >= MaxTupleAttributeNumber)
1048 elog(ERROR, "too many columns in remote table \"%s.%s\"",
1049 nspname, relname);
1050
1051 ExecClearTuple(slot);
1052 }
1053 ExecDropSingleTupleTableSlot(slot);
1054
1055 lrel->natts = natt;
1056
1057 walrcv_clear_result(res);
1058
1059 /*
1060 * Get relation's row filter expressions. DISTINCT avoids the same
1061 * expression of a table in multiple publications from being included
1062 * multiple times in the final expression.
1063 *
1064 * We need to copy the row even if it matches just one of the
1065 * publications, so we later combine all the quals with OR.
1066 *
1067 * For initial synchronization, row filtering can be ignored in following
1068 * cases:
1069 *
1070 * 1) one of the subscribed publications for the table hasn't specified
1071 * any row filter
1072 *
1073 * 2) one of the subscribed publications has puballtables set to true
1074 *
1075 * 3) one of the subscribed publications is declared as TABLES IN SCHEMA
1076 * that includes this relation
1077 */
1078 if (server_version >= 150000)
1079 {
1080 /* Reuse the already-built pub_names. */
1081 Assert(pub_names != NULL);
1082
1083 /* Check for row filters. */
1084 resetStringInfo(&cmd);
1085 appendStringInfo(&cmd,
1086 "SELECT DISTINCT pg_get_expr(gpt.qual, gpt.relid)"
1087 " FROM pg_publication p,"
1088 " LATERAL pg_get_publication_tables(p.pubname) gpt"
1089 " WHERE gpt.relid = %u"
1090 " AND p.pubname IN ( %s )",
1091 lrel->remoteid,
1092 pub_names->data);
1093
1094 res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 1, qualRow);
1095
1096 if (res->status != WALRCV_OK_TUPLES)
1097 ereport(ERROR,
1098 (errmsg("could not fetch table WHERE clause info for table \"%s.%s\" from publisher: %s",
1099 nspname, relname, res->err)));
1100
1101 /*
1102 * Multiple row filter expressions for the same table will be combined
1103 * by COPY using OR. If any of the filter expressions for this table
1104 * are null, it means the whole table will be copied. In this case it
1105 * is not necessary to construct a unified row filter expression at
1106 * all.
1107 */
1108 slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
1109 while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
1110 {
1111 Datum rf = slot_getattr(slot, 1, &isnull);
1112
1113 if (!isnull)
1114 *qual = lappend(*qual, makeString(TextDatumGetCString(rf)));
1115 else
1116 {
1117 /* Ignore filters and cleanup as necessary. */
1118 if (*qual)
1119 {
1120 list_free_deep(*qual);
1121 *qual = NIL;
1122 }
1123 break;
1124 }
1125
1126 ExecClearTuple(slot);
1127 }
1128 ExecDropSingleTupleTableSlot(slot);
1129
1130 walrcv_clear_result(res);
1131 destroyStringInfo(pub_names);
1132 }
1133
1134 pfree(cmd.data);
1135 }
1136
1137 /*
1138 * Copy existing data of a table from publisher.
1139 *
1140 * Caller is responsible for locking the local relation.
1141 */
1142 static void
1143 copy_table(Relation rel)
1144 {
1145 LogicalRepRelMapEntry *relmapentry;
1146 LogicalRepRelation lrel;
1147 List *qual = NIL;
1148 WalRcvExecResult *res;
1149 StringInfoData cmd;
1150 CopyFromState cstate;
1151 List *attnamelist;
1152 ParseState *pstate;
1153 List *options = NIL;
1154 bool gencol_published = false;
1155
1156 /* Get the publisher relation info. */
1157 fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)),
1158 RelationGetRelationName(rel), &lrel, &qual,
1159 &gencol_published);
1160
1161 /* Put the relation into relmap. */
1162 logicalrep_relmap_update(&lrel);
1163
1164 /* Map the publisher relation to local one. */
1165 relmapentry = logicalrep_rel_open(lrel.remoteid, NoLock);
1166 Assert(rel == relmapentry->localrel);
1167
1168 /* Start copy on the publisher. */
1169 initStringInfo(&cmd);
1170
1171 /* Regular table with no row filter or generated columns */
1172 if (lrel.relkind == RELKIND_RELATION && qual == NIL && !gencol_published)
1173 {
1174 appendStringInfo(&cmd, "COPY %s",
1175 quote_qualified_identifier(lrel.nspname, lrel.relname));
1176
1177 /* If the table has columns, then specify the columns */
1178 if (lrel.natts)
1179 {
1180 appendStringInfoString(&cmd, " (");
1181
1182 /*
1183 * XXX Do we need to list the columns in all cases? Maybe we're
1184 * replicating all columns?
1185 */
1186 for (int i = 0; i < lrel.natts; i++)
1187 {
1188 if (i > 0)
1189 appendStringInfoString(&cmd, ", ");
1190
1191 appendStringInfoString(&cmd, quote_identifier(lrel.attnames[i]));
1192 }
1193
1194 appendStringInfoChar(&cmd, ')');
1195 }
1196
1197 appendStringInfoString(&cmd, " TO STDOUT");
1198 }
1199 else
1200 {
1201 /*
1202 * For non-tables and tables with row filters, we need to do COPY
1203 * (SELECT ...), but we can't just do SELECT * because we may need to
1204 * copy only subset of columns including generated columns. For tables
1205 * with any row filters, build a SELECT query with OR'ed row filters
1206 * for COPY.
1207 *
1208 * We also need to use this same COPY (SELECT ...) syntax when
1209 * generated columns are published, because copy of generated columns
1210 * is not supported by the normal COPY.
1211 */
1212 appendStringInfoString(&cmd, "COPY (SELECT ");
1213 for (int i = 0; i < lrel.natts; i++)
1214 {
1215 appendStringInfoString(&cmd, quote_identifier(lrel.attnames[i]));
1216 if (i < lrel.natts - 1)
1217 appendStringInfoString(&cmd, ", ");
1218 }
1219
1220 appendStringInfoString(&cmd, " FROM ");
1221
1222 /*
1223 * For regular tables, make sure we don't copy data from a child that
1224 * inherits the named table as those will be copied separately.
1225 */
1226 if (lrel.relkind == RELKIND_RELATION)
1227 appendStringInfoString(&cmd, "ONLY ");
1228
1229 appendStringInfoString(&cmd, quote_qualified_identifier(lrel.nspname, lrel.relname));
1230 /* list of OR'ed filters */
1231 if (qual != NIL)
1232 {
1233 ListCell *lc;
1234 char *q = strVal(linitial(qual));
1235
1236 appendStringInfo(&cmd, " WHERE %s", q);
1237 for_each_from(lc, qual, 1)
1238 {
1239 q = strVal(lfirst(lc));
1240 appendStringInfo(&cmd, " OR %s", q);
1241 }
1242 list_free_deep(qual);
1243 }
1244
1245 appendStringInfoString(&cmd, ") TO STDOUT");
1246 }
1247
1248 /*
1249 * Prior to v16, initial table synchronization will use text format even
1250 * if the binary option is enabled for a subscription.
1251 */
1252 if (walrcv_server_version(LogRepWorkerWalRcvConn) >= 160000 &&
1253 MySubscription->binary)
1254 {
1255 appendStringInfoString(&cmd, " WITH (FORMAT binary)");
1256 options = list_make1(makeDefElem("format",
1257 (Node *) makeString("binary"), -1));
1258 }
1259
1260 res = walrcv_exec(LogRepWorkerWalRcvConn, cmd.data, 0, NULL);
1261 pfree(cmd.data);
1262 if (res->status != WALRCV_OK_COPY_OUT)
1263 ereport(ERROR,
1264 (errcode(ERRCODE_CONNECTION_FAILURE),
1265 errmsg("could not start initial contents copy for table \"%s.%s\": %s",
1266 lrel.nspname, lrel.relname, res->err)));
1267 walrcv_clear_result(res);
1268
1269 copybuf = makeStringInfo();
1270
1271 pstate = make_parsestate(NULL);
1272 (void) addRangeTableEntryForRelation(pstate, rel, AccessShareLock,
1273 NULL, false, false);
1274
1275 attnamelist = make_copy_attnamelist(relmapentry);
1276 cstate = BeginCopyFrom(pstate, rel, NULL, NULL, false, copy_read_data, attnamelist, options);
1277
1278 /* Do the copy */
1279 (void) CopyFrom(cstate);
1280 EndCopyFrom(cstate);
1281
1282 logicalrep_rel_close(relmapentry, NoLock);
1283 }
1284
1285 /*
1286 * Determine the tablesync slot name.
1287 *
1288 * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
1289 * on slot name length. We append system_identifier to avoid slot_name
1290 * collision with subscriptions in other clusters. With the current scheme
1291 * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
1292 * length of slot_name will be 50.
1293 *
1294 * The returned slot name is stored in the supplied buffer (syncslotname) with
1295 * the given size.
1296 *
1297 * Note: We don't use the subscription slot name as part of tablesync slot name
1298 * because we are responsible for cleaning up these slots and it could become
1299 * impossible to recalculate what name to cleanup if the subscription slot name
1300 * had changed.
1301 */
1302 void
1303 ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
1304 char *syncslotname, Size szslot)
1305 {
1306 snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
1307 relid, GetSystemIdentifier());
1308 }
1309
1310 /*
1311 * Start syncing the table in the sync worker.
1312 *
1313 * If nothing needs to be done to sync the table, we exit the worker without
1314 * any further action.
1315 *
1316 * The returned slot name is palloc'ed in current memory context.
1317 */
1318 static char *
1319 LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
1320 {
1321 char *slotname;
1322 char *err;
1323 char relstate;
1324 XLogRecPtr relstate_lsn;
1325 Relation rel;
1326 AclResult aclresult;
1327 WalRcvExecResult *res;
1328 char originname[NAMEDATALEN];
1329 RepOriginId originid;
1330 UserContext ucxt;
1331 bool must_use_password;
1332 bool run_as_owner;
1333
1334 /* Check the state of the table synchronization. */
1335 StartTransactionCommand();
1336 relstate = GetSubscriptionRelState(MyLogicalRepWorker->subid,
1337 MyLogicalRepWorker->relid,
1338 &relstate_lsn);
1339 CommitTransactionCommand();
1340
1341 /* Is the use of a password mandatory? */
1342 must_use_password = MySubscription->passwordrequired &&
1343 !MySubscription->ownersuperuser;
1344
1345 SpinLockAcquire(&MyLogicalRepWorker->relmutex);
1346 MyLogicalRepWorker->relstate = relstate;
1347 MyLogicalRepWorker->relstate_lsn = relstate_lsn;
1348 SpinLockRelease(&MyLogicalRepWorker->relmutex);
1349
1350 /*
1351 * If synchronization is already done or no longer necessary, exit now
1352 * that we've updated shared memory state.
1353 */
1354 switch (relstate)
1355 {
1356 case SUBREL_STATE_SYNCDONE:
1357 case SUBREL_STATE_READY:
1358 case SUBREL_STATE_UNKNOWN:
1359 finish_sync_worker(); /* doesn't return */
1360 }
1361
1362 /* Calculate the name of the tablesync slot. */
1363 slotname = (char *) palloc(NAMEDATALEN);
1364 ReplicationSlotNameForTablesync(MySubscription->oid,
1365 MyLogicalRepWorker->relid,
1366 slotname,
1367 NAMEDATALEN);
1368
1369 /*
1370 * Here we use the slot name instead of the subscription name as the
1371 * application_name, so that it is different from the leader apply worker,
1372 * so that synchronous replication can distinguish them.
1373 */
1374 LogRepWorkerWalRcvConn =
1375 walrcv_connect(MySubscription->conninfo, true, true,
1376 must_use_password,
1377 slotname, &err);
1378 if (LogRepWorkerWalRcvConn == NULL)
1379 ereport(ERROR,
1380 (errcode(ERRCODE_CONNECTION_FAILURE),
1381 errmsg("table synchronization worker for subscription \"%s\" could not connect to the publisher: %s",
1382 MySubscription->name, err)));
1383
1384 Assert(MyLogicalRepWorker->relstate == SUBREL_STATE_INIT ||
1385 MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
1386 MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
1387
1388 /* Assign the origin tracking record name. */
1389 ReplicationOriginNameForLogicalRep(MySubscription->oid,
1390 MyLogicalRepWorker->relid,
1391 originname,
1392 sizeof(originname));
1393
1394 if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
1395 {
1396 /*
1397 * We have previously errored out before finishing the copy so the
1398 * replication slot might exist. We want to remove the slot if it
1399 * already exists and proceed.
1400 *
1401 * XXX We could also instead try to drop the slot, last time we failed
1402 * but for that, we might need to clean up the copy state as it might
1403 * be in the middle of fetching the rows. Also, if there is a network
1404 * breakdown then it wouldn't have succeeded so trying it next time
1405 * seems like a better bet.
1406 */
1407 ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
1408 }
1409 else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
1410 {
1411 /*
1412 * The COPY phase was previously done, but tablesync then crashed
1413 * before it was able to finish normally.
1414 */
1415 StartTransactionCommand();
1416
1417 /*
1418 * The origin tracking name must already exist. It was created first
1419 * time this tablesync was launched.
1420 */
1421 originid = replorigin_by_name(originname, false);
1422 replorigin_session_setup(originid, 0);
1423 replorigin_session_origin = originid;
1424 *origin_startpos = replorigin_session_get_progress(false);
1425
1426 CommitTransactionCommand();
1427
1428 goto copy_table_done;
1429 }
1430
1431 SpinLockAcquire(&MyLogicalRepWorker->relmutex);
1432 MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
1433 MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
1434 SpinLockRelease(&MyLogicalRepWorker->relmutex);
1435
1436 /*
1437 * Update the state, create the replication origin, and make them visible
1438 * to others.
1439 */
1440 StartTransactionCommand();
1441 UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
1442 MyLogicalRepWorker->relid,
1443 MyLogicalRepWorker->relstate,
1444 MyLogicalRepWorker->relstate_lsn,
1445 false);
1446
1447 /*
1448 * Create the replication origin in a separate transaction from the one
1449 * that sets up the origin in shared memory. This prevents the risk that
1450 * changes to the origin in shared memory cannot be rolled back if the
1451 * transaction aborts.
1452 */
1453 originid = replorigin_by_name(originname, true);
1454 if (!OidIsValid(originid))
1455 originid = replorigin_create(originname);
1456
1457 CommitTransactionCommand();
1458 pgstat_report_stat(true);
1459
1460 StartTransactionCommand();
1461
1462 /*
1463 * Use a standard write lock here. It might be better to disallow access
1464 * to the table while it's being synchronized. But we don't want to block
1465 * the main apply process from working and it has to open the relation in
1466 * RowExclusiveLock when remapping remote relation id to local one.
1467 */
1468 rel = table_open(MyLogicalRepWorker->relid, RowExclusiveLock);
1469
1470 /*
1471 * Start a transaction in the remote node in REPEATABLE READ mode. This
1472 * ensures that both the replication slot we create (see below) and the
1473 * COPY are consistent with each other.
1474 */
1475 res = walrcv_exec(LogRepWorkerWalRcvConn,
1476 "BEGIN READ ONLY ISOLATION LEVEL REPEATABLE READ",
1477 0, NULL);
1478 if (res->status != WALRCV_OK_COMMAND)
1479 ereport(ERROR,
1480 (errcode(ERRCODE_CONNECTION_FAILURE),
1481 errmsg("table copy could not start transaction on publisher: %s",
1482 res->err)));
1483 walrcv_clear_result(res);
1484
1485 /*
1486 * Create a new permanent logical decoding slot. This slot will be used
1487 * for the catchup phase after COPY is done, so tell it to use the
1488 * snapshot to make the final data consistent.
1489 */
1490 walrcv_create_slot(LogRepWorkerWalRcvConn,
1491 slotname, false /* permanent */ , false /* two_phase */ ,
1492 MySubscription->failover,
1493 CRS_USE_SNAPSHOT, origin_startpos);
1494
1495 /*
1496 * Advance the origin to the LSN got from walrcv_create_slot and then set
1497 * up the origin. The advancement is WAL logged for the purpose of
1498 * recovery. Locks are to prevent the replication origin from vanishing
1499 * while advancing.
1500 *
1501 * The purpose of doing these before the copy is to avoid doing the copy
1502 * again due to any error in advancing or setting up origin tracking.
1503 */
1504 LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
1505 replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
1506 true /* go backward */ , true /* WAL log */ );
1507 UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
1508
1509 replorigin_session_setup(originid, 0);
1510 replorigin_session_origin = originid;
1511
1512 /*
1513 * Make sure that the copy command runs as the table owner, unless the
1514 * user has opted out of that behaviour.
1515 */
1516 run_as_owner = MySubscription->runasowner;
1517 if (!run_as_owner)
1518 SwitchToUntrustedUser(rel->rd_rel->relowner, &ucxt);
1519
1520 /*
1521 * Check that our table sync worker has permission to insert into the
1522 * target table.
1523 */
1524 aclresult = pg_class_aclcheck(RelationGetRelid(rel), GetUserId(),
1525 ACL_INSERT);
1526 if (aclresult != ACLCHECK_OK)
1527 aclcheck_error(aclresult,
1528 get_relkind_objtype(rel->rd_rel->relkind),
1529 RelationGetRelationName(rel));
1530
1531 /*
1532 * COPY FROM does not honor RLS policies. That is not a problem for
1533 * subscriptions owned by roles with BYPASSRLS privilege (or superuser,
1534 * who has it implicitly), but other roles should not be able to
1535 * circumvent RLS. Disallow logical replication into RLS enabled
1536 * relations for such roles.
1537 */
1538 if (check_enable_rls(RelationGetRelid(rel), InvalidOid, false) == RLS_ENABLED)
1539 ereport(ERROR,
1540 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1541 errmsg("user \"%s\" cannot replicate into relation with row-level security enabled: \"%s\"",
1542 GetUserNameFromId(GetUserId(), true),
1543 RelationGetRelationName(rel))));
1544
1545 /* Now do the initial data copy */
1546 PushActiveSnapshot(GetTransactionSnapshot());
1547 copy_table(rel);
1548 PopActiveSnapshot();
1549
1550 res = walrcv_exec(LogRepWorkerWalRcvConn, "COMMIT", 0, NULL);
1551 if (res->status != WALRCV_OK_COMMAND)
1552 ereport(ERROR,
1553 (errcode(ERRCODE_CONNECTION_FAILURE),
1554 errmsg("table copy could not finish transaction on publisher: %s",
1555 res->err)));
1556 walrcv_clear_result(res);
1557
1558 if (!run_as_owner)
1559 RestoreUserContext(&ucxt);
1560
1561 table_close(rel, NoLock);
1562
1563 /* Make the copy visible. */
1564 CommandCounterIncrement();
1565
1566 /*
1567 * Update the persisted state to indicate the COPY phase is done; make it
1568 * visible to others.
1569 */
1570 UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
1571 MyLogicalRepWorker->relid,
1572 SUBREL_STATE_FINISHEDCOPY,
1573 MyLogicalRepWorker->relstate_lsn,
1574 false);
1575
1576 CommitTransactionCommand();
1577
1578 copy_table_done:
1579
1580 elog(DEBUG1,
1581 "LogicalRepSyncTableStart: '%s' origin_startpos lsn %X/%X",
1582 originname, LSN_FORMAT_ARGS(*origin_startpos));
1583
1584 /*
1585 * We are done with the initial data synchronization, update the state.
1586 */
1587 SpinLockAcquire(&MyLogicalRepWorker->relmutex);
1588 MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCWAIT;
1589 MyLogicalRepWorker->relstate_lsn = *origin_startpos;
1590 SpinLockRelease(&MyLogicalRepWorker->relmutex);
1591
1592 /*
1593 * Finally, wait until the leader apply worker tells us to catch up and
1594 * then return to let LogicalRepApplyLoop do it.
1595 */
1596 wait_for_worker_state_change(SUBREL_STATE_CATCHUP);
1597 return slotname;
1598 }
1599
1600 /*
1601 * Common code to fetch the up-to-date sync state info into the static lists.
1602 *
1603 * Returns true if subscription has 1 or more tables, else false.
1604 *
1605 * Note: If this function started the transaction (indicated by the parameter)
1606 * then it is the caller's responsibility to commit it.
1607 */
1608 static bool
1609 FetchTableStates(bool *started_tx)
1610 {
1611 static bool has_subrels = false;
1612
1613 *started_tx = false;
1614
1615 if (table_states_validity != SYNC_TABLE_STATE_VALID)
1616 {
1617 MemoryContext oldctx;
1618 List *rstates;
1619 ListCell *lc;
1620 SubscriptionRelState *rstate;
1621
1622 table_states_validity = SYNC_TABLE_STATE_REBUILD_STARTED;
1623
1624 /* Clean the old lists. */
1625 list_free_deep(table_states_not_ready);
1626 table_states_not_ready = NIL;
1627
1628 if (!IsTransactionState())
1629 {
1630 StartTransactionCommand();
1631 *started_tx = true;
1632 }
1633
1634 /* Fetch all non-ready tables. */
1635 rstates = GetSubscriptionRelations(MySubscription->oid, true);
1636
1637 /* Allocate the tracking info in a permanent memory context. */
1638 oldctx = MemoryContextSwitchTo(CacheMemoryContext);
1639 foreach(lc, rstates)
1640 {
1641 rstate = palloc(sizeof(SubscriptionRelState));
1642 memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
1643 table_states_not_ready = lappend(table_states_not_ready, rstate);
1644 }
1645 MemoryContextSwitchTo(oldctx);
1646
1647 /*
1648 * Does the subscription have tables?
1649 *
1650 * If there were not-READY relations found then we know it does. But
1651 * if table_states_not_ready was empty we still need to check again to
1652 * see if there are 0 tables.
1653 */
1654 has_subrels = (table_states_not_ready != NIL) ||
1655 HasSubscriptionRelations(MySubscription->oid);
1656
1657 /*
1658 * If the subscription relation cache has been invalidated since we
1659 * entered this routine, we still use and return the relations we just
1660 * finished constructing, to avoid infinite loops, but we leave the
1661 * table states marked as stale so that we'll rebuild it again on next
1662 * access. Otherwise, we mark the table states as valid.
1663 */
1664 if (table_states_validity == SYNC_TABLE_STATE_REBUILD_STARTED)
1665 table_states_validity = SYNC_TABLE_STATE_VALID;
1666 }
1667
1668 return has_subrels;
1669 }
1670
1671 /*
1672 * Execute the initial sync with error handling. Disable the subscription,
1673 * if it's required.
1674 *
1675 * Allocate the slot name in long-lived context on return. Note that we don't
1676 * handle FATAL errors which are probably because of system resource error and
1677 * are not repeatable.
1678 */
1679 static void
1680 start_table_sync(XLogRecPtr *origin_startpos, char **slotname)
1681 {
1682 char *sync_slotname = NULL;
1683
1684 Assert(am_tablesync_worker());
1685
1686 PG_TRY();
1687 {
1688 /* Call initial sync. */
1689 sync_slotname = LogicalRepSyncTableStart(origin_startpos);
1690 }
1691 PG_CATCH();
1692 {
1693 if (MySubscription->disableonerr)
1694 DisableSubscriptionAndExit();
1695 else
1696 {
1697 /*
1698 * Report the worker failed during table synchronization. Abort
1699 * the current transaction so that the stats message is sent in an
1700 * idle state.
1701 */
1702 AbortOutOfAnyTransaction();
1703 pgstat_report_subscription_error(MySubscription->oid, false);
1704
1705 PG_RE_THROW();
1706 }
1707 }
1708 PG_END_TRY();
1709
1710 /* allocate slot name in long-lived context */
1711 *slotname = MemoryContextStrdup(ApplyContext, sync_slotname);
1712 pfree(sync_slotname);
1713 }
1714
1715 /*
1716 * Runs the tablesync worker.
1717 *
1718 * It starts syncing tables. After a successful sync, sets streaming options
1719 * and starts streaming to catchup with apply worker.
1720 */
1721 static void
1722 run_tablesync_worker()
1723 {
1724 char originname[NAMEDATALEN];
1725 XLogRecPtr origin_startpos = InvalidXLogRecPtr;
1726 char *slotname = NULL;
1727 WalRcvStreamOptions options;
1728
1729 start_table_sync(&origin_startpos, &slotname);
1730
1731 ReplicationOriginNameForLogicalRep(MySubscription->oid,
1732 MyLogicalRepWorker->relid,
1733 originname,
1734 sizeof(originname));
1735
1736 set_apply_error_context_origin(originname);
1737
1738 set_stream_options(&options, slotname, &origin_startpos);
1739
1740 walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
1741
1742 /* Apply the changes till we catchup with the apply worker. */
1743 start_apply(origin_startpos);
1744 }
1745
1746 /* Logical Replication Tablesync worker entry point */
1747 void
1748 TablesyncWorkerMain(Datum main_arg)
1749 {
1750 int worker_slot = DatumGetInt32(main_arg);
1751
1752 SetupApplyOrSyncWorker(worker_slot);
1753
1754 run_tablesync_worker();
1755
1756 finish_sync_worker();
1757 }
1758
1759 /*
1760 * If the subscription has no tables then return false.
1761 *
1762 * Otherwise, are all tablesyncs READY?
1763 *
1764 * Note: This function is not suitable to be called from outside of apply or
1765 * tablesync workers because MySubscription needs to be already initialized.
1766 */
1767 bool
1768 AllTablesyncsReady(void)
1769 {
1770 bool started_tx = false;
1771 bool has_subrels = false;
1772
1773 /* We need up-to-date sync state info for subscription tables here. */
1774 has_subrels = FetchTableStates(&started_tx);
1775
1776 if (started_tx)
1777 {
1778 CommitTransactionCommand();
1779 pgstat_report_stat(true);
1780 }
1781
1782 /*
1783 * Return false when there are no tables in subscription or not all tables
1784 * are in ready state; true otherwise.
1785 */
1786 return has_subrels && (table_states_not_ready == NIL);
1787 }
1788
1789 /*
1790 * Update the two_phase state of the specified subscription in pg_subscription.
1791 */
1792 void
1793 UpdateTwoPhaseState(Oid suboid, char new_state)
1794 {
1795 Relation rel;
1796 HeapTuple tup;
1797 bool nulls[Natts_pg_subscription];
1798 bool replaces[Natts_pg_subscription];
1799 Datum values[Natts_pg_subscription];
1800
1801 Assert(new_state == LOGICALREP_TWOPHASE_STATE_DISABLED ||
1802 new_state == LOGICALREP_TWOPHASE_STATE_PENDING ||
1803 new_state == LOGICALREP_TWOPHASE_STATE_ENABLED);
1804
1805 rel = table_open(SubscriptionRelationId, RowExclusiveLock);
1806 tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(suboid));
1807 if (!HeapTupleIsValid(tup))
1808 elog(ERROR,
1809 "cache lookup failed for subscription oid %u",
1810 suboid);
1811
1812 /* Form a new tuple. */
1813 memset(values, 0, sizeof(values));
1814 memset(nulls, false, sizeof(nulls));
1815 memset(replaces, false, sizeof(replaces));
1816
1817 /* And update/set two_phase state */
1818 values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(new_state);
1819 replaces[Anum_pg_subscription_subtwophasestate - 1] = true;
1820
1821 tup = heap_modify_tuple(tup, RelationGetDescr(rel),
1822 values, nulls, replaces);
1823 CatalogTupleUpdate(rel, &tup->t_self, tup);
1824
1825 heap_freetuple(tup);
1826 table_close(rel, RowExclusiveLock);
1827 }