Debian Codegraph @

src/backend/replication/logical/worker.c

raw
1 /*-------------------------------------------------------------------------
2 * worker.c
3 * PostgreSQL logical replication worker (apply)
4 *
5 * Copyright (c) 2016-2025, PostgreSQL Global Development Group
6 *
7 * IDENTIFICATION
8 * src/backend/replication/logical/worker.c
9 *
10 * NOTES
11 * This file contains the worker which applies logical changes as they come
12 * from remote logical replication stream.
13 *
14 * The main worker (apply) is started by logical replication worker
15 * launcher for every enabled subscription in a database. It uses
16 * walsender protocol to communicate with publisher.
17 *
18 * This module includes server facing code and shares libpqwalreceiver
19 * module with walreceiver for providing the libpq specific functionality.
20 *
21 *
22 * STREAMED TRANSACTIONS
23 * ---------------------
24 * Streamed transactions (large transactions exceeding a memory limit on the
25 * upstream) are applied using one of two approaches:
26 *
27 * 1) Write to temporary files and apply when the final commit arrives
28 *
29 * This approach is used when the user has set the subscription's streaming
30 * option as on.
31 *
32 * Unlike the regular (non-streamed) case, handling streamed transactions has
33 * to handle aborts of both the toplevel transaction and subtransactions. This
34 * is achieved by tracking offsets for subtransactions, which is then used
35 * to truncate the file with serialized changes.
36 *
37 * The files are placed in tmp file directory by default, and the filenames
38 * include both the XID of the toplevel transaction and OID of the
39 * subscription. This is necessary so that different workers processing a
40 * remote transaction with the same XID doesn't interfere.
41 *
42 * We use BufFiles instead of using normal temporary files because (a) the
43 * BufFile infrastructure supports temporary files that exceed the OS file size
44 * limit, (b) provides a way for automatic clean up on the error and (c) provides
45 * a way to survive these files across local transactions and allow to open and
46 * close at stream start and close. We decided to use FileSet
47 * infrastructure as without that it deletes the files on the closure of the
48 * file and if we decide to keep stream files open across the start/stop stream
49 * then it will consume a lot of memory (more than 8K for each BufFile and
50 * there could be multiple such BufFiles as the subscriber could receive
51 * multiple start/stop streams for different transactions before getting the
52 * commit). Moreover, if we don't use FileSet then we also need to invent
53 * a new way to pass filenames to BufFile APIs so that we are allowed to open
54 * the file we desired across multiple stream-open calls for the same
55 * transaction.
56 *
57 * 2) Parallel apply workers.
58 *
59 * This approach is used when the user has set the subscription's streaming
60 * option as parallel. See logical/applyparallelworker.c for information about
61 * this approach.
62 *
63 * TWO_PHASE TRANSACTIONS
64 * ----------------------
65 * Two phase transactions are replayed at prepare and then committed or
66 * rolled back at commit prepared and rollback prepared respectively. It is
67 * possible to have a prepared transaction that arrives at the apply worker
68 * when the tablesync is busy doing the initial copy. In this case, the apply
69 * worker skips all the prepared operations [e.g. inserts] while the tablesync
70 * is still busy (see the condition of should_apply_changes_for_rel). The
71 * tablesync worker might not get such a prepared transaction because say it
72 * was prior to the initial consistent point but might have got some later
73 * commits. Now, the tablesync worker will exit without doing anything for the
74 * prepared transaction skipped by the apply worker as the sync location for it
75 * will be already ahead of the apply worker's current location. This would lead
76 * to an "empty prepare", because later when the apply worker does the commit
77 * prepare, there is nothing in it (the inserts were skipped earlier).
78 *
79 * To avoid this, and similar prepare confusions the subscription's two_phase
80 * commit is enabled only after the initial sync is over. The two_phase option
81 * has been implemented as a tri-state with values DISABLED, PENDING, and
82 * ENABLED.
83 *
84 * Even if the user specifies they want a subscription with two_phase = on,
85 * internally it will start with a tri-state of PENDING which only becomes
86 * ENABLED after all tablesync initializations are completed - i.e. when all
87 * tablesync workers have reached their READY state. In other words, the value
88 * PENDING is only a temporary state for subscription start-up.
89 *
90 * Until the two_phase is properly available (ENABLED) the subscription will
91 * behave as if two_phase = off. When the apply worker detects that all
92 * tablesyncs have become READY (while the tri-state was PENDING) it will
93 * restart the apply worker process. This happens in
94 * process_syncing_tables_for_apply.
95 *
96 * When the (re-started) apply worker finds that all tablesyncs are READY for a
97 * two_phase tri-state of PENDING it start streaming messages with the
98 * two_phase option which in turn enables the decoding of two-phase commits at
99 * the publisher. Then, it updates the tri-state value from PENDING to ENABLED.
100 * Now, it is possible that during the time we have not enabled two_phase, the
101 * publisher (replication server) would have skipped some prepares but we
102 * ensure that such prepares are sent along with commit prepare, see
103 * ReorderBufferFinishPrepared.
104 *
105 * If the subscription has no tables then a two_phase tri-state PENDING is
106 * left unchanged. This lets the user still do an ALTER SUBSCRIPTION REFRESH
107 * PUBLICATION which might otherwise be disallowed (see below).
108 *
109 * If ever a user needs to be aware of the tri-state value, they can fetch it
110 * from the pg_subscription catalog (see column subtwophasestate).
111 *
112 * Finally, to avoid problems mentioned in previous paragraphs from any
113 * subsequent (not READY) tablesyncs (need to toggle two_phase option from 'on'
114 * to 'off' and then again back to 'on') there is a restriction for
115 * ALTER SUBSCRIPTION REFRESH PUBLICATION. This command is not permitted when
116 * the two_phase tri-state is ENABLED, except when copy_data = false.
117 *
118 * We can get prepare of the same GID more than once for the genuine cases
119 * where we have defined multiple subscriptions for publications on the same
120 * server and prepared transaction has operations on tables subscribed to those
121 * subscriptions. For such cases, if we use the GID sent by publisher one of
122 * the prepares will be successful and others will fail, in which case the
123 * server will send them again. Now, this can lead to a deadlock if user has
124 * set synchronous_standby_names for all the subscriptions on subscriber. To
125 * avoid such deadlocks, we generate a unique GID (consisting of the
126 * subscription oid and the xid of the prepared transaction) for each prepare
127 * transaction on the subscriber.
128 *
129 * FAILOVER
130 * ----------------------
131 * The logical slot on the primary can be synced to the standby by specifying
132 * failover = true when creating the subscription. Enabling failover allows us
133 * to smoothly transition to the promoted standby, ensuring that we can
134 * subscribe to the new primary without losing any data.
135 *-------------------------------------------------------------------------
136 */
137
138 #include "postgres.h"
139
140 #include <sys/stat.h>
141 #include <unistd.h>
142
143 #include "access/table.h"
144 #include "access/tableam.h"
145 #include "access/twophase.h"
146 #include "access/xact.h"
147 #include "catalog/indexing.h"
148 #include "catalog/pg_inherits.h"
149 #include "catalog/pg_subscription.h"
150 #include "catalog/pg_subscription_rel.h"
151 #include "commands/tablecmds.h"
152 #include "commands/trigger.h"
153 #include "executor/executor.h"
154 #include "executor/execPartition.h"
155 #include "libpq/pqformat.h"
156 #include "miscadmin.h"
157 #include "optimizer/optimizer.h"
158 #include "parser/parse_relation.h"
159 #include "pgstat.h"
160 #include "postmaster/bgworker.h"
161 #include "postmaster/interrupt.h"
162 #include "postmaster/walwriter.h"
163 #include "replication/conflict.h"
164 #include "replication/logicallauncher.h"
165 #include "replication/logicalproto.h"
166 #include "replication/logicalrelation.h"
167 #include "replication/logicalworker.h"
168 #include "replication/origin.h"
169 #include "replication/walreceiver.h"
170 #include "replication/worker_internal.h"
171 #include "rewrite/rewriteHandler.h"
172 #include "storage/buffile.h"
173 #include "storage/ipc.h"
174 #include "storage/lmgr.h"
175 #include "tcop/tcopprot.h"
176 #include "utils/acl.h"
177 #include "utils/dynahash.h"
178 #include "utils/guc.h"
179 #include "utils/inval.h"
180 #include "utils/lsyscache.h"
181 #include "utils/memutils.h"
182 #include "utils/pg_lsn.h"
183 #include "utils/rel.h"
184 #include "utils/rls.h"
185 #include "utils/snapmgr.h"
186 #include "utils/syscache.h"
187 #include "utils/usercontext.h"
188
189 #define NAPTIME_PER_CYCLE 1000 /* max sleep time between cycles (1s) */
190
191 typedef struct FlushPosition
192 {
193 dlist_node node;
194 XLogRecPtr local_end;
195 XLogRecPtr remote_end;
196 } FlushPosition;
197
198 static dlist_head lsn_mapping = DLIST_STATIC_INIT(lsn_mapping);
199
200 typedef struct ApplyExecutionData
201 {
202 EState *estate; /* executor state, used to track resources */
203
204 LogicalRepRelMapEntry *targetRel; /* replication target rel */
205 ResultRelInfo *targetRelInfo; /* ResultRelInfo for same */
206
207 /* These fields are used when the target relation is partitioned: */
208 ModifyTableState *mtstate; /* dummy ModifyTable state */
209 PartitionTupleRouting *proute; /* partition routing info */
210 } ApplyExecutionData;
211
212 /* Struct for saving and restoring apply errcontext information */
213 typedef struct ApplyErrorCallbackArg
214 {
215 LogicalRepMsgType command; /* 0 if invalid */
216 LogicalRepRelMapEntry *rel;
217
218 /* Remote node information */
219 int remote_attnum; /* -1 if invalid */
220 TransactionId remote_xid;
221 XLogRecPtr finish_lsn;
222 char *origin_name;
223 } ApplyErrorCallbackArg;
224
225 /*
226 * The action to be taken for the changes in the transaction.
227 *
228 * TRANS_LEADER_APPLY:
229 * This action means that we are in the leader apply worker or table sync
230 * worker. The changes of the transaction are either directly applied or
231 * are read from temporary files (for streaming transactions) and then
232 * applied by the worker.
233 *
234 * TRANS_LEADER_SERIALIZE:
235 * This action means that we are in the leader apply worker or table sync
236 * worker. Changes are written to temporary files and then applied when the
237 * final commit arrives.
238 *
239 * TRANS_LEADER_SEND_TO_PARALLEL:
240 * This action means that we are in the leader apply worker and need to send
241 * the changes to the parallel apply worker.
242 *
243 * TRANS_LEADER_PARTIAL_SERIALIZE:
244 * This action means that we are in the leader apply worker and have sent some
245 * changes directly to the parallel apply worker and the remaining changes are
246 * serialized to a file, due to timeout while sending data. The parallel apply
247 * worker will apply these serialized changes when the final commit arrives.
248 *
249 * We can't use TRANS_LEADER_SERIALIZE for this case because, in addition to
250 * serializing changes, the leader worker also needs to serialize the
251 * STREAM_XXX message to a file, and wait for the parallel apply worker to
252 * finish the transaction when processing the transaction finish command. So
253 * this new action was introduced to keep the code and logic clear.
254 *
255 * TRANS_PARALLEL_APPLY:
256 * This action means that we are in the parallel apply worker and changes of
257 * the transaction are applied directly by the worker.
258 */
259 typedef enum
260 {
261 /* The action for non-streaming transactions. */
262 TRANS_LEADER_APPLY,
263
264 /* Actions for streaming transactions. */
265 TRANS_LEADER_SERIALIZE,
266 TRANS_LEADER_SEND_TO_PARALLEL,
267 TRANS_LEADER_PARTIAL_SERIALIZE,
268 TRANS_PARALLEL_APPLY,
269 } TransApplyAction;
270
271 /* errcontext tracker */
272 static ApplyErrorCallbackArg apply_error_callback_arg =
273 {
274 .command = 0,
275 .rel = NULL,
276 .remote_attnum = -1,
277 .remote_xid = InvalidTransactionId,
278 .finish_lsn = InvalidXLogRecPtr,
279 .origin_name = NULL,
280 };
281
282 ErrorContextCallback *apply_error_context_stack = NULL;
283
284 MemoryContext ApplyMessageContext = NULL;
285 MemoryContext ApplyContext = NULL;
286
287 /* per stream context for streaming transactions */
288 static MemoryContext LogicalStreamingContext = NULL;
289
290 WalReceiverConn *LogRepWorkerWalRcvConn = NULL;
291
292 Subscription *MySubscription = NULL;
293 static bool MySubscriptionValid = false;
294
295 static List *on_commit_wakeup_workers_subids = NIL;
296
297 bool in_remote_transaction = false;
298 static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr;
299
300 /* fields valid only when processing streamed transaction */
301 static bool in_streamed_transaction = false;
302
303 static TransactionId stream_xid = InvalidTransactionId;
304
305 /*
306 * The number of changes applied by parallel apply worker during one streaming
307 * block.
308 */
309 static uint32 parallel_stream_nchanges = 0;
310
311 /* Are we initializing an apply worker? */
312 bool InitializingApplyWorker = false;
313
314 /*
315 * We enable skipping all data modification changes (INSERT, UPDATE, etc.) for
316 * the subscription if the remote transaction's finish LSN matches the subskiplsn.
317 * Once we start skipping changes, we don't stop it until we skip all changes of
318 * the transaction even if pg_subscription is updated and MySubscription->skiplsn
319 * gets changed or reset during that. Also, in streaming transaction cases (streaming = on),
320 * we don't skip receiving and spooling the changes since we decide whether or not
321 * to skip applying the changes when starting to apply changes. The subskiplsn is
322 * cleared after successfully skipping the transaction or applying non-empty
323 * transaction. The latter prevents the mistakenly specified subskiplsn from
324 * being left. Note that we cannot skip the streaming transactions when using
325 * parallel apply workers because we cannot get the finish LSN before applying
326 * the changes. So, we don't start parallel apply worker when finish LSN is set
327 * by the user.
328 */
329 static XLogRecPtr skip_xact_finish_lsn = InvalidXLogRecPtr;
330 #define is_skipping_changes() (unlikely(!XLogRecPtrIsInvalid(skip_xact_finish_lsn)))
331
332 /* BufFile handle of the current streaming file */
333 static BufFile *stream_fd = NULL;
334
335 typedef struct SubXactInfo
336 {
337 TransactionId xid; /* XID of the subxact */
338 int fileno; /* file number in the buffile */
339 off_t offset; /* offset in the file */
340 } SubXactInfo;
341
342 /* Sub-transaction data for the current streaming transaction */
343 typedef struct ApplySubXactData
344 {
345 uint32 nsubxacts; /* number of sub-transactions */
346 uint32 nsubxacts_max; /* current capacity of subxacts */
347 TransactionId subxact_last; /* xid of the last sub-transaction */
348 SubXactInfo *subxacts; /* sub-xact offset in changes file */
349 } ApplySubXactData;
350
351 static ApplySubXactData subxact_data = {0, 0, InvalidTransactionId, NULL};
352
353 static inline void subxact_filename(char *path, Oid subid, TransactionId xid);
354 static inline void changes_filename(char *path, Oid subid, TransactionId xid);
355
356 /*
357 * Information about subtransactions of a given toplevel transaction.
358 */
359 static void subxact_info_write(Oid subid, TransactionId xid);
360 static void subxact_info_read(Oid subid, TransactionId xid);
361 static void subxact_info_add(TransactionId xid);
362 static inline void cleanup_subxact_info(void);
363
364 /*
365 * Serialize and deserialize changes for a toplevel transaction.
366 */
367 static void stream_open_file(Oid subid, TransactionId xid,
368 bool first_segment);
369 static void stream_write_change(char action, StringInfo s);
370 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
371 static void stream_close_file(void);
372
373 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
374
375 static void apply_handle_commit_internal(LogicalRepCommitData *commit_data);
376 static void apply_handle_insert_internal(ApplyExecutionData *edata,
377 ResultRelInfo *relinfo,
378 TupleTableSlot *remoteslot);
379 static void apply_handle_update_internal(ApplyExecutionData *edata,
380 ResultRelInfo *relinfo,
381 TupleTableSlot *remoteslot,
382 LogicalRepTupleData *newtup,
383 Oid localindexoid);
384 static void apply_handle_delete_internal(ApplyExecutionData *edata,
385 ResultRelInfo *relinfo,
386 TupleTableSlot *remoteslot,
387 Oid localindexoid);
388 static bool FindReplTupleInLocalRel(ApplyExecutionData *edata, Relation localrel,
389 LogicalRepRelation *remoterel,
390 Oid localidxoid,
391 TupleTableSlot *remoteslot,
392 TupleTableSlot **localslot);
393 static void apply_handle_tuple_routing(ApplyExecutionData *edata,
394 TupleTableSlot *remoteslot,
395 LogicalRepTupleData *newtup,
396 CmdType operation);
397
398 /* Functions for skipping changes */
399 static void maybe_start_skipping_changes(XLogRecPtr finish_lsn);
400 static void stop_skipping_changes(void);
401 static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn);
402
403 /* Functions for apply error callback */
404 static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn);
405 static inline void reset_apply_error_context_info(void);
406
407 static TransApplyAction get_transaction_apply_action(TransactionId xid,
408 ParallelApplyWorkerInfo **winfo);
409
410 static void replorigin_reset(int code, Datum arg);
411
412 /*
413 * Form the origin name for the subscription.
414 *
415 * This is a common function for tablesync and other workers. Tablesync workers
416 * must pass a valid relid. Other callers must pass relid = InvalidOid.
417 *
418 * Return the name in the supplied buffer.
419 */
420 void
421 ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
422 char *originname, Size szoriginname)
423 {
424 if (OidIsValid(relid))
425 {
426 /* Replication origin name for tablesync workers. */
427 snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
428 }
429 else
430 {
431 /* Replication origin name for non-tablesync workers. */
432 snprintf(originname, szoriginname, "pg_%u", suboid);
433 }
434 }
435
436 /*
437 * Should this worker apply changes for given relation.
438 *
439 * This is mainly needed for initial relation data sync as that runs in
440 * separate worker process running in parallel and we need some way to skip
441 * changes coming to the leader apply worker during the sync of a table.
442 *
443 * Note we need to do smaller or equals comparison for SYNCDONE state because
444 * it might hold position of end of initial slot consistent point WAL
445 * record + 1 (ie start of next record) and next record can be COMMIT of
446 * transaction we are now processing (which is what we set remote_final_lsn
447 * to in apply_handle_begin).
448 *
449 * Note that for streaming transactions that are being applied in the parallel
450 * apply worker, we disallow applying changes if the target table in the
451 * subscription is not in the READY state, because we cannot decide whether to
452 * apply the change as we won't know remote_final_lsn by that time.
453 *
454 * We already checked this in pa_can_start() before assigning the
455 * streaming transaction to the parallel worker, but it also needs to be
456 * checked here because if the user executes ALTER SUBSCRIPTION ... REFRESH
457 * PUBLICATION in parallel, the new table can be added to pg_subscription_rel
458 * while applying this transaction.
459 */
460 static bool
461 should_apply_changes_for_rel(LogicalRepRelMapEntry *rel)
462 {
463 switch (MyLogicalRepWorker->type)
464 {
465 case WORKERTYPE_TABLESYNC:
466 return MyLogicalRepWorker->relid == rel->localreloid;
467
468 case WORKERTYPE_PARALLEL_APPLY:
469 /* We don't synchronize rel's that are in unknown state. */
470 if (rel->state != SUBREL_STATE_READY &&
471 rel->state != SUBREL_STATE_UNKNOWN)
472 ereport(ERROR,
473 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
474 errmsg("logical replication parallel apply worker for subscription \"%s\" will stop",
475 MySubscription->name),
476 errdetail("Cannot handle streamed replication transactions using parallel apply workers until all tables have been synchronized.")));
477
478 return rel->state == SUBREL_STATE_READY;
479
480 case WORKERTYPE_APPLY:
481 return (rel->state == SUBREL_STATE_READY ||
482 (rel->state == SUBREL_STATE_SYNCDONE &&
483 rel->statelsn <= remote_final_lsn));
484
485 case WORKERTYPE_UNKNOWN:
486 /* Should never happen. */
487 elog(ERROR, "Unknown worker type");
488 }
489
490 return false; /* dummy for compiler */
491 }
492
493 /*
494 * Begin one step (one INSERT, UPDATE, etc) of a replication transaction.
495 *
496 * Start a transaction, if this is the first step (else we keep using the
497 * existing transaction).
498 * Also provide a global snapshot and ensure we run in ApplyMessageContext.
499 */
500 static void
501 begin_replication_step(void)
502 {
503 SetCurrentStatementStartTimestamp();
504
505 if (!IsTransactionState())
506 {
507 StartTransactionCommand();
508 maybe_reread_subscription();
509 }
510
511 PushActiveSnapshot(GetTransactionSnapshot());
512
513 MemoryContextSwitchTo(ApplyMessageContext);
514 }
515
516 /*
517 * Finish up one step of a replication transaction.
518 * Callers of begin_replication_step() must also call this.
519 *
520 * We don't close out the transaction here, but we should increment
521 * the command counter to make the effects of this step visible.
522 */
523 static void
524 end_replication_step(void)
525 {
526 PopActiveSnapshot();
527
528 CommandCounterIncrement();
529 }
530
531 /*
532 * Handle streamed transactions for both the leader apply worker and the
533 * parallel apply workers.
534 *
535 * In the streaming case (receiving a block of the streamed transaction), for
536 * serialize mode, simply redirect it to a file for the proper toplevel
537 * transaction, and for parallel mode, the leader apply worker will send the
538 * changes to parallel apply workers and the parallel apply worker will define
539 * savepoints if needed. (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE
540 * messages will be applied by both leader apply worker and parallel apply
541 * workers).
542 *
543 * Returns true for streamed transactions (when the change is either serialized
544 * to file or sent to parallel apply worker), false otherwise (regular mode or
545 * needs to be processed by parallel apply worker).
546 *
547 * Exception: If the message being processed is LOGICAL_REP_MSG_RELATION
548 * or LOGICAL_REP_MSG_TYPE, return false even if the message needs to be sent
549 * to a parallel apply worker.
550 */
551 static bool
552 handle_streamed_transaction(LogicalRepMsgType action, StringInfo s)
553 {
554 TransactionId current_xid;
555 ParallelApplyWorkerInfo *winfo;
556 TransApplyAction apply_action;
557 StringInfoData original_msg;
558
559 apply_action = get_transaction_apply_action(stream_xid, &winfo);
560
561 /* not in streaming mode */
562 if (apply_action == TRANS_LEADER_APPLY)
563 return false;
564
565 Assert(TransactionIdIsValid(stream_xid));
566
567 /*
568 * The parallel apply worker needs the xid in this message to decide
569 * whether to define a savepoint, so save the original message that has
570 * not moved the cursor after the xid. We will serialize this message to a
571 * file in PARTIAL_SERIALIZE mode.
572 */
573 original_msg = *s;
574
575 /*
576 * We should have received XID of the subxact as the first part of the
577 * message, so extract it.
578 */
579 current_xid = pq_getmsgint(s, 4);
580
581 if (!TransactionIdIsValid(current_xid))
582 ereport(ERROR,
583 (errcode(ERRCODE_PROTOCOL_VIOLATION),
584 errmsg_internal("invalid transaction ID in streamed replication transaction")));
585
586 switch (apply_action)
587 {
588 case TRANS_LEADER_SERIALIZE:
589 Assert(stream_fd);
590
591 /* Add the new subxact to the array (unless already there). */
592 subxact_info_add(current_xid);
593
594 /* Write the change to the current file */
595 stream_write_change(action, s);
596 return true;
597
598 case TRANS_LEADER_SEND_TO_PARALLEL:
599 Assert(winfo);
600
601 /*
602 * XXX The publisher side doesn't always send relation/type update
603 * messages after the streaming transaction, so also update the
604 * relation/type in leader apply worker. See function
605 * cleanup_rel_sync_cache.
606 */
607 if (pa_send_data(winfo, s->len, s->data))
608 return (action != LOGICAL_REP_MSG_RELATION &&
609 action != LOGICAL_REP_MSG_TYPE);
610
611 /*
612 * Switch to serialize mode when we are not able to send the
613 * change to parallel apply worker.
614 */
615 pa_switch_to_partial_serialize(winfo, false);
616
617 /* fall through */
618 case TRANS_LEADER_PARTIAL_SERIALIZE:
619 stream_write_change(action, &original_msg);
620
621 /* Same reason as TRANS_LEADER_SEND_TO_PARALLEL case. */
622 return (action != LOGICAL_REP_MSG_RELATION &&
623 action != LOGICAL_REP_MSG_TYPE);
624
625 case TRANS_PARALLEL_APPLY:
626 parallel_stream_nchanges += 1;
627
628 /* Define a savepoint for a subxact if needed. */
629 pa_start_subtrans(current_xid, stream_xid);
630 return false;
631
632 default:
633 elog(ERROR, "unexpected apply action: %d", (int) apply_action);
634 return false; /* silence compiler warning */
635 }
636 }
637
638 /*
639 * Executor state preparation for evaluation of constraint expressions,
640 * indexes and triggers for the specified relation.
641 *
642 * Note that the caller must open and close any indexes to be updated.
643 */
644 static ApplyExecutionData *
645 create_edata_for_relation(LogicalRepRelMapEntry *rel)
646 {
647 ApplyExecutionData *edata;
648 EState *estate;
649 RangeTblEntry *rte;
650 List *perminfos = NIL;
651 ResultRelInfo *resultRelInfo;
652
653 edata = (ApplyExecutionData *) palloc0(sizeof(ApplyExecutionData));
654 edata->targetRel = rel;
655
656 edata->estate = estate = CreateExecutorState();
657
658 rte = makeNode(RangeTblEntry);
659 rte->rtekind = RTE_RELATION;
660 rte->relid = RelationGetRelid(rel->localrel);
661 rte->relkind = rel->localrel->rd_rel->relkind;
662 rte->rellockmode = AccessShareLock;
663
664 addRTEPermissionInfo(&perminfos, rte);
665
666 ExecInitRangeTable(estate, list_make1(rte), perminfos,
667 bms_make_singleton(1));
668
669 edata->targetRelInfo = resultRelInfo = makeNode(ResultRelInfo);
670
671 /*
672 * Use Relation opened by logicalrep_rel_open() instead of opening it
673 * again.
674 */
675 InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0);
676
677 /*
678 * We put the ResultRelInfo in the es_opened_result_relations list, even
679 * though we don't populate the es_result_relations array. That's a bit
680 * bogus, but it's enough to make ExecGetTriggerResultRel() find them.
681 *
682 * ExecOpenIndices() is not called here either, each execution path doing
683 * an apply operation being responsible for that.
684 */
685 estate->es_opened_result_relations =
686 lappend(estate->es_opened_result_relations, resultRelInfo);
687
688 estate->es_output_cid = GetCurrentCommandId(true);
689
690 /* Prepare to catch AFTER triggers. */
691 AfterTriggerBeginQuery();
692
693 /* other fields of edata remain NULL for now */
694
695 return edata;
696 }
697
698 /*
699 * Finish any operations related to the executor state created by
700 * create_edata_for_relation().
701 */
702 static void
703 finish_edata(ApplyExecutionData *edata)
704 {
705 EState *estate = edata->estate;
706
707 /* Handle any queued AFTER triggers. */
708 AfterTriggerEndQuery(estate);
709
710 /* Shut down tuple routing, if any was done. */
711 if (edata->proute)
712 ExecCleanupTupleRouting(edata->mtstate, edata->proute);
713
714 /*
715 * Cleanup. It might seem that we should call ExecCloseResultRelations()
716 * here, but we intentionally don't. It would close the rel we added to
717 * es_opened_result_relations above, which is wrong because we took no
718 * corresponding refcount. We rely on ExecCleanupTupleRouting() to close
719 * any other relations opened during execution.
720 */
721 ExecResetTupleTable(estate->es_tupleTable, false);
722 FreeExecutorState(estate);
723 pfree(edata);
724 }
725
726 /*
727 * Executes default values for columns for which we can't map to remote
728 * relation columns.
729 *
730 * This allows us to support tables which have more columns on the downstream
731 * than on the upstream.
732 */
733 static void
734 slot_fill_defaults(LogicalRepRelMapEntry *rel, EState *estate,
735 TupleTableSlot *slot)
736 {
737 TupleDesc desc = RelationGetDescr(rel->localrel);
738 int num_phys_attrs = desc->natts;
739 int i;
740 int attnum,
741 num_defaults = 0;
742 int *defmap;
743 ExprState **defexprs;
744 ExprContext *econtext;
745
746 econtext = GetPerTupleExprContext(estate);
747
748 /* We got all the data via replication, no need to evaluate anything. */
749 if (num_phys_attrs == rel->remoterel.natts)
750 return;
751
752 defmap = (int *) palloc(num_phys_attrs * sizeof(int));
753 defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
754
755 Assert(rel->attrmap->maplen == num_phys_attrs);
756 for (attnum = 0; attnum < num_phys_attrs; attnum++)
757 {
758 Expr *defexpr;
759
760 if (TupleDescAttr(desc, attnum)->attisdropped || TupleDescAttr(desc, attnum)->attgenerated)
761 continue;
762
763 if (rel->attrmap->attnums[attnum] >= 0)
764 continue;
765
766 defexpr = (Expr *) build_column_default(rel->localrel, attnum + 1);
767
768 if (defexpr != NULL)
769 {
770 /* Run the expression through planner */
771 defexpr = expression_planner(defexpr);
772
773 /* Initialize executable expression in copycontext */
774 defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
775 defmap[num_defaults] = attnum;
776 num_defaults++;
777 }
778 }
779
780 for (i = 0; i < num_defaults; i++)
781 slot->tts_values[defmap[i]] =
782 ExecEvalExpr(defexprs[i], econtext, &slot->tts_isnull[defmap[i]]);
783 }
784
785 /*
786 * Store tuple data into slot.
787 *
788 * Incoming data can be either text or binary format.
789 */
790 static void
791 slot_store_data(TupleTableSlot *slot, LogicalRepRelMapEntry *rel,
792 LogicalRepTupleData *tupleData)
793 {
794 int natts = slot->tts_tupleDescriptor->natts;
795 int i;
796
797 ExecClearTuple(slot);
798
799 /* Call the "in" function for each non-dropped, non-null attribute */
800 Assert(natts == rel->attrmap->maplen);
801 for (i = 0; i < natts; i++)
802 {
803 Form_pg_attribute att = TupleDescAttr(slot->tts_tupleDescriptor, i);
804 int remoteattnum = rel->attrmap->attnums[i];
805
806 if (!att->attisdropped && remoteattnum >= 0)
807 {
808 StringInfo colvalue;
809
810 if (remoteattnum >= tupleData->ncols)
811 ereport(ERROR,
812 (errcode(ERRCODE_PROTOCOL_VIOLATION),
813 errmsg("logical replication column %d not found in tuple: only %d column(s) received",
814 remoteattnum + 1, tupleData->ncols)));
815
816 colvalue = &tupleData->colvalues[remoteattnum];
817
818 /* Set attnum for error callback */
819 apply_error_callback_arg.remote_attnum = remoteattnum;
820
821 if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_TEXT)
822 {
823 Oid typinput;
824 Oid typioparam;
825
826 getTypeInputInfo(att->atttypid, &typinput, &typioparam);
827 slot->tts_values[i] =
828 OidInputFunctionCall(typinput, colvalue->data,
829 typioparam, att->atttypmod);
830 slot->tts_isnull[i] = false;
831 }
832 else if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_BINARY)
833 {
834 Oid typreceive;
835 Oid typioparam;
836
837 /*
838 * In some code paths we may be asked to re-parse the same
839 * tuple data. Reset the StringInfo's cursor so that works.
840 */
841 colvalue->cursor = 0;
842
843 getTypeBinaryInputInfo(att->atttypid, &typreceive, &typioparam);
844 slot->tts_values[i] =
845 OidReceiveFunctionCall(typreceive, colvalue,
846 typioparam, att->atttypmod);
847
848 /* Trouble if it didn't eat the whole buffer */
849 if (colvalue->cursor != colvalue->len)
850 ereport(ERROR,
851 (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
852 errmsg("incorrect binary data format in logical replication column %d",
853 remoteattnum + 1)));
854 slot->tts_isnull[i] = false;
855 }
856 else
857 {
858 /*
859 * NULL value from remote. (We don't expect to see
860 * LOGICALREP_COLUMN_UNCHANGED here, but if we do, treat it as
861 * NULL.)
862 */
863 slot->tts_values[i] = (Datum) 0;
864 slot->tts_isnull[i] = true;
865 }
866
867 /* Reset attnum for error callback */
868 apply_error_callback_arg.remote_attnum = -1;
869 }
870 else
871 {
872 /*
873 * We assign NULL to dropped attributes and missing values
874 * (missing values should be later filled using
875 * slot_fill_defaults).
876 */
877 slot->tts_values[i] = (Datum) 0;
878 slot->tts_isnull[i] = true;
879 }
880 }
881
882 ExecStoreVirtualTuple(slot);
883 }
884
885 /*
886 * Replace updated columns with data from the LogicalRepTupleData struct.
887 * This is somewhat similar to heap_modify_tuple but also calls the type
888 * input functions on the user data.
889 *
890 * "slot" is filled with a copy of the tuple in "srcslot", replacing
891 * columns provided in "tupleData" and leaving others as-is.
892 *
893 * Caution: unreplaced pass-by-ref columns in "slot" will point into the
894 * storage for "srcslot". This is OK for current usage, but someday we may
895 * need to materialize "slot" at the end to make it independent of "srcslot".
896 */
897 static void
898 slot_modify_data(TupleTableSlot *slot, TupleTableSlot *srcslot,
899 LogicalRepRelMapEntry *rel,
900 LogicalRepTupleData *tupleData)
901 {
902 int natts = slot->tts_tupleDescriptor->natts;
903 int i;
904
905 /* We'll fill "slot" with a virtual tuple, so we must start with ... */
906 ExecClearTuple(slot);
907
908 /*
909 * Copy all the column data from srcslot, so that we'll have valid values
910 * for unreplaced columns.
911 */
912 Assert(natts == srcslot->tts_tupleDescriptor->natts);
913 slot_getallattrs(srcslot);
914 memcpy(slot->tts_values, srcslot->tts_values, natts * sizeof(Datum));
915 memcpy(slot->tts_isnull, srcslot->tts_isnull, natts * sizeof(bool));
916
917 /* Call the "in" function for each replaced attribute */
918 Assert(natts == rel->attrmap->maplen);
919 for (i = 0; i < natts; i++)
920 {
921 Form_pg_attribute att = TupleDescAttr(slot->tts_tupleDescriptor, i);
922 int remoteattnum = rel->attrmap->attnums[i];
923
924 if (remoteattnum < 0)
925 continue;
926
927 if (remoteattnum >= tupleData->ncols)
928 ereport(ERROR,
929 (errcode(ERRCODE_PROTOCOL_VIOLATION),
930 errmsg("logical replication column %d not found in tuple: only %d column(s) received",
931 remoteattnum + 1, tupleData->ncols)));
932
933 if (tupleData->colstatus[remoteattnum] != LOGICALREP_COLUMN_UNCHANGED)
934 {
935 StringInfo colvalue = &tupleData->colvalues[remoteattnum];
936
937 /* Set attnum for error callback */
938 apply_error_callback_arg.remote_attnum = remoteattnum;
939
940 if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_TEXT)
941 {
942 Oid typinput;
943 Oid typioparam;
944
945 getTypeInputInfo(att->atttypid, &typinput, &typioparam);
946 slot->tts_values[i] =
947 OidInputFunctionCall(typinput, colvalue->data,
948 typioparam, att->atttypmod);
949 slot->tts_isnull[i] = false;
950 }
951 else if (tupleData->colstatus[remoteattnum] == LOGICALREP_COLUMN_BINARY)
952 {
953 Oid typreceive;
954 Oid typioparam;
955
956 /*
957 * In some code paths we may be asked to re-parse the same
958 * tuple data. Reset the StringInfo's cursor so that works.
959 */
960 colvalue->cursor = 0;
961
962 getTypeBinaryInputInfo(att->atttypid, &typreceive, &typioparam);
963 slot->tts_values[i] =
964 OidReceiveFunctionCall(typreceive, colvalue,
965 typioparam, att->atttypmod);
966
967 /* Trouble if it didn't eat the whole buffer */
968 if (colvalue->cursor != colvalue->len)
969 ereport(ERROR,
970 (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
971 errmsg("incorrect binary data format in logical replication column %d",
972 remoteattnum + 1)));
973 slot->tts_isnull[i] = false;
974 }
975 else
976 {
977 /* must be LOGICALREP_COLUMN_NULL */
978 slot->tts_values[i] = (Datum) 0;
979 slot->tts_isnull[i] = true;
980 }
981
982 /* Reset attnum for error callback */
983 apply_error_callback_arg.remote_attnum = -1;
984 }
985 }
986
987 /* And finally, declare that "slot" contains a valid virtual tuple */
988 ExecStoreVirtualTuple(slot);
989 }
990
991 /*
992 * Handle BEGIN message.
993 */
994 static void
995 apply_handle_begin(StringInfo s)
996 {
997 LogicalRepBeginData begin_data;
998
999 /* There must not be an active streaming transaction. */
1000 Assert(!TransactionIdIsValid(stream_xid));
1001
1002 logicalrep_read_begin(s, &begin_data);
1003 set_apply_error_context_xact(begin_data.xid, begin_data.final_lsn);
1004
1005 remote_final_lsn = begin_data.final_lsn;
1006
1007 maybe_start_skipping_changes(begin_data.final_lsn);
1008
1009 in_remote_transaction = true;
1010
1011 pgstat_report_activity(STATE_RUNNING, NULL);
1012 }
1013
1014 /*
1015 * Handle COMMIT message.
1016 *
1017 * TODO, support tracking of multiple origins
1018 */
1019 static void
1020 apply_handle_commit(StringInfo s)
1021 {
1022 LogicalRepCommitData commit_data;
1023
1024 logicalrep_read_commit(s, &commit_data);
1025
1026 if (commit_data.commit_lsn != remote_final_lsn)
1027 ereport(ERROR,
1028 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1029 errmsg_internal("incorrect commit LSN %X/%X in commit message (expected %X/%X)",
1030 LSN_FORMAT_ARGS(commit_data.commit_lsn),
1031 LSN_FORMAT_ARGS(remote_final_lsn))));
1032
1033 apply_handle_commit_internal(&commit_data);
1034
1035 /* Process any tables that are being synchronized in parallel. */
1036 process_syncing_tables(commit_data.end_lsn);
1037
1038 pgstat_report_activity(STATE_IDLE, NULL);
1039 reset_apply_error_context_info();
1040 }
1041
1042 /*
1043 * Handle BEGIN PREPARE message.
1044 */
1045 static void
1046 apply_handle_begin_prepare(StringInfo s)
1047 {
1048 LogicalRepPreparedTxnData begin_data;
1049
1050 /* Tablesync should never receive prepare. */
1051 if (am_tablesync_worker())
1052 ereport(ERROR,
1053 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1054 errmsg_internal("tablesync worker received a BEGIN PREPARE message")));
1055
1056 /* There must not be an active streaming transaction. */
1057 Assert(!TransactionIdIsValid(stream_xid));
1058
1059 logicalrep_read_begin_prepare(s, &begin_data);
1060 set_apply_error_context_xact(begin_data.xid, begin_data.prepare_lsn);
1061
1062 remote_final_lsn = begin_data.prepare_lsn;
1063
1064 maybe_start_skipping_changes(begin_data.prepare_lsn);
1065
1066 in_remote_transaction = true;
1067
1068 pgstat_report_activity(STATE_RUNNING, NULL);
1069 }
1070
1071 /*
1072 * Common function to prepare the GID.
1073 */
1074 static void
1075 apply_handle_prepare_internal(LogicalRepPreparedTxnData *prepare_data)
1076 {
1077 char gid[GIDSIZE];
1078
1079 /*
1080 * Compute unique GID for two_phase transactions. We don't use GID of
1081 * prepared transaction sent by server as that can lead to deadlock when
1082 * we have multiple subscriptions from same node point to publications on
1083 * the same node. See comments atop worker.c
1084 */
1085 TwoPhaseTransactionGid(MySubscription->oid, prepare_data->xid,
1086 gid, sizeof(gid));
1087
1088 /*
1089 * BeginTransactionBlock is necessary to balance the EndTransactionBlock
1090 * called within the PrepareTransactionBlock below.
1091 */
1092 if (!IsTransactionBlock())
1093 {
1094 BeginTransactionBlock();
1095 CommitTransactionCommand(); /* Completes the preceding Begin command. */
1096 }
1097
1098 /*
1099 * Update origin state so we can restart streaming from correct position
1100 * in case of crash.
1101 */
1102 replorigin_session_origin_lsn = prepare_data->end_lsn;
1103 replorigin_session_origin_timestamp = prepare_data->prepare_time;
1104
1105 PrepareTransactionBlock(gid);
1106 }
1107
1108 /*
1109 * Handle PREPARE message.
1110 */
1111 static void
1112 apply_handle_prepare(StringInfo s)
1113 {
1114 LogicalRepPreparedTxnData prepare_data;
1115
1116 logicalrep_read_prepare(s, &prepare_data);
1117
1118 if (prepare_data.prepare_lsn != remote_final_lsn)
1119 ereport(ERROR,
1120 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1121 errmsg_internal("incorrect prepare LSN %X/%X in prepare message (expected %X/%X)",
1122 LSN_FORMAT_ARGS(prepare_data.prepare_lsn),
1123 LSN_FORMAT_ARGS(remote_final_lsn))));
1124
1125 /*
1126 * Unlike commit, here, we always prepare the transaction even though no
1127 * change has happened in this transaction or all changes are skipped. It
1128 * is done this way because at commit prepared time, we won't know whether
1129 * we have skipped preparing a transaction because of those reasons.
1130 *
1131 * XXX, We can optimize such that at commit prepared time, we first check
1132 * whether we have prepared the transaction or not but that doesn't seem
1133 * worthwhile because such cases shouldn't be common.
1134 */
1135 begin_replication_step();
1136
1137 apply_handle_prepare_internal(&prepare_data);
1138
1139 end_replication_step();
1140 CommitTransactionCommand();
1141 pgstat_report_stat(false);
1142
1143 /*
1144 * It is okay not to set the local_end LSN for the prepare because we
1145 * always flush the prepare record. So, we can send the acknowledgment of
1146 * the remote_end LSN as soon as prepare is finished.
1147 *
1148 * XXX For the sake of consistency with commit, we could have set it with
1149 * the LSN of prepare but as of now we don't track that value similar to
1150 * XactLastCommitEnd, and adding it for this purpose doesn't seems worth
1151 * it.
1152 */
1153 store_flush_position(prepare_data.end_lsn, InvalidXLogRecPtr);
1154
1155 in_remote_transaction = false;
1156
1157 /* Process any tables that are being synchronized in parallel. */
1158 process_syncing_tables(prepare_data.end_lsn);
1159
1160 /*
1161 * Since we have already prepared the transaction, in a case where the
1162 * server crashes before clearing the subskiplsn, it will be left but the
1163 * transaction won't be resent. But that's okay because it's a rare case
1164 * and the subskiplsn will be cleared when finishing the next transaction.
1165 */
1166 stop_skipping_changes();
1167 clear_subscription_skip_lsn(prepare_data.prepare_lsn);
1168
1169 pgstat_report_activity(STATE_IDLE, NULL);
1170 reset_apply_error_context_info();
1171 }
1172
1173 /*
1174 * Handle a COMMIT PREPARED of a previously PREPARED transaction.
1175 *
1176 * Note that we don't need to wait here if the transaction was prepared in a
1177 * parallel apply worker. In that case, we have already waited for the prepare
1178 * to finish in apply_handle_stream_prepare() which will ensure all the
1179 * operations in that transaction have happened in the subscriber, so no
1180 * concurrent transaction can cause deadlock or transaction dependency issues.
1181 */
1182 static void
1183 apply_handle_commit_prepared(StringInfo s)
1184 {
1185 LogicalRepCommitPreparedTxnData prepare_data;
1186 char gid[GIDSIZE];
1187
1188 logicalrep_read_commit_prepared(s, &prepare_data);
1189 set_apply_error_context_xact(prepare_data.xid, prepare_data.commit_lsn);
1190
1191 /* Compute GID for two_phase transactions. */
1192 TwoPhaseTransactionGid(MySubscription->oid, prepare_data.xid,
1193 gid, sizeof(gid));
1194
1195 /* There is no transaction when COMMIT PREPARED is called */
1196 begin_replication_step();
1197
1198 /*
1199 * Update origin state so we can restart streaming from correct position
1200 * in case of crash.
1201 */
1202 replorigin_session_origin_lsn = prepare_data.end_lsn;
1203 replorigin_session_origin_timestamp = prepare_data.commit_time;
1204
1205 FinishPreparedTransaction(gid, true);
1206 end_replication_step();
1207 CommitTransactionCommand();
1208 pgstat_report_stat(false);
1209
1210 store_flush_position(prepare_data.end_lsn, XactLastCommitEnd);
1211 in_remote_transaction = false;
1212
1213 /* Process any tables that are being synchronized in parallel. */
1214 process_syncing_tables(prepare_data.end_lsn);
1215
1216 clear_subscription_skip_lsn(prepare_data.end_lsn);
1217
1218 pgstat_report_activity(STATE_IDLE, NULL);
1219 reset_apply_error_context_info();
1220 }
1221
1222 /*
1223 * Handle a ROLLBACK PREPARED of a previously PREPARED TRANSACTION.
1224 *
1225 * Note that we don't need to wait here if the transaction was prepared in a
1226 * parallel apply worker. In that case, we have already waited for the prepare
1227 * to finish in apply_handle_stream_prepare() which will ensure all the
1228 * operations in that transaction have happened in the subscriber, so no
1229 * concurrent transaction can cause deadlock or transaction dependency issues.
1230 */
1231 static void
1232 apply_handle_rollback_prepared(StringInfo s)
1233 {
1234 LogicalRepRollbackPreparedTxnData rollback_data;
1235 char gid[GIDSIZE];
1236
1237 logicalrep_read_rollback_prepared(s, &rollback_data);
1238 set_apply_error_context_xact(rollback_data.xid, rollback_data.rollback_end_lsn);
1239
1240 /* Compute GID for two_phase transactions. */
1241 TwoPhaseTransactionGid(MySubscription->oid, rollback_data.xid,
1242 gid, sizeof(gid));
1243
1244 /*
1245 * It is possible that we haven't received prepare because it occurred
1246 * before walsender reached a consistent point or the two_phase was still
1247 * not enabled by that time, so in such cases, we need to skip rollback
1248 * prepared.
1249 */
1250 if (LookupGXact(gid, rollback_data.prepare_end_lsn,
1251 rollback_data.prepare_time))
1252 {
1253 /*
1254 * Update origin state so we can restart streaming from correct
1255 * position in case of crash.
1256 */
1257 replorigin_session_origin_lsn = rollback_data.rollback_end_lsn;
1258 replorigin_session_origin_timestamp = rollback_data.rollback_time;
1259
1260 /* There is no transaction when ABORT/ROLLBACK PREPARED is called */
1261 begin_replication_step();
1262 FinishPreparedTransaction(gid, false);
1263 end_replication_step();
1264 CommitTransactionCommand();
1265
1266 clear_subscription_skip_lsn(rollback_data.rollback_end_lsn);
1267 }
1268
1269 pgstat_report_stat(false);
1270
1271 /*
1272 * It is okay not to set the local_end LSN for the rollback of prepared
1273 * transaction because we always flush the WAL record for it. See
1274 * apply_handle_prepare.
1275 */
1276 store_flush_position(rollback_data.rollback_end_lsn, InvalidXLogRecPtr);
1277 in_remote_transaction = false;
1278
1279 /* Process any tables that are being synchronized in parallel. */
1280 process_syncing_tables(rollback_data.rollback_end_lsn);
1281
1282 pgstat_report_activity(STATE_IDLE, NULL);
1283 reset_apply_error_context_info();
1284 }
1285
1286 /*
1287 * Handle STREAM PREPARE.
1288 */
1289 static void
1290 apply_handle_stream_prepare(StringInfo s)
1291 {
1292 LogicalRepPreparedTxnData prepare_data;
1293 ParallelApplyWorkerInfo *winfo;
1294 TransApplyAction apply_action;
1295
1296 /* Save the message before it is consumed. */
1297 StringInfoData original_msg = *s;
1298
1299 if (in_streamed_transaction)
1300 ereport(ERROR,
1301 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1302 errmsg_internal("STREAM PREPARE message without STREAM STOP")));
1303
1304 /* Tablesync should never receive prepare. */
1305 if (am_tablesync_worker())
1306 ereport(ERROR,
1307 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1308 errmsg_internal("tablesync worker received a STREAM PREPARE message")));
1309
1310 logicalrep_read_stream_prepare(s, &prepare_data);
1311 set_apply_error_context_xact(prepare_data.xid, prepare_data.prepare_lsn);
1312
1313 apply_action = get_transaction_apply_action(prepare_data.xid, &winfo);
1314
1315 switch (apply_action)
1316 {
1317 case TRANS_LEADER_APPLY:
1318
1319 /*
1320 * The transaction has been serialized to file, so replay all the
1321 * spooled operations.
1322 */
1323 apply_spooled_messages(MyLogicalRepWorker->stream_fileset,
1324 prepare_data.xid, prepare_data.prepare_lsn);
1325
1326 /* Mark the transaction as prepared. */
1327 apply_handle_prepare_internal(&prepare_data);
1328
1329 CommitTransactionCommand();
1330
1331 /*
1332 * It is okay not to set the local_end LSN for the prepare because
1333 * we always flush the prepare record. See apply_handle_prepare.
1334 */
1335 store_flush_position(prepare_data.end_lsn, InvalidXLogRecPtr);
1336
1337 in_remote_transaction = false;
1338
1339 /* Unlink the files with serialized changes and subxact info. */
1340 stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid);
1341
1342 elog(DEBUG1, "finished processing the STREAM PREPARE command");
1343 break;
1344
1345 case TRANS_LEADER_SEND_TO_PARALLEL:
1346 Assert(winfo);
1347
1348 if (pa_send_data(winfo, s->len, s->data))
1349 {
1350 /* Finish processing the streaming transaction. */
1351 pa_xact_finish(winfo, prepare_data.end_lsn);
1352 break;
1353 }
1354
1355 /*
1356 * Switch to serialize mode when we are not able to send the
1357 * change to parallel apply worker.
1358 */
1359 pa_switch_to_partial_serialize(winfo, true);
1360
1361 /* fall through */
1362 case TRANS_LEADER_PARTIAL_SERIALIZE:
1363 Assert(winfo);
1364
1365 stream_open_and_write_change(prepare_data.xid,
1366 LOGICAL_REP_MSG_STREAM_PREPARE,
1367 &original_msg);
1368
1369 pa_set_fileset_state(winfo->shared, FS_SERIALIZE_DONE);
1370
1371 /* Finish processing the streaming transaction. */
1372 pa_xact_finish(winfo, prepare_data.end_lsn);
1373 break;
1374
1375 case TRANS_PARALLEL_APPLY:
1376
1377 /*
1378 * If the parallel apply worker is applying spooled messages then
1379 * close the file before preparing.
1380 */
1381 if (stream_fd)
1382 stream_close_file();
1383
1384 begin_replication_step();
1385
1386 /* Mark the transaction as prepared. */
1387 apply_handle_prepare_internal(&prepare_data);
1388
1389 end_replication_step();
1390
1391 CommitTransactionCommand();
1392
1393 /*
1394 * It is okay not to set the local_end LSN for the prepare because
1395 * we always flush the prepare record. See apply_handle_prepare.
1396 */
1397 MyParallelShared->last_commit_end = InvalidXLogRecPtr;
1398
1399 pa_set_xact_state(MyParallelShared, PARALLEL_TRANS_FINISHED);
1400 pa_unlock_transaction(MyParallelShared->xid, AccessExclusiveLock);
1401
1402 pa_reset_subtrans();
1403
1404 elog(DEBUG1, "finished processing the STREAM PREPARE command");
1405 break;
1406
1407 default:
1408 elog(ERROR, "unexpected apply action: %d", (int) apply_action);
1409 break;
1410 }
1411
1412 pgstat_report_stat(false);
1413
1414 /* Process any tables that are being synchronized in parallel. */
1415 process_syncing_tables(prepare_data.end_lsn);
1416
1417 /*
1418 * Similar to prepare case, the subskiplsn could be left in a case of
1419 * server crash but it's okay. See the comments in apply_handle_prepare().
1420 */
1421 stop_skipping_changes();
1422 clear_subscription_skip_lsn(prepare_data.prepare_lsn);
1423
1424 pgstat_report_activity(STATE_IDLE, NULL);
1425
1426 reset_apply_error_context_info();
1427 }
1428
1429 /*
1430 * Handle ORIGIN message.
1431 *
1432 * TODO, support tracking of multiple origins
1433 */
1434 static void
1435 apply_handle_origin(StringInfo s)
1436 {
1437 /*
1438 * ORIGIN message can only come inside streaming transaction or inside
1439 * remote transaction and before any actual writes.
1440 */
1441 if (!in_streamed_transaction &&
1442 (!in_remote_transaction ||
1443 (IsTransactionState() && !am_tablesync_worker())))
1444 ereport(ERROR,
1445 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1446 errmsg_internal("ORIGIN message sent out of order")));
1447 }
1448
1449 /*
1450 * Initialize fileset (if not already done).
1451 *
1452 * Create a new file when first_segment is true, otherwise open the existing
1453 * file.
1454 */
1455 void
1456 stream_start_internal(TransactionId xid, bool first_segment)
1457 {
1458 begin_replication_step();
1459
1460 /*
1461 * Initialize the worker's stream_fileset if we haven't yet. This will be
1462 * used for the entire duration of the worker so create it in a permanent
1463 * context. We create this on the very first streaming message from any
1464 * transaction and then use it for this and other streaming transactions.
1465 * Now, we could create a fileset at the start of the worker as well but
1466 * then we won't be sure that it will ever be used.
1467 */
1468 if (!MyLogicalRepWorker->stream_fileset)
1469 {
1470 MemoryContext oldctx;
1471
1472 oldctx = MemoryContextSwitchTo(ApplyContext);
1473
1474 MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet));
1475 FileSetInit(MyLogicalRepWorker->stream_fileset);
1476
1477 MemoryContextSwitchTo(oldctx);
1478 }
1479
1480 /* Open the spool file for this transaction. */
1481 stream_open_file(MyLogicalRepWorker->subid, xid, first_segment);
1482
1483 /* If this is not the first segment, open existing subxact file. */
1484 if (!first_segment)
1485 subxact_info_read(MyLogicalRepWorker->subid, xid);
1486
1487 end_replication_step();
1488 }
1489
1490 /*
1491 * Handle STREAM START message.
1492 */
1493 static void
1494 apply_handle_stream_start(StringInfo s)
1495 {
1496 bool first_segment;
1497 ParallelApplyWorkerInfo *winfo;
1498 TransApplyAction apply_action;
1499
1500 /* Save the message before it is consumed. */
1501 StringInfoData original_msg = *s;
1502
1503 if (in_streamed_transaction)
1504 ereport(ERROR,
1505 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1506 errmsg_internal("duplicate STREAM START message")));
1507
1508 /* There must not be an active streaming transaction. */
1509 Assert(!TransactionIdIsValid(stream_xid));
1510
1511 /* notify handle methods we're processing a remote transaction */
1512 in_streamed_transaction = true;
1513
1514 /* extract XID of the top-level transaction */
1515 stream_xid = logicalrep_read_stream_start(s, &first_segment);
1516
1517 if (!TransactionIdIsValid(stream_xid))
1518 ereport(ERROR,
1519 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1520 errmsg_internal("invalid transaction ID in streamed replication transaction")));
1521
1522 set_apply_error_context_xact(stream_xid, InvalidXLogRecPtr);
1523
1524 /* Try to allocate a worker for the streaming transaction. */
1525 if (first_segment)
1526 pa_allocate_worker(stream_xid);
1527
1528 apply_action = get_transaction_apply_action(stream_xid, &winfo);
1529
1530 switch (apply_action)
1531 {
1532 case TRANS_LEADER_SERIALIZE:
1533
1534 /*
1535 * Function stream_start_internal starts a transaction. This
1536 * transaction will be committed on the stream stop unless it is a
1537 * tablesync worker in which case it will be committed after
1538 * processing all the messages. We need this transaction for
1539 * handling the BufFile, used for serializing the streaming data
1540 * and subxact info.
1541 */
1542 stream_start_internal(stream_xid, first_segment);
1543 break;
1544
1545 case TRANS_LEADER_SEND_TO_PARALLEL:
1546 Assert(winfo);
1547
1548 /*
1549 * Once we start serializing the changes, the parallel apply
1550 * worker will wait for the leader to release the stream lock
1551 * until the end of the transaction. So, we don't need to release
1552 * the lock or increment the stream count in that case.
1553 */
1554 if (pa_send_data(winfo, s->len, s->data))
1555 {
1556 /*
1557 * Unlock the shared object lock so that the parallel apply
1558 * worker can continue to receive changes.
1559 */
1560 if (!first_segment)
1561 pa_unlock_stream(winfo->shared->xid, AccessExclusiveLock);
1562
1563 /*
1564 * Increment the number of streaming blocks waiting to be
1565 * processed by parallel apply worker.
1566 */
1567 pg_atomic_add_fetch_u32(&winfo->shared->pending_stream_count, 1);
1568
1569 /* Cache the parallel apply worker for this transaction. */
1570 pa_set_stream_apply_worker(winfo);
1571 break;
1572 }
1573
1574 /*
1575 * Switch to serialize mode when we are not able to send the
1576 * change to parallel apply worker.
1577 */
1578 pa_switch_to_partial_serialize(winfo, !first_segment);
1579
1580 /* fall through */
1581 case TRANS_LEADER_PARTIAL_SERIALIZE:
1582 Assert(winfo);
1583
1584 /*
1585 * Open the spool file unless it was already opened when switching
1586 * to serialize mode. The transaction started in
1587 * stream_start_internal will be committed on the stream stop.
1588 */
1589 if (apply_action != TRANS_LEADER_SEND_TO_PARALLEL)
1590 stream_start_internal(stream_xid, first_segment);
1591
1592 stream_write_change(LOGICAL_REP_MSG_STREAM_START, &original_msg);
1593
1594 /* Cache the parallel apply worker for this transaction. */
1595 pa_set_stream_apply_worker(winfo);
1596 break;
1597
1598 case TRANS_PARALLEL_APPLY:
1599 if (first_segment)
1600 {
1601 /* Hold the lock until the end of the transaction. */
1602 pa_lock_transaction(MyParallelShared->xid, AccessExclusiveLock);
1603 pa_set_xact_state(MyParallelShared, PARALLEL_TRANS_STARTED);
1604
1605 /*
1606 * Signal the leader apply worker, as it may be waiting for
1607 * us.
1608 */
1609 logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
1610 }
1611
1612 parallel_stream_nchanges = 0;
1613 break;
1614
1615 default:
1616 elog(ERROR, "unexpected apply action: %d", (int) apply_action);
1617 break;
1618 }
1619
1620 pgstat_report_activity(STATE_RUNNING, NULL);
1621 }
1622
1623 /*
1624 * Update the information about subxacts and close the file.
1625 *
1626 * This function should be called when the stream_start_internal function has
1627 * been called.
1628 */
1629 void
1630 stream_stop_internal(TransactionId xid)
1631 {
1632 /*
1633 * Serialize information about subxacts for the toplevel transaction, then
1634 * close the stream messages spool file.
1635 */
1636 subxact_info_write(MyLogicalRepWorker->subid, xid);
1637 stream_close_file();
1638
1639 /* We must be in a valid transaction state */
1640 Assert(IsTransactionState());
1641
1642 /* Commit the per-stream transaction */
1643 CommitTransactionCommand();
1644
1645 /* Reset per-stream context */
1646 MemoryContextReset(LogicalStreamingContext);
1647 }
1648
1649 /*
1650 * Handle STREAM STOP message.
1651 */
1652 static void
1653 apply_handle_stream_stop(StringInfo s)
1654 {
1655 ParallelApplyWorkerInfo *winfo;
1656 TransApplyAction apply_action;
1657
1658 if (!in_streamed_transaction)
1659 ereport(ERROR,
1660 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1661 errmsg_internal("STREAM STOP message without STREAM START")));
1662
1663 apply_action = get_transaction_apply_action(stream_xid, &winfo);
1664
1665 switch (apply_action)
1666 {
1667 case TRANS_LEADER_SERIALIZE:
1668 stream_stop_internal(stream_xid);
1669 break;
1670
1671 case TRANS_LEADER_SEND_TO_PARALLEL:
1672 Assert(winfo);
1673
1674 /*
1675 * Lock before sending the STREAM_STOP message so that the leader
1676 * can hold the lock first and the parallel apply worker will wait
1677 * for leader to release the lock. See Locking Considerations atop
1678 * applyparallelworker.c.
1679 */
1680 pa_lock_stream(winfo->shared->xid, AccessExclusiveLock);
1681
1682 if (pa_send_data(winfo, s->len, s->data))
1683 {
1684 pa_set_stream_apply_worker(NULL);
1685 break;
1686 }
1687
1688 /*
1689 * Switch to serialize mode when we are not able to send the
1690 * change to parallel apply worker.
1691 */
1692 pa_switch_to_partial_serialize(winfo, true);
1693
1694 /* fall through */
1695 case TRANS_LEADER_PARTIAL_SERIALIZE:
1696 stream_write_change(LOGICAL_REP_MSG_STREAM_STOP, s);
1697 stream_stop_internal(stream_xid);
1698 pa_set_stream_apply_worker(NULL);
1699 break;
1700
1701 case TRANS_PARALLEL_APPLY:
1702 elog(DEBUG1, "applied %u changes in the streaming chunk",
1703 parallel_stream_nchanges);
1704
1705 /*
1706 * By the time parallel apply worker is processing the changes in
1707 * the current streaming block, the leader apply worker may have
1708 * sent multiple streaming blocks. This can lead to parallel apply
1709 * worker start waiting even when there are more chunk of streams
1710 * in the queue. So, try to lock only if there is no message left
1711 * in the queue. See Locking Considerations atop
1712 * applyparallelworker.c.
1713 *
1714 * Note that here we have a race condition where we can start
1715 * waiting even when there are pending streaming chunks. This can
1716 * happen if the leader sends another streaming block and acquires
1717 * the stream lock again after the parallel apply worker checks
1718 * that there is no pending streaming block and before it actually
1719 * starts waiting on a lock. We can handle this case by not
1720 * allowing the leader to increment the stream block count during
1721 * the time parallel apply worker acquires the lock but it is not
1722 * clear whether that is worth the complexity.
1723 *
1724 * Now, if this missed chunk contains rollback to savepoint, then
1725 * there is a risk of deadlock which probably shouldn't happen
1726 * after restart.
1727 */
1728 pa_decr_and_wait_stream_block();
1729 break;
1730
1731 default:
1732 elog(ERROR, "unexpected apply action: %d", (int) apply_action);
1733 break;
1734 }
1735
1736 in_streamed_transaction = false;
1737 stream_xid = InvalidTransactionId;
1738
1739 /*
1740 * The parallel apply worker could be in a transaction in which case we
1741 * need to report the state as STATE_IDLEINTRANSACTION.
1742 */
1743 if (IsTransactionOrTransactionBlock())
1744 pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL);
1745 else
1746 pgstat_report_activity(STATE_IDLE, NULL);
1747
1748 reset_apply_error_context_info();
1749 }
1750
1751 /*
1752 * Helper function to handle STREAM ABORT message when the transaction was
1753 * serialized to file.
1754 */
1755 static void
1756 stream_abort_internal(TransactionId xid, TransactionId subxid)
1757 {
1758 /*
1759 * If the two XIDs are the same, it's in fact abort of toplevel xact, so
1760 * just delete the files with serialized info.
1761 */
1762 if (xid == subxid)
1763 stream_cleanup_files(MyLogicalRepWorker->subid, xid);
1764 else
1765 {
1766 /*
1767 * OK, so it's a subxact. We need to read the subxact file for the
1768 * toplevel transaction, determine the offset tracked for the subxact,
1769 * and truncate the file with changes. We also remove the subxacts
1770 * with higher offsets (or rather higher XIDs).
1771 *
1772 * We intentionally scan the array from the tail, because we're likely
1773 * aborting a change for the most recent subtransactions.
1774 *
1775 * We can't use the binary search here as subxact XIDs won't
1776 * necessarily arrive in sorted order, consider the case where we have
1777 * released the savepoint for multiple subtransactions and then
1778 * performed rollback to savepoint for one of the earlier
1779 * sub-transaction.
1780 */
1781 int64 i;
1782 int64 subidx;
1783 BufFile *fd;
1784 bool found = false;
1785 char path[MAXPGPATH];
1786
1787 subidx = -1;
1788 begin_replication_step();
1789 subxact_info_read(MyLogicalRepWorker->subid, xid);
1790
1791 for (i = subxact_data.nsubxacts; i > 0; i--)
1792 {
1793 if (subxact_data.subxacts[i - 1].xid == subxid)
1794 {
1795 subidx = (i - 1);
1796 found = true;
1797 break;
1798 }
1799 }
1800
1801 /*
1802 * If it's an empty sub-transaction then we will not find the subxid
1803 * here so just cleanup the subxact info and return.
1804 */
1805 if (!found)
1806 {
1807 /* Cleanup the subxact info */
1808 cleanup_subxact_info();
1809 end_replication_step();
1810 CommitTransactionCommand();
1811 return;
1812 }
1813
1814 /* open the changes file */
1815 changes_filename(path, MyLogicalRepWorker->subid, xid);
1816 fd = BufFileOpenFileSet(MyLogicalRepWorker->stream_fileset, path,
1817 O_RDWR, false);
1818
1819 /* OK, truncate the file at the right offset */
1820 BufFileTruncateFileSet(fd, subxact_data.subxacts[subidx].fileno,
1821 subxact_data.subxacts[subidx].offset);
1822 BufFileClose(fd);
1823
1824 /* discard the subxacts added later */
1825 subxact_data.nsubxacts = subidx;
1826
1827 /* write the updated subxact list */
1828 subxact_info_write(MyLogicalRepWorker->subid, xid);
1829
1830 end_replication_step();
1831 CommitTransactionCommand();
1832 }
1833 }
1834
1835 /*
1836 * Handle STREAM ABORT message.
1837 */
1838 static void
1839 apply_handle_stream_abort(StringInfo s)
1840 {
1841 TransactionId xid;
1842 TransactionId subxid;
1843 LogicalRepStreamAbortData abort_data;
1844 ParallelApplyWorkerInfo *winfo;
1845 TransApplyAction apply_action;
1846
1847 /* Save the message before it is consumed. */
1848 StringInfoData original_msg = *s;
1849 bool toplevel_xact;
1850
1851 if (in_streamed_transaction)
1852 ereport(ERROR,
1853 (errcode(ERRCODE_PROTOCOL_VIOLATION),
1854 errmsg_internal("STREAM ABORT message without STREAM STOP")));
1855
1856 /* We receive abort information only when we can apply in parallel. */
1857 logicalrep_read_stream_abort(s, &abort_data,
1858 MyLogicalRepWorker->parallel_apply);
1859
1860 xid = abort_data.xid;
1861 subxid = abort_data.subxid;
1862 toplevel_xact = (xid == subxid);
1863
1864 set_apply_error_context_xact(subxid, abort_data.abort_lsn);
1865
1866 apply_action = get_transaction_apply_action(xid, &winfo);
1867
1868 switch (apply_action)
1869 {
1870 case TRANS_LEADER_APPLY:
1871
1872 /*
1873 * We are in the leader apply worker and the transaction has been
1874 * serialized to file.
1875 */
1876 stream_abort_internal(xid, subxid);
1877
1878 elog(DEBUG1, "finished processing the STREAM ABORT command");
1879 break;
1880
1881 case TRANS_LEADER_SEND_TO_PARALLEL:
1882 Assert(winfo);
1883
1884 /*
1885 * For the case of aborting the subtransaction, we increment the
1886 * number of streaming blocks and take the lock again before
1887 * sending the STREAM_ABORT to ensure that the parallel apply
1888 * worker will wait on the lock for the next set of changes after
1889 * processing the STREAM_ABORT message if it is not already
1890 * waiting for STREAM_STOP message.
1891 *
1892 * It is important to perform this locking before sending the
1893 * STREAM_ABORT message so that the leader can hold the lock first
1894 * and the parallel apply worker will wait for the leader to
1895 * release the lock. This is the same as what we do in
1896 * apply_handle_stream_stop. See Locking Considerations atop
1897 * applyparallelworker.c.
1898 */
1899 if (!toplevel_xact)
1900 {
1901 pa_unlock_stream(xid, AccessExclusiveLock);
1902 pg_atomic_add_fetch_u32(&winfo->shared->pending_stream_count, 1);
1903 pa_lock_stream(xid, AccessExclusiveLock);
1904 }
1905
1906 if (pa_send_data(winfo, s->len, s->data))
1907 {
1908 /*
1909 * Unlike STREAM_COMMIT and STREAM_PREPARE, we don't need to
1910 * wait here for the parallel apply worker to finish as that
1911 * is not required to maintain the commit order and won't have
1912 * the risk of failures due to transaction dependencies and
1913 * deadlocks. However, it is possible that before the parallel
1914 * worker finishes and we clear the worker info, the xid
1915 * wraparound happens on the upstream and a new transaction
1916 * with the same xid can appear and that can lead to duplicate
1917 * entries in ParallelApplyTxnHash. Yet another problem could
1918 * be that we may have serialized the changes in partial
1919 * serialize mode and the file containing xact changes may
1920 * already exist, and after xid wraparound trying to create
1921 * the file for the same xid can lead to an error. To avoid
1922 * these problems, we decide to wait for the aborts to finish.
1923 *
1924 * Note, it is okay to not update the flush location position
1925 * for aborts as in worst case that means such a transaction
1926 * won't be sent again after restart.
1927 */
1928 if (toplevel_xact)
1929 pa_xact_finish(winfo, InvalidXLogRecPtr);
1930
1931 break;
1932 }
1933
1934 /*
1935 * Switch to serialize mode when we are not able to send the
1936 * change to parallel apply worker.
1937 */
1938 pa_switch_to_partial_serialize(winfo, true);
1939
1940 /* fall through */
1941 case TRANS_LEADER_PARTIAL_SERIALIZE:
1942 Assert(winfo);
1943
1944 /*
1945 * Parallel apply worker might have applied some changes, so write
1946 * the STREAM_ABORT message so that it can rollback the
1947 * subtransaction if needed.
1948 */
1949 stream_open_and_write_change(xid, LOGICAL_REP_MSG_STREAM_ABORT,
1950 &original_msg);
1951
1952 if (toplevel_xact)
1953 {
1954 pa_set_fileset_state(winfo->shared, FS_SERIALIZE_DONE);
1955 pa_xact_finish(winfo, InvalidXLogRecPtr);
1956 }
1957 break;
1958
1959 case TRANS_PARALLEL_APPLY:
1960
1961 /*
1962 * If the parallel apply worker is applying spooled messages then
1963 * close the file before aborting.
1964 */
1965 if (toplevel_xact && stream_fd)
1966 stream_close_file();
1967
1968 pa_stream_abort(&abort_data);
1969
1970 /*
1971 * We need to wait after processing rollback to savepoint for the
1972 * next set of changes.
1973 *
1974 * We have a race condition here due to which we can start waiting
1975 * here when there are more chunk of streams in the queue. See
1976 * apply_handle_stream_stop.
1977 */
1978 if (!toplevel_xact)
1979 pa_decr_and_wait_stream_block();
1980
1981 elog(DEBUG1, "finished processing the STREAM ABORT command");
1982 break;
1983
1984 default:
1985 elog(ERROR, "unexpected apply action: %d", (int) apply_action);
1986 break;
1987 }
1988
1989 reset_apply_error_context_info();
1990 }
1991
1992 /*
1993 * Ensure that the passed location is fileset's end.
1994 */
1995 static void
1996 ensure_last_message(FileSet *stream_fileset, TransactionId xid, int fileno,
1997 off_t offset)
1998 {
1999 char path[MAXPGPATH];
2000 BufFile *fd;
2001 int last_fileno;
2002 off_t last_offset;
2003
2004 Assert(!IsTransactionState());
2005
2006 begin_replication_step();
2007
2008 changes_filename(path, MyLogicalRepWorker->subid, xid);
2009
2010 fd = BufFileOpenFileSet(stream_fileset, path, O_RDONLY, false);
2011
2012 BufFileSeek(fd, 0, 0, SEEK_END);
2013 BufFileTell(fd, &last_fileno, &last_offset);
2014
2015 BufFileClose(fd);
2016
2017 end_replication_step();
2018
2019 if (last_fileno != fileno || last_offset != offset)
2020 elog(ERROR, "unexpected message left in streaming transaction's changes file \"%s\"",
2021 path);
2022 }
2023
2024 /*
2025 * Common spoolfile processing.
2026 */
2027 void
2028 apply_spooled_messages(FileSet *stream_fileset, TransactionId xid,
2029 XLogRecPtr lsn)
2030 {
2031 int nchanges;
2032 char path[MAXPGPATH];
2033 char *buffer = NULL;
2034 MemoryContext oldcxt;
2035 ResourceOwner oldowner;
2036 int fileno;
2037 off_t offset;
2038
2039 if (!am_parallel_apply_worker())
2040 maybe_start_skipping_changes(lsn);
2041
2042 /* Make sure we have an open transaction */
2043 begin_replication_step();
2044
2045 /*
2046 * Allocate file handle and memory required to process all the messages in
2047 * TopTransactionContext to avoid them getting reset after each message is
2048 * processed.
2049 */
2050 oldcxt = MemoryContextSwitchTo(TopTransactionContext);
2051
2052 /* Open the spool file for the committed/prepared transaction */
2053 changes_filename(path, MyLogicalRepWorker->subid, xid);
2054 elog(DEBUG1, "replaying changes from file \"%s\"", path);
2055
2056 /*
2057 * Make sure the file is owned by the toplevel transaction so that the
2058 * file will not be accidentally closed when aborting a subtransaction.
2059 */
2060 oldowner = CurrentResourceOwner;
2061 CurrentResourceOwner = TopTransactionResourceOwner;
2062
2063 stream_fd = BufFileOpenFileSet(stream_fileset, path, O_RDONLY, false);
2064
2065 CurrentResourceOwner = oldowner;
2066
2067 buffer = palloc(BLCKSZ);
2068
2069 MemoryContextSwitchTo(oldcxt);
2070
2071 remote_final_lsn = lsn;
2072
2073 /*
2074 * Make sure the handle apply_dispatch methods are aware we're in a remote
2075 * transaction.
2076 */
2077 in_remote_transaction = true;
2078 pgstat_report_activity(STATE_RUNNING, NULL);
2079
2080 end_replication_step();
2081
2082 /*
2083 * Read the entries one by one and pass them through the same logic as in
2084 * apply_dispatch.
2085 */
2086 nchanges = 0;
2087 while (true)
2088 {
2089 StringInfoData s2;
2090 size_t nbytes;
2091 int len;
2092
2093 CHECK_FOR_INTERRUPTS();
2094
2095 /* read length of the on-disk record */
2096 nbytes = BufFileReadMaybeEOF(stream_fd, &len, sizeof(len), true);
2097
2098 /* have we reached end of the file? */
2099 if (nbytes == 0)
2100 break;
2101
2102 /* do we have a correct length? */
2103 if (len <= 0)
2104 elog(ERROR, "incorrect length %d in streaming transaction's changes file \"%s\"",
2105 len, path);
2106
2107 /* make sure we have sufficiently large buffer */
2108 buffer = repalloc(buffer, len);
2109
2110 /* and finally read the data into the buffer */
2111 BufFileReadExact(stream_fd, buffer, len);
2112
2113 BufFileTell(stream_fd, &fileno, &offset);
2114
2115 /* init a stringinfo using the buffer and call apply_dispatch */
2116 initReadOnlyStringInfo(&s2, buffer, len);
2117
2118 /* Ensure we are reading the data into our memory context. */
2119 oldcxt = MemoryContextSwitchTo(ApplyMessageContext);
2120
2121 apply_dispatch(&s2);
2122
2123 MemoryContextReset(ApplyMessageContext);
2124
2125 MemoryContextSwitchTo(oldcxt);
2126
2127 nchanges++;
2128
2129 /*
2130 * It is possible the file has been closed because we have processed
2131 * the transaction end message like stream_commit in which case that
2132 * must be the last message.
2133 */
2134 if (!stream_fd)
2135 {
2136 ensure_last_message(stream_fileset, xid, fileno, offset);
2137 break;
2138 }
2139
2140 if (nchanges % 1000 == 0)
2141 elog(DEBUG1, "replayed %d changes from file \"%s\"",
2142 nchanges, path);
2143 }
2144
2145 if (stream_fd)
2146 stream_close_file();
2147
2148 elog(DEBUG1, "replayed %d (all) changes from file \"%s\"",
2149 nchanges, path);
2150
2151 return;
2152 }
2153
2154 /*
2155 * Handle STREAM COMMIT message.
2156 */
2157 static void
2158 apply_handle_stream_commit(StringInfo s)
2159 {
2160 TransactionId xid;
2161 LogicalRepCommitData commit_data;
2162 ParallelApplyWorkerInfo *winfo;
2163 TransApplyAction apply_action;
2164
2165 /* Save the message before it is consumed. */
2166 StringInfoData original_msg = *s;
2167
2168 if (in_streamed_transaction)
2169 ereport(ERROR,
2170 (errcode(ERRCODE_PROTOCOL_VIOLATION),
2171 errmsg_internal("STREAM COMMIT message without STREAM STOP")));
2172
2173 xid = logicalrep_read_stream_commit(s, &commit_data);
2174 set_apply_error_context_xact(xid, commit_data.commit_lsn);
2175
2176 apply_action = get_transaction_apply_action(xid, &winfo);
2177
2178 switch (apply_action)
2179 {
2180 case TRANS_LEADER_APPLY:
2181
2182 /*
2183 * The transaction has been serialized to file, so replay all the
2184 * spooled operations.
2185 */
2186 apply_spooled_messages(MyLogicalRepWorker->stream_fileset, xid,
2187 commit_data.commit_lsn);
2188
2189 apply_handle_commit_internal(&commit_data);
2190
2191 /* Unlink the files with serialized changes and subxact info. */
2192 stream_cleanup_files(MyLogicalRepWorker->subid, xid);
2193
2194 elog(DEBUG1, "finished processing the STREAM COMMIT command");
2195 break;
2196
2197 case TRANS_LEADER_SEND_TO_PARALLEL:
2198 Assert(winfo);
2199
2200 if (pa_send_data(winfo, s->len, s->data))
2201 {
2202 /* Finish processing the streaming transaction. */
2203 pa_xact_finish(winfo, commit_data.end_lsn);
2204 break;
2205 }
2206
2207 /*
2208 * Switch to serialize mode when we are not able to send the
2209 * change to parallel apply worker.
2210 */
2211 pa_switch_to_partial_serialize(winfo, true);
2212
2213 /* fall through */
2214 case TRANS_LEADER_PARTIAL_SERIALIZE:
2215 Assert(winfo);
2216
2217 stream_open_and_write_change(xid, LOGICAL_REP_MSG_STREAM_COMMIT,
2218 &original_msg);
2219
2220 pa_set_fileset_state(winfo->shared, FS_SERIALIZE_DONE);
2221
2222 /* Finish processing the streaming transaction. */
2223 pa_xact_finish(winfo, commit_data.end_lsn);
2224 break;
2225
2226 case TRANS_PARALLEL_APPLY:
2227
2228 /*
2229 * If the parallel apply worker is applying spooled messages then
2230 * close the file before committing.
2231 */
2232 if (stream_fd)
2233 stream_close_file();
2234
2235 apply_handle_commit_internal(&commit_data);
2236
2237 MyParallelShared->last_commit_end = XactLastCommitEnd;
2238
2239 /*
2240 * It is important to set the transaction state as finished before
2241 * releasing the lock. See pa_wait_for_xact_finish.
2242 */
2243 pa_set_xact_state(MyParallelShared, PARALLEL_TRANS_FINISHED);
2244 pa_unlock_transaction(xid, AccessExclusiveLock);
2245
2246 pa_reset_subtrans();
2247
2248 elog(DEBUG1, "finished processing the STREAM COMMIT command");
2249 break;
2250
2251 default:
2252 elog(ERROR, "unexpected apply action: %d", (int) apply_action);
2253 break;
2254 }
2255
2256 /* Process any tables that are being synchronized in parallel. */
2257 process_syncing_tables(commit_data.end_lsn);
2258
2259 pgstat_report_activity(STATE_IDLE, NULL);
2260
2261 reset_apply_error_context_info();
2262 }
2263
2264 /*
2265 * Helper function for apply_handle_commit and apply_handle_stream_commit.
2266 */
2267 static void
2268 apply_handle_commit_internal(LogicalRepCommitData *commit_data)
2269 {
2270 if (is_skipping_changes())
2271 {
2272 stop_skipping_changes();
2273
2274 /*
2275 * Start a new transaction to clear the subskiplsn, if not started
2276 * yet.
2277 */
2278 if (!IsTransactionState())
2279 StartTransactionCommand();
2280 }
2281
2282 if (IsTransactionState())
2283 {
2284 /*
2285 * The transaction is either non-empty or skipped, so we clear the
2286 * subskiplsn.
2287 */
2288 clear_subscription_skip_lsn(commit_data->commit_lsn);
2289
2290 /*
2291 * Update origin state so we can restart streaming from correct
2292 * position in case of crash.
2293 */
2294 replorigin_session_origin_lsn = commit_data->end_lsn;
2295 replorigin_session_origin_timestamp = commit_data->committime;
2296
2297 CommitTransactionCommand();
2298
2299 if (IsTransactionBlock())
2300 {
2301 EndTransactionBlock(false);
2302 CommitTransactionCommand();
2303 }
2304
2305 pgstat_report_stat(false);
2306
2307 store_flush_position(commit_data->end_lsn, XactLastCommitEnd);
2308 }
2309 else
2310 {
2311 /* Process any invalidation messages that might have accumulated. */
2312 AcceptInvalidationMessages();
2313 maybe_reread_subscription();
2314 }
2315
2316 in_remote_transaction = false;
2317 }
2318
2319 /*
2320 * Handle RELATION message.
2321 *
2322 * Note we don't do validation against local schema here. The validation
2323 * against local schema is postponed until first change for given relation
2324 * comes as we only care about it when applying changes for it anyway and we
2325 * do less locking this way.
2326 */
2327 static void
2328 apply_handle_relation(StringInfo s)
2329 {
2330 LogicalRepRelation *rel;
2331
2332 if (handle_streamed_transaction(LOGICAL_REP_MSG_RELATION, s))
2333 return;
2334
2335 rel = logicalrep_read_rel(s);
2336 logicalrep_relmap_update(rel);
2337
2338 /* Also reset all entries in the partition map that refer to remoterel. */
2339 logicalrep_partmap_reset_relmap(rel);
2340 }
2341
2342 /*
2343 * Handle TYPE message.
2344 *
2345 * This implementation pays no attention to TYPE messages; we expect the user
2346 * to have set things up so that the incoming data is acceptable to the input
2347 * functions for the locally subscribed tables. Hence, we just read and
2348 * discard the message.
2349 */
2350 static void
2351 apply_handle_type(StringInfo s)
2352 {
2353 LogicalRepTyp typ;
2354
2355 if (handle_streamed_transaction(LOGICAL_REP_MSG_TYPE, s))
2356 return;
2357
2358 logicalrep_read_typ(s, &typ);
2359 }
2360
2361 /*
2362 * Check that we (the subscription owner) have sufficient privileges on the
2363 * target relation to perform the given operation.
2364 */
2365 static void
2366 TargetPrivilegesCheck(Relation rel, AclMode mode)
2367 {
2368 Oid relid;
2369 AclResult aclresult;
2370
2371 relid = RelationGetRelid(rel);
2372 aclresult = pg_class_aclcheck(relid, GetUserId(), mode);
2373 if (aclresult != ACLCHECK_OK)
2374 aclcheck_error(aclresult,
2375 get_relkind_objtype(rel->rd_rel->relkind),
2376 get_rel_name(relid));
2377
2378 /*
2379 * We lack the infrastructure to honor RLS policies. It might be possible
2380 * to add such infrastructure here, but tablesync workers lack it, too, so
2381 * we don't bother. RLS does not ordinarily apply to TRUNCATE commands,
2382 * but it seems dangerous to replicate a TRUNCATE and then refuse to
2383 * replicate subsequent INSERTs, so we forbid all commands the same.
2384 */
2385 if (check_enable_rls(relid, InvalidOid, false) == RLS_ENABLED)
2386 ereport(ERROR,
2387 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2388 errmsg("user \"%s\" cannot replicate into relation with row-level security enabled: \"%s\"",
2389 GetUserNameFromId(GetUserId(), true),
2390 RelationGetRelationName(rel))));
2391 }
2392
2393 /*
2394 * Handle INSERT message.
2395 */
2396
2397 static void
2398 apply_handle_insert(StringInfo s)
2399 {
2400 LogicalRepRelMapEntry *rel;
2401 LogicalRepTupleData newtup;
2402 LogicalRepRelId relid;
2403 UserContext ucxt;
2404 ApplyExecutionData *edata;
2405 EState *estate;
2406 TupleTableSlot *remoteslot;
2407 MemoryContext oldctx;
2408 bool run_as_owner;
2409
2410 /*
2411 * Quick return if we are skipping data modification changes or handling
2412 * streamed transactions.
2413 */
2414 if (is_skipping_changes() ||
2415 handle_streamed_transaction(LOGICAL_REP_MSG_INSERT, s))
2416 return;
2417
2418 begin_replication_step();
2419
2420 relid = logicalrep_read_insert(s, &newtup);
2421 rel = logicalrep_rel_open(relid, RowExclusiveLock);
2422 if (!should_apply_changes_for_rel(rel))
2423 {
2424 /*
2425 * The relation can't become interesting in the middle of the
2426 * transaction so it's safe to unlock it.
2427 */
2428 logicalrep_rel_close(rel, RowExclusiveLock);
2429 end_replication_step();
2430 return;
2431 }
2432
2433 /*
2434 * Make sure that any user-supplied code runs as the table owner, unless
2435 * the user has opted out of that behavior.
2436 */
2437 run_as_owner = MySubscription->runasowner;
2438 if (!run_as_owner)
2439 SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt);
2440
2441 /* Set relation for error callback */
2442 apply_error_callback_arg.rel = rel;
2443
2444 /* Initialize the executor state. */
2445 edata = create_edata_for_relation(rel);
2446 estate = edata->estate;
2447 remoteslot = ExecInitExtraTupleSlot(estate,
2448 RelationGetDescr(rel->localrel),
2449 &TTSOpsVirtual);
2450
2451 /* Process and store remote tuple in the slot */
2452 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2453 slot_store_data(remoteslot, rel, &newtup);
2454 slot_fill_defaults(rel, estate, remoteslot);
2455 MemoryContextSwitchTo(oldctx);
2456
2457 /* For a partitioned table, insert the tuple into a partition. */
2458 if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2459 apply_handle_tuple_routing(edata,
2460 remoteslot, NULL, CMD_INSERT);
2461 else
2462 {
2463 ResultRelInfo *relinfo = edata->targetRelInfo;
2464
2465 ExecOpenIndices(relinfo, false);
2466 apply_handle_insert_internal(edata, relinfo, remoteslot);
2467 ExecCloseIndices(relinfo);
2468 }
2469
2470 finish_edata(edata);
2471
2472 /* Reset relation for error callback */
2473 apply_error_callback_arg.rel = NULL;
2474
2475 if (!run_as_owner)
2476 RestoreUserContext(&ucxt);
2477
2478 logicalrep_rel_close(rel, NoLock);
2479
2480 end_replication_step();
2481 }
2482
2483 /*
2484 * Workhorse for apply_handle_insert()
2485 * relinfo is for the relation we're actually inserting into
2486 * (could be a child partition of edata->targetRelInfo)
2487 */
2488 static void
2489 apply_handle_insert_internal(ApplyExecutionData *edata,
2490 ResultRelInfo *relinfo,
2491 TupleTableSlot *remoteslot)
2492 {
2493 EState *estate = edata->estate;
2494
2495 /* Caller should have opened indexes already. */
2496 Assert(relinfo->ri_IndexRelationDescs != NULL ||
2497 !relinfo->ri_RelationDesc->rd_rel->relhasindex ||
2498 RelationGetIndexList(relinfo->ri_RelationDesc) == NIL);
2499
2500 /* Caller will not have done this bit. */
2501 Assert(relinfo->ri_onConflictArbiterIndexes == NIL);
2502 InitConflictIndexes(relinfo);
2503
2504 /* Do the insert. */
2505 TargetPrivilegesCheck(relinfo->ri_RelationDesc, ACL_INSERT);
2506 ExecSimpleRelationInsert(relinfo, estate, remoteslot);
2507 }
2508
2509 /*
2510 * Check if the logical replication relation is updatable and throw
2511 * appropriate error if it isn't.
2512 */
2513 static void
2514 check_relation_updatable(LogicalRepRelMapEntry *rel)
2515 {
2516 /*
2517 * For partitioned tables, we only need to care if the target partition is
2518 * updatable (aka has PK or RI defined for it).
2519 */
2520 if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2521 return;
2522
2523 /* Updatable, no error. */
2524 if (rel->updatable)
2525 return;
2526
2527 /*
2528 * We are in error mode so it's fine this is somewhat slow. It's better to
2529 * give user correct error.
2530 */
2531 if (OidIsValid(GetRelationIdentityOrPK(rel->localrel)))
2532 {
2533 ereport(ERROR,
2534 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2535 errmsg("publisher did not send replica identity column "
2536 "expected by the logical replication target relation \"%s.%s\"",
2537 rel->remoterel.nspname, rel->remoterel.relname)));
2538 }
2539
2540 ereport(ERROR,
2541 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
2542 errmsg("logical replication target relation \"%s.%s\" has "
2543 "neither REPLICA IDENTITY index nor PRIMARY "
2544 "KEY and published relation does not have "
2545 "REPLICA IDENTITY FULL",
2546 rel->remoterel.nspname, rel->remoterel.relname)));
2547 }
2548
2549 /*
2550 * Handle UPDATE message.
2551 *
2552 * TODO: FDW support
2553 */
2554 static void
2555 apply_handle_update(StringInfo s)
2556 {
2557 LogicalRepRelMapEntry *rel;
2558 LogicalRepRelId relid;
2559 UserContext ucxt;
2560 ApplyExecutionData *edata;
2561 EState *estate;
2562 LogicalRepTupleData oldtup;
2563 LogicalRepTupleData newtup;
2564 bool has_oldtup;
2565 TupleTableSlot *remoteslot;
2566 RTEPermissionInfo *target_perminfo;
2567 MemoryContext oldctx;
2568 bool run_as_owner;
2569
2570 /*
2571 * Quick return if we are skipping data modification changes or handling
2572 * streamed transactions.
2573 */
2574 if (is_skipping_changes() ||
2575 handle_streamed_transaction(LOGICAL_REP_MSG_UPDATE, s))
2576 return;
2577
2578 begin_replication_step();
2579
2580 relid = logicalrep_read_update(s, &has_oldtup, &oldtup,
2581 &newtup);
2582 rel = logicalrep_rel_open(relid, RowExclusiveLock);
2583 if (!should_apply_changes_for_rel(rel))
2584 {
2585 /*
2586 * The relation can't become interesting in the middle of the
2587 * transaction so it's safe to unlock it.
2588 */
2589 logicalrep_rel_close(rel, RowExclusiveLock);
2590 end_replication_step();
2591 return;
2592 }
2593
2594 /* Set relation for error callback */
2595 apply_error_callback_arg.rel = rel;
2596
2597 /* Check if we can do the update. */
2598 check_relation_updatable(rel);
2599
2600 /*
2601 * Make sure that any user-supplied code runs as the table owner, unless
2602 * the user has opted out of that behavior.
2603 */
2604 run_as_owner = MySubscription->runasowner;
2605 if (!run_as_owner)
2606 SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt);
2607
2608 /* Initialize the executor state. */
2609 edata = create_edata_for_relation(rel);
2610 estate = edata->estate;
2611 remoteslot = ExecInitExtraTupleSlot(estate,
2612 RelationGetDescr(rel->localrel),
2613 &TTSOpsVirtual);
2614
2615 /*
2616 * Populate updatedCols so that per-column triggers can fire, and so
2617 * executor can correctly pass down indexUnchanged hint. This could
2618 * include more columns than were actually changed on the publisher
2619 * because the logical replication protocol doesn't contain that
2620 * information. But it would for example exclude columns that only exist
2621 * on the subscriber, since we are not touching those.
2622 */
2623 target_perminfo = list_nth(estate->es_rteperminfos, 0);
2624 for (int i = 0; i < remoteslot->tts_tupleDescriptor->natts; i++)
2625 {
2626 Form_pg_attribute att = TupleDescAttr(remoteslot->tts_tupleDescriptor, i);
2627 int remoteattnum = rel->attrmap->attnums[i];
2628
2629 if (!att->attisdropped && remoteattnum >= 0)
2630 {
2631 if (remoteattnum >= newtup.ncols)
2632 ereport(ERROR,
2633 (errcode(ERRCODE_PROTOCOL_VIOLATION),
2634 errmsg("logical replication column %d not found in tuple: only %d column(s) received",
2635 remoteattnum + 1, newtup.ncols)));
2636
2637 if (newtup.colstatus[remoteattnum] != LOGICALREP_COLUMN_UNCHANGED)
2638 target_perminfo->updatedCols =
2639 bms_add_member(target_perminfo->updatedCols,
2640 i + 1 - FirstLowInvalidHeapAttributeNumber);
2641 }
2642 }
2643
2644 /* Build the search tuple. */
2645 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2646 slot_store_data(remoteslot, rel,
2647 has_oldtup ? &oldtup : &newtup);
2648 MemoryContextSwitchTo(oldctx);
2649
2650 /* For a partitioned table, apply update to correct partition. */
2651 if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2652 apply_handle_tuple_routing(edata,
2653 remoteslot, &newtup, CMD_UPDATE);
2654 else
2655 apply_handle_update_internal(edata, edata->targetRelInfo,
2656 remoteslot, &newtup, rel->localindexoid);
2657
2658 finish_edata(edata);
2659
2660 /* Reset relation for error callback */
2661 apply_error_callback_arg.rel = NULL;
2662
2663 if (!run_as_owner)
2664 RestoreUserContext(&ucxt);
2665
2666 logicalrep_rel_close(rel, NoLock);
2667
2668 end_replication_step();
2669 }
2670
2671 /*
2672 * Workhorse for apply_handle_update()
2673 * relinfo is for the relation we're actually updating in
2674 * (could be a child partition of edata->targetRelInfo)
2675 */
2676 static void
2677 apply_handle_update_internal(ApplyExecutionData *edata,
2678 ResultRelInfo *relinfo,
2679 TupleTableSlot *remoteslot,
2680 LogicalRepTupleData *newtup,
2681 Oid localindexoid)
2682 {
2683 EState *estate = edata->estate;
2684 LogicalRepRelMapEntry *relmapentry = edata->targetRel;
2685 Relation localrel = relinfo->ri_RelationDesc;
2686 EPQState epqstate;
2687 TupleTableSlot *localslot = NULL;
2688 ConflictTupleInfo conflicttuple = {0};
2689 bool found;
2690 MemoryContext oldctx;
2691
2692 EvalPlanQualInit(&epqstate, estate, NULL, NIL, -1, NIL);
2693 ExecOpenIndices(relinfo, false);
2694
2695 found = FindReplTupleInLocalRel(edata, localrel,
2696 &relmapentry->remoterel,
2697 localindexoid,
2698 remoteslot, &localslot);
2699
2700 /*
2701 * Tuple found.
2702 *
2703 * Note this will fail if there are other conflicting unique indexes.
2704 */
2705 if (found)
2706 {
2707 /*
2708 * Report the conflict if the tuple was modified by a different
2709 * origin.
2710 */
2711 if (GetTupleTransactionInfo(localslot, &conflicttuple.xmin,
2712 &conflicttuple.origin, &conflicttuple.ts) &&
2713 conflicttuple.origin != replorigin_session_origin)
2714 {
2715 TupleTableSlot *newslot;
2716
2717 /* Store the new tuple for conflict reporting */
2718 newslot = table_slot_create(localrel, &estate->es_tupleTable);
2719 slot_store_data(newslot, relmapentry, newtup);
2720
2721 conflicttuple.slot = localslot;
2722
2723 ReportApplyConflict(estate, relinfo, LOG, CT_UPDATE_ORIGIN_DIFFERS,
2724 remoteslot, newslot,
2725 list_make1(&conflicttuple));
2726 }
2727
2728 /* Process and store remote tuple in the slot */
2729 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2730 slot_modify_data(remoteslot, localslot, relmapentry, newtup);
2731 MemoryContextSwitchTo(oldctx);
2732
2733 EvalPlanQualSetSlot(&epqstate, remoteslot);
2734
2735 InitConflictIndexes(relinfo);
2736
2737 /* Do the actual update. */
2738 TargetPrivilegesCheck(relinfo->ri_RelationDesc, ACL_UPDATE);
2739 ExecSimpleRelationUpdate(relinfo, estate, &epqstate, localslot,
2740 remoteslot);
2741 }
2742 else
2743 {
2744 TupleTableSlot *newslot = localslot;
2745
2746 /* Store the new tuple for conflict reporting */
2747 slot_store_data(newslot, relmapentry, newtup);
2748
2749 /*
2750 * The tuple to be updated could not be found. Do nothing except for
2751 * emitting a log message.
2752 */
2753 ReportApplyConflict(estate, relinfo, LOG, CT_UPDATE_MISSING,
2754 remoteslot, newslot, list_make1(&conflicttuple));
2755 }
2756
2757 /* Cleanup. */
2758 ExecCloseIndices(relinfo);
2759 EvalPlanQualEnd(&epqstate);
2760 }
2761
2762 /*
2763 * Handle DELETE message.
2764 *
2765 * TODO: FDW support
2766 */
2767 static void
2768 apply_handle_delete(StringInfo s)
2769 {
2770 LogicalRepRelMapEntry *rel;
2771 LogicalRepTupleData oldtup;
2772 LogicalRepRelId relid;
2773 UserContext ucxt;
2774 ApplyExecutionData *edata;
2775 EState *estate;
2776 TupleTableSlot *remoteslot;
2777 MemoryContext oldctx;
2778 bool run_as_owner;
2779
2780 /*
2781 * Quick return if we are skipping data modification changes or handling
2782 * streamed transactions.
2783 */
2784 if (is_skipping_changes() ||
2785 handle_streamed_transaction(LOGICAL_REP_MSG_DELETE, s))
2786 return;
2787
2788 begin_replication_step();
2789
2790 relid = logicalrep_read_delete(s, &oldtup);
2791 rel = logicalrep_rel_open(relid, RowExclusiveLock);
2792 if (!should_apply_changes_for_rel(rel))
2793 {
2794 /*
2795 * The relation can't become interesting in the middle of the
2796 * transaction so it's safe to unlock it.
2797 */
2798 logicalrep_rel_close(rel, RowExclusiveLock);
2799 end_replication_step();
2800 return;
2801 }
2802
2803 /* Set relation for error callback */
2804 apply_error_callback_arg.rel = rel;
2805
2806 /* Check if we can do the delete. */
2807 check_relation_updatable(rel);
2808
2809 /*
2810 * Make sure that any user-supplied code runs as the table owner, unless
2811 * the user has opted out of that behavior.
2812 */
2813 run_as_owner = MySubscription->runasowner;
2814 if (!run_as_owner)
2815 SwitchToUntrustedUser(rel->localrel->rd_rel->relowner, &ucxt);
2816
2817 /* Initialize the executor state. */
2818 edata = create_edata_for_relation(rel);
2819 estate = edata->estate;
2820 remoteslot = ExecInitExtraTupleSlot(estate,
2821 RelationGetDescr(rel->localrel),
2822 &TTSOpsVirtual);
2823
2824 /* Build the search tuple. */
2825 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
2826 slot_store_data(remoteslot, rel, &oldtup);
2827 MemoryContextSwitchTo(oldctx);
2828
2829 /* For a partitioned table, apply delete to correct partition. */
2830 if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
2831 apply_handle_tuple_routing(edata,
2832 remoteslot, NULL, CMD_DELETE);
2833 else
2834 {
2835 ResultRelInfo *relinfo = edata->targetRelInfo;
2836
2837 ExecOpenIndices(relinfo, false);
2838 apply_handle_delete_internal(edata, relinfo,
2839 remoteslot, rel->localindexoid);
2840 ExecCloseIndices(relinfo);
2841 }
2842
2843 finish_edata(edata);
2844
2845 /* Reset relation for error callback */
2846 apply_error_callback_arg.rel = NULL;
2847
2848 if (!run_as_owner)
2849 RestoreUserContext(&ucxt);
2850
2851 logicalrep_rel_close(rel, NoLock);
2852
2853 end_replication_step();
2854 }
2855
2856 /*
2857 * Workhorse for apply_handle_delete()
2858 * relinfo is for the relation we're actually deleting from
2859 * (could be a child partition of edata->targetRelInfo)
2860 */
2861 static void
2862 apply_handle_delete_internal(ApplyExecutionData *edata,
2863 ResultRelInfo *relinfo,
2864 TupleTableSlot *remoteslot,
2865 Oid localindexoid)
2866 {
2867 EState *estate = edata->estate;
2868 Relation localrel = relinfo->ri_RelationDesc;
2869 LogicalRepRelation *remoterel = &edata->targetRel->remoterel;
2870 EPQState epqstate;
2871 TupleTableSlot *localslot;
2872 ConflictTupleInfo conflicttuple = {0};
2873 bool found;
2874
2875 EvalPlanQualInit(&epqstate, estate, NULL, NIL, -1, NIL);
2876
2877 /* Caller should have opened indexes already. */
2878 Assert(relinfo->ri_IndexRelationDescs != NULL ||
2879 !localrel->rd_rel->relhasindex ||
2880 RelationGetIndexList(localrel) == NIL);
2881
2882 found = FindReplTupleInLocalRel(edata, localrel, remoterel, localindexoid,
2883 remoteslot, &localslot);
2884
2885 /* If found delete it. */
2886 if (found)
2887 {
2888 /*
2889 * Report the conflict if the tuple was modified by a different
2890 * origin.
2891 */
2892 if (GetTupleTransactionInfo(localslot, &conflicttuple.xmin,
2893 &conflicttuple.origin, &conflicttuple.ts) &&
2894 conflicttuple.origin != replorigin_session_origin)
2895 {
2896 conflicttuple.slot = localslot;
2897 ReportApplyConflict(estate, relinfo, LOG, CT_DELETE_ORIGIN_DIFFERS,
2898 remoteslot, NULL,
2899 list_make1(&conflicttuple));
2900 }
2901
2902 EvalPlanQualSetSlot(&epqstate, localslot);
2903
2904 /* Do the actual delete. */
2905 TargetPrivilegesCheck(relinfo->ri_RelationDesc, ACL_DELETE);
2906 ExecSimpleRelationDelete(relinfo, estate, &epqstate, localslot);
2907 }
2908 else
2909 {
2910 /*
2911 * The tuple to be deleted could not be found. Do nothing except for
2912 * emitting a log message.
2913 */
2914 ReportApplyConflict(estate, relinfo, LOG, CT_DELETE_MISSING,
2915 remoteslot, NULL, list_make1(&conflicttuple));
2916 }
2917
2918 /* Cleanup. */
2919 EvalPlanQualEnd(&epqstate);
2920 }
2921
2922 /*
2923 * Try to find a tuple received from the publication side (in 'remoteslot') in
2924 * the corresponding local relation using either replica identity index,
2925 * primary key, index or if needed, sequential scan.
2926 *
2927 * Local tuple, if found, is returned in '*localslot'.
2928 */
2929 static bool
2930 FindReplTupleInLocalRel(ApplyExecutionData *edata, Relation localrel,
2931 LogicalRepRelation *remoterel,
2932 Oid localidxoid,
2933 TupleTableSlot *remoteslot,
2934 TupleTableSlot **localslot)
2935 {
2936 EState *estate = edata->estate;
2937 bool found;
2938
2939 /*
2940 * Regardless of the top-level operation, we're performing a read here, so
2941 * check for SELECT privileges.
2942 */
2943 TargetPrivilegesCheck(localrel, ACL_SELECT);
2944
2945 *localslot = table_slot_create(localrel, &estate->es_tupleTable);
2946
2947 Assert(OidIsValid(localidxoid) ||
2948 (remoterel->replident == REPLICA_IDENTITY_FULL));
2949
2950 if (OidIsValid(localidxoid))
2951 {
2952 #ifdef USE_ASSERT_CHECKING
2953 Relation idxrel = index_open(localidxoid, AccessShareLock);
2954
2955 /* Index must be PK, RI, or usable for REPLICA IDENTITY FULL tables */
2956 Assert(GetRelationIdentityOrPK(localrel) == localidxoid ||
2957 (remoterel->replident == REPLICA_IDENTITY_FULL &&
2958 IsIndexUsableForReplicaIdentityFull(idxrel,
2959 edata->targetRel->attrmap)));
2960 index_close(idxrel, AccessShareLock);
2961 #endif
2962
2963 found = RelationFindReplTupleByIndex(localrel, localidxoid,
2964 LockTupleExclusive,
2965 remoteslot, *localslot);
2966 }
2967 else
2968 found = RelationFindReplTupleSeq(localrel, LockTupleExclusive,
2969 remoteslot, *localslot);
2970
2971 return found;
2972 }
2973
2974 /*
2975 * This handles insert, update, delete on a partitioned table.
2976 */
2977 static void
2978 apply_handle_tuple_routing(ApplyExecutionData *edata,
2979 TupleTableSlot *remoteslot,
2980 LogicalRepTupleData *newtup,
2981 CmdType operation)
2982 {
2983 EState *estate = edata->estate;
2984 LogicalRepRelMapEntry *relmapentry = edata->targetRel;
2985 ResultRelInfo *relinfo = edata->targetRelInfo;
2986 Relation parentrel = relinfo->ri_RelationDesc;
2987 ModifyTableState *mtstate;
2988 PartitionTupleRouting *proute;
2989 ResultRelInfo *partrelinfo;
2990 Relation partrel;
2991 TupleTableSlot *remoteslot_part;
2992 TupleConversionMap *map;
2993 MemoryContext oldctx;
2994 LogicalRepRelMapEntry *part_entry = NULL;
2995 AttrMap *attrmap = NULL;
2996
2997 /* ModifyTableState is needed for ExecFindPartition(). */
2998 edata->mtstate = mtstate = makeNode(ModifyTableState);
2999 mtstate->ps.plan = NULL;
3000 mtstate->ps.state = estate;
3001 mtstate->operation = operation;
3002 mtstate->resultRelInfo = relinfo;
3003
3004 /* ... as is PartitionTupleRouting. */
3005 edata->proute = proute = ExecSetupPartitionTupleRouting(estate, parentrel);
3006
3007 /*
3008 * Find the partition to which the "search tuple" belongs.
3009 */
3010 Assert(remoteslot != NULL);
3011 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3012 partrelinfo = ExecFindPartition(mtstate, relinfo, proute,
3013 remoteslot, estate);
3014 Assert(partrelinfo != NULL);
3015 partrel = partrelinfo->ri_RelationDesc;
3016
3017 /*
3018 * Check for supported relkind. We need this since partitions might be of
3019 * unsupported relkinds; and the set of partitions can change, so checking
3020 * at CREATE/ALTER SUBSCRIPTION would be insufficient.
3021 */
3022 CheckSubscriptionRelkind(partrel->rd_rel->relkind,
3023 get_namespace_name(RelationGetNamespace(partrel)),
3024 RelationGetRelationName(partrel));
3025
3026 /*
3027 * To perform any of the operations below, the tuple must match the
3028 * partition's rowtype. Convert if needed or just copy, using a dedicated
3029 * slot to store the tuple in any case.
3030 */
3031 remoteslot_part = partrelinfo->ri_PartitionTupleSlot;
3032 if (remoteslot_part == NULL)
3033 remoteslot_part = table_slot_create(partrel, &estate->es_tupleTable);
3034 map = ExecGetRootToChildMap(partrelinfo, estate);
3035 if (map != NULL)
3036 {
3037 attrmap = map->attrMap;
3038 remoteslot_part = execute_attr_map_slot(attrmap, remoteslot,
3039 remoteslot_part);
3040 }
3041 else
3042 {
3043 remoteslot_part = ExecCopySlot(remoteslot_part, remoteslot);
3044 slot_getallattrs(remoteslot_part);
3045 }
3046 MemoryContextSwitchTo(oldctx);
3047
3048 /* Check if we can do the update or delete on the leaf partition. */
3049 if (operation == CMD_UPDATE || operation == CMD_DELETE)
3050 {
3051 part_entry = logicalrep_partition_open(relmapentry, partrel,
3052 attrmap);
3053 check_relation_updatable(part_entry);
3054 }
3055
3056 switch (operation)
3057 {
3058 case CMD_INSERT:
3059 apply_handle_insert_internal(edata, partrelinfo,
3060 remoteslot_part);
3061 break;
3062
3063 case CMD_DELETE:
3064 apply_handle_delete_internal(edata, partrelinfo,
3065 remoteslot_part,
3066 part_entry->localindexoid);
3067 break;
3068
3069 case CMD_UPDATE:
3070
3071 /*
3072 * For UPDATE, depending on whether or not the updated tuple
3073 * satisfies the partition's constraint, perform a simple UPDATE
3074 * of the partition or move the updated tuple into a different
3075 * suitable partition.
3076 */
3077 {
3078 TupleTableSlot *localslot;
3079 ResultRelInfo *partrelinfo_new;
3080 Relation partrel_new;
3081 bool found;
3082 EPQState epqstate;
3083 ConflictTupleInfo conflicttuple = {0};
3084
3085 /* Get the matching local tuple from the partition. */
3086 found = FindReplTupleInLocalRel(edata, partrel,
3087 &part_entry->remoterel,
3088 part_entry->localindexoid,
3089 remoteslot_part, &localslot);
3090 if (!found)
3091 {
3092 TupleTableSlot *newslot = localslot;
3093
3094 /* Store the new tuple for conflict reporting */
3095 slot_store_data(newslot, part_entry, newtup);
3096
3097 /*
3098 * The tuple to be updated could not be found. Do nothing
3099 * except for emitting a log message.
3100 */
3101 ReportApplyConflict(estate, partrelinfo, LOG,
3102 CT_UPDATE_MISSING, remoteslot_part,
3103 newslot, list_make1(&conflicttuple));
3104
3105 return;
3106 }
3107
3108 /*
3109 * Report the conflict if the tuple was modified by a
3110 * different origin.
3111 */
3112 if (GetTupleTransactionInfo(localslot, &conflicttuple.xmin,
3113 &conflicttuple.origin,
3114 &conflicttuple.ts) &&
3115 conflicttuple.origin != replorigin_session_origin)
3116 {
3117 TupleTableSlot *newslot;
3118
3119 /* Store the new tuple for conflict reporting */
3120 newslot = table_slot_create(partrel, &estate->es_tupleTable);
3121 slot_store_data(newslot, part_entry, newtup);
3122
3123 conflicttuple.slot = localslot;
3124
3125 ReportApplyConflict(estate, partrelinfo, LOG, CT_UPDATE_ORIGIN_DIFFERS,
3126 remoteslot_part, newslot,
3127 list_make1(&conflicttuple));
3128 }
3129
3130 /*
3131 * Apply the update to the local tuple, putting the result in
3132 * remoteslot_part.
3133 */
3134 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3135 slot_modify_data(remoteslot_part, localslot, part_entry,
3136 newtup);
3137 MemoryContextSwitchTo(oldctx);
3138
3139 EvalPlanQualInit(&epqstate, estate, NULL, NIL, -1, NIL);
3140
3141 /*
3142 * Does the updated tuple still satisfy the current
3143 * partition's constraint?
3144 */
3145 if (!partrel->rd_rel->relispartition ||
3146 ExecPartitionCheck(partrelinfo, remoteslot_part, estate,
3147 false))
3148 {
3149 /*
3150 * Yes, so simply UPDATE the partition. We don't call
3151 * apply_handle_update_internal() here, which would
3152 * normally do the following work, to avoid repeating some
3153 * work already done above to find the local tuple in the
3154 * partition.
3155 */
3156 InitConflictIndexes(partrelinfo);
3157
3158 EvalPlanQualSetSlot(&epqstate, remoteslot_part);
3159 TargetPrivilegesCheck(partrelinfo->ri_RelationDesc,
3160 ACL_UPDATE);
3161 ExecSimpleRelationUpdate(partrelinfo, estate, &epqstate,
3162 localslot, remoteslot_part);
3163 }
3164 else
3165 {
3166 /* Move the tuple into the new partition. */
3167
3168 /*
3169 * New partition will be found using tuple routing, which
3170 * can only occur via the parent table. We might need to
3171 * convert the tuple to the parent's rowtype. Note that
3172 * this is the tuple found in the partition, not the
3173 * original search tuple received by this function.
3174 */
3175 if (map)
3176 {
3177 TupleConversionMap *PartitionToRootMap =
3178 convert_tuples_by_name(RelationGetDescr(partrel),
3179 RelationGetDescr(parentrel));
3180
3181 remoteslot =
3182 execute_attr_map_slot(PartitionToRootMap->attrMap,
3183 remoteslot_part, remoteslot);
3184 }
3185 else
3186 {
3187 remoteslot = ExecCopySlot(remoteslot, remoteslot_part);
3188 slot_getallattrs(remoteslot);
3189 }
3190
3191 /* Find the new partition. */
3192 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3193 partrelinfo_new = ExecFindPartition(mtstate, relinfo,
3194 proute, remoteslot,
3195 estate);
3196 MemoryContextSwitchTo(oldctx);
3197 Assert(partrelinfo_new != partrelinfo);
3198 partrel_new = partrelinfo_new->ri_RelationDesc;
3199
3200 /* Check that new partition also has supported relkind. */
3201 CheckSubscriptionRelkind(partrel_new->rd_rel->relkind,
3202 get_namespace_name(RelationGetNamespace(partrel_new)),
3203 RelationGetRelationName(partrel_new));
3204
3205 /* DELETE old tuple found in the old partition. */
3206 EvalPlanQualSetSlot(&epqstate, localslot);
3207 TargetPrivilegesCheck(partrelinfo->ri_RelationDesc, ACL_DELETE);
3208 ExecSimpleRelationDelete(partrelinfo, estate, &epqstate, localslot);
3209
3210 /* INSERT new tuple into the new partition. */
3211
3212 /*
3213 * Convert the replacement tuple to match the destination
3214 * partition rowtype.
3215 */
3216 oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
3217 remoteslot_part = partrelinfo_new->ri_PartitionTupleSlot;
3218 if (remoteslot_part == NULL)
3219 remoteslot_part = table_slot_create(partrel_new,
3220 &estate->es_tupleTable);
3221 map = ExecGetRootToChildMap(partrelinfo_new, estate);
3222 if (map != NULL)
3223 {
3224 remoteslot_part = execute_attr_map_slot(map->attrMap,
3225 remoteslot,
3226 remoteslot_part);
3227 }
3228 else
3229 {
3230 remoteslot_part = ExecCopySlot(remoteslot_part,
3231 remoteslot);
3232 slot_getallattrs(remoteslot);
3233 }
3234 MemoryContextSwitchTo(oldctx);
3235 apply_handle_insert_internal(edata, partrelinfo_new,
3236 remoteslot_part);
3237 }
3238
3239 EvalPlanQualEnd(&epqstate);
3240 }
3241 break;
3242
3243 default:
3244 elog(ERROR, "unrecognized CmdType: %d", (int) operation);
3245 break;
3246 }
3247 }
3248
3249 /*
3250 * Handle TRUNCATE message.
3251 *
3252 * TODO: FDW support
3253 */
3254 static void
3255 apply_handle_truncate(StringInfo s)
3256 {
3257 bool cascade = false;
3258 bool restart_seqs = false;
3259 List *remote_relids = NIL;
3260 List *remote_rels = NIL;
3261 List *rels = NIL;
3262 List *part_rels = NIL;
3263 List *relids = NIL;
3264 List *relids_logged = NIL;
3265 ListCell *lc;
3266 LOCKMODE lockmode = AccessExclusiveLock;
3267
3268 /*
3269 * Quick return if we are skipping data modification changes or handling
3270 * streamed transactions.
3271 */
3272 if (is_skipping_changes() ||
3273 handle_streamed_transaction(LOGICAL_REP_MSG_TRUNCATE, s))
3274 return;
3275
3276 begin_replication_step();
3277
3278 remote_relids = logicalrep_read_truncate(s, &cascade, &restart_seqs);
3279
3280 foreach(lc, remote_relids)
3281 {
3282 LogicalRepRelId relid = lfirst_oid(lc);
3283 LogicalRepRelMapEntry *rel;
3284
3285 rel = logicalrep_rel_open(relid, lockmode);
3286 if (!should_apply_changes_for_rel(rel))
3287 {
3288 /*
3289 * The relation can't become interesting in the middle of the
3290 * transaction so it's safe to unlock it.
3291 */
3292 logicalrep_rel_close(rel, lockmode);
3293 continue;
3294 }
3295
3296 remote_rels = lappend(remote_rels, rel);
3297 TargetPrivilegesCheck(rel->localrel, ACL_TRUNCATE);
3298 rels = lappend(rels, rel->localrel);
3299 relids = lappend_oid(relids, rel->localreloid);
3300 if (RelationIsLogicallyLogged(rel->localrel))
3301 relids_logged = lappend_oid(relids_logged, rel->localreloid);
3302
3303 /*
3304 * Truncate partitions if we got a message to truncate a partitioned
3305 * table.
3306 */
3307 if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
3308 {
3309 ListCell *child;
3310 List *children = find_all_inheritors(rel->localreloid,
3311 lockmode,
3312 NULL);
3313
3314 foreach(child, children)
3315 {
3316 Oid childrelid = lfirst_oid(child);
3317 Relation childrel;
3318
3319 if (list_member_oid(relids, childrelid))
3320 continue;
3321
3322 /* find_all_inheritors already got lock */
3323 childrel = table_open(childrelid, NoLock);
3324
3325 /*
3326 * Ignore temp tables of other backends. See similar code in
3327 * ExecuteTruncate().
3328 */
3329 if (RELATION_IS_OTHER_TEMP(childrel))
3330 {
3331 table_close(childrel, lockmode);
3332 continue;
3333 }
3334
3335 TargetPrivilegesCheck(childrel, ACL_TRUNCATE);
3336 rels = lappend(rels, childrel);
3337 part_rels = lappend(part_rels, childrel);
3338 relids = lappend_oid(relids, childrelid);
3339 /* Log this relation only if needed for logical decoding */
3340 if (RelationIsLogicallyLogged(childrel))
3341 relids_logged = lappend_oid(relids_logged, childrelid);
3342 }
3343 }
3344 }
3345
3346 /*
3347 * Even if we used CASCADE on the upstream primary we explicitly default
3348 * to replaying changes without further cascading. This might be later
3349 * changeable with a user specified option.
3350 *
3351 * MySubscription->runasowner tells us whether we want to execute
3352 * replication actions as the subscription owner; the last argument to
3353 * TruncateGuts tells it whether we want to switch to the table owner.
3354 * Those are exactly opposite conditions.
3355 */
3356 ExecuteTruncateGuts(rels,
3357 relids,
3358 relids_logged,
3359 DROP_RESTRICT,
3360 restart_seqs,
3361 !MySubscription->runasowner);
3362 foreach(lc, remote_rels)
3363 {
3364 LogicalRepRelMapEntry *rel = lfirst(lc);
3365
3366 logicalrep_rel_close(rel, NoLock);
3367 }
3368 foreach(lc, part_rels)
3369 {
3370 Relation rel = lfirst(lc);
3371
3372 table_close(rel, NoLock);
3373 }
3374
3375 end_replication_step();
3376 }
3377
3378
3379 /*
3380 * Logical replication protocol message dispatcher.
3381 */
3382 void
3383 apply_dispatch(StringInfo s)
3384 {
3385 LogicalRepMsgType action = pq_getmsgbyte(s);
3386 LogicalRepMsgType saved_command;
3387
3388 /*
3389 * Set the current command being applied. Since this function can be
3390 * called recursively when applying spooled changes, save the current
3391 * command.
3392 */
3393 saved_command = apply_error_callback_arg.command;
3394 apply_error_callback_arg.command = action;
3395
3396 switch (action)
3397 {
3398 case LOGICAL_REP_MSG_BEGIN:
3399 apply_handle_begin(s);
3400 break;
3401
3402 case LOGICAL_REP_MSG_COMMIT:
3403 apply_handle_commit(s);
3404 break;
3405
3406 case LOGICAL_REP_MSG_INSERT:
3407 apply_handle_insert(s);
3408 break;
3409
3410 case LOGICAL_REP_MSG_UPDATE:
3411 apply_handle_update(s);
3412 break;
3413
3414 case LOGICAL_REP_MSG_DELETE:
3415 apply_handle_delete(s);
3416 break;
3417
3418 case LOGICAL_REP_MSG_TRUNCATE:
3419 apply_handle_truncate(s);
3420 break;
3421
3422 case LOGICAL_REP_MSG_RELATION:
3423 apply_handle_relation(s);
3424 break;
3425
3426 case LOGICAL_REP_MSG_TYPE:
3427 apply_handle_type(s);
3428 break;
3429
3430 case LOGICAL_REP_MSG_ORIGIN:
3431 apply_handle_origin(s);
3432 break;
3433
3434 case LOGICAL_REP_MSG_MESSAGE:
3435
3436 /*
3437 * Logical replication does not use generic logical messages yet.
3438 * Although, it could be used by other applications that use this
3439 * output plugin.
3440 */
3441 break;
3442
3443 case LOGICAL_REP_MSG_STREAM_START:
3444 apply_handle_stream_start(s);
3445 break;
3446
3447 case LOGICAL_REP_MSG_STREAM_STOP:
3448 apply_handle_stream_stop(s);
3449 break;
3450
3451 case LOGICAL_REP_MSG_STREAM_ABORT:
3452 apply_handle_stream_abort(s);
3453 break;
3454
3455 case LOGICAL_REP_MSG_STREAM_COMMIT:
3456 apply_handle_stream_commit(s);
3457 break;
3458
3459 case LOGICAL_REP_MSG_BEGIN_PREPARE:
3460 apply_handle_begin_prepare(s);
3461 break;
3462
3463 case LOGICAL_REP_MSG_PREPARE:
3464 apply_handle_prepare(s);
3465 break;
3466
3467 case LOGICAL_REP_MSG_COMMIT_PREPARED:
3468 apply_handle_commit_prepared(s);
3469 break;
3470
3471 case LOGICAL_REP_MSG_ROLLBACK_PREPARED:
3472 apply_handle_rollback_prepared(s);
3473 break;
3474
3475 case LOGICAL_REP_MSG_STREAM_PREPARE:
3476 apply_handle_stream_prepare(s);
3477 break;
3478
3479 default:
3480 ereport(ERROR,
3481 (errcode(ERRCODE_PROTOCOL_VIOLATION),
3482 errmsg("invalid logical replication message type \"??? (%d)\"", action)));
3483 }
3484
3485 /* Reset the current command */
3486 apply_error_callback_arg.command = saved_command;
3487 }
3488
3489 /*
3490 * Figure out which write/flush positions to report to the walsender process.
3491 *
3492 * We can't simply report back the last LSN the walsender sent us because the
3493 * local transaction might not yet be flushed to disk locally. Instead we
3494 * build a list that associates local with remote LSNs for every commit. When
3495 * reporting back the flush position to the sender we iterate that list and
3496 * check which entries on it are already locally flushed. Those we can report
3497 * as having been flushed.
3498 *
3499 * The have_pending_txes is true if there are outstanding transactions that
3500 * need to be flushed.
3501 */
3502 static void
3503 get_flush_position(XLogRecPtr *write, XLogRecPtr *flush,
3504 bool *have_pending_txes)
3505 {
3506 dlist_mutable_iter iter;
3507 XLogRecPtr local_flush = GetFlushRecPtr(NULL);
3508
3509 *write = InvalidXLogRecPtr;
3510 *flush = InvalidXLogRecPtr;
3511
3512 dlist_foreach_modify(iter, &lsn_mapping)
3513 {
3514 FlushPosition *pos =
3515 dlist_container(FlushPosition, node, iter.cur);
3516
3517 *write = pos->remote_end;
3518
3519 if (pos->local_end <= local_flush)
3520 {
3521 *flush = pos->remote_end;
3522 dlist_delete(iter.cur);
3523 pfree(pos);
3524 }
3525 else
3526 {
3527 /*
3528 * Don't want to uselessly iterate over the rest of the list which
3529 * could potentially be long. Instead get the last element and
3530 * grab the write position from there.
3531 */
3532 pos = dlist_tail_element(FlushPosition, node,
3533 &lsn_mapping);
3534 *write = pos->remote_end;
3535 *have_pending_txes = true;
3536 return;
3537 }
3538 }
3539
3540 *have_pending_txes = !dlist_is_empty(&lsn_mapping);
3541 }
3542
3543 /*
3544 * Store current remote/local lsn pair in the tracking list.
3545 */
3546 void
3547 store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn)
3548 {
3549 FlushPosition *flushpos;
3550
3551 /*
3552 * Skip for parallel apply workers, because the lsn_mapping is maintained
3553 * by the leader apply worker.
3554 */
3555 if (am_parallel_apply_worker())
3556 return;
3557
3558 /* Need to do this in permanent context */
3559 MemoryContextSwitchTo(ApplyContext);
3560
3561 /* Track commit lsn */
3562 flushpos = (FlushPosition *) palloc(sizeof(FlushPosition));
3563 flushpos->local_end = local_lsn;
3564 flushpos->remote_end = remote_lsn;
3565
3566 dlist_push_tail(&lsn_mapping, &flushpos->node);
3567 MemoryContextSwitchTo(ApplyMessageContext);
3568 }
3569
3570
3571 /* Update statistics of the worker. */
3572 static void
3573 UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time, bool reply)
3574 {
3575 MyLogicalRepWorker->last_lsn = last_lsn;
3576 MyLogicalRepWorker->last_send_time = send_time;
3577 MyLogicalRepWorker->last_recv_time = GetCurrentTimestamp();
3578 if (reply)
3579 {
3580 MyLogicalRepWorker->reply_lsn = last_lsn;
3581 MyLogicalRepWorker->reply_time = send_time;
3582 }
3583 }
3584
3585 /*
3586 * Apply main loop.
3587 */
3588 static void
3589 LogicalRepApplyLoop(XLogRecPtr last_received)
3590 {
3591 TimestampTz last_recv_timestamp = GetCurrentTimestamp();
3592 bool ping_sent = false;
3593 TimeLineID tli;
3594 ErrorContextCallback errcallback;
3595
3596 /*
3597 * Init the ApplyMessageContext which we clean up after each replication
3598 * protocol message.
3599 */
3600 ApplyMessageContext = AllocSetContextCreate(ApplyContext,
3601 "ApplyMessageContext",
3602 ALLOCSET_DEFAULT_SIZES);
3603
3604 /*
3605 * This memory context is used for per-stream data when the streaming mode
3606 * is enabled. This context is reset on each stream stop.
3607 */
3608 LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
3609 "LogicalStreamingContext",
3610 ALLOCSET_DEFAULT_SIZES);
3611
3612 /* mark as idle, before starting to loop */
3613 pgstat_report_activity(STATE_IDLE, NULL);
3614
3615 /*
3616 * Push apply error context callback. Fields will be filled while applying
3617 * a change.
3618 */
3619 errcallback.callback = apply_error_callback;
3620 errcallback.previous = error_context_stack;
3621 error_context_stack = &errcallback;
3622 apply_error_context_stack = error_context_stack;
3623
3624 /* This outer loop iterates once per wait. */
3625 for (;;)
3626 {
3627 pgsocket fd = PGINVALID_SOCKET;
3628 int rc;
3629 int len;
3630 char *buf = NULL;
3631 bool endofstream = false;
3632 long wait_time;
3633
3634 CHECK_FOR_INTERRUPTS();
3635
3636 MemoryContextSwitchTo(ApplyMessageContext);
3637
3638 len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
3639
3640 if (len != 0)
3641 {
3642 /* Loop to process all available data (without blocking). */
3643 for (;;)
3644 {
3645 CHECK_FOR_INTERRUPTS();
3646
3647 if (len == 0)
3648 {
3649 break;
3650 }
3651 else if (len < 0)
3652 {
3653 ereport(LOG,
3654 (errmsg("data stream from publisher has ended")));
3655 endofstream = true;
3656 break;
3657 }
3658 else
3659 {
3660 int c;
3661 StringInfoData s;
3662
3663 if (ConfigReloadPending)
3664 {
3665 ConfigReloadPending = false;
3666 ProcessConfigFile(PGC_SIGHUP);
3667 }
3668
3669 /* Reset timeout. */
3670 last_recv_timestamp = GetCurrentTimestamp();
3671 ping_sent = false;
3672
3673 /* Ensure we are reading the data into our memory context. */
3674 MemoryContextSwitchTo(ApplyMessageContext);
3675
3676 initReadOnlyStringInfo(&s, buf, len);
3677
3678 c = pq_getmsgbyte(&s);
3679
3680 if (c == 'w')
3681 {
3682 XLogRecPtr start_lsn;
3683 XLogRecPtr end_lsn;
3684 TimestampTz send_time;
3685
3686 start_lsn = pq_getmsgint64(&s);
3687 end_lsn = pq_getmsgint64(&s);
3688 send_time = pq_getmsgint64(&s);
3689
3690 if (last_received < start_lsn)
3691 last_received = start_lsn;
3692
3693 if (last_received < end_lsn)
3694 last_received = end_lsn;
3695
3696 UpdateWorkerStats(last_received, send_time, false);
3697
3698 apply_dispatch(&s);
3699 }
3700 else if (c == 'k')
3701 {
3702 XLogRecPtr end_lsn;
3703 TimestampTz timestamp;
3704 bool reply_requested;
3705
3706 end_lsn = pq_getmsgint64(&s);
3707 timestamp = pq_getmsgint64(&s);
3708 reply_requested = pq_getmsgbyte(&s);
3709
3710 if (last_received < end_lsn)
3711 last_received = end_lsn;
3712
3713 send_feedback(last_received, reply_requested, false);
3714 UpdateWorkerStats(last_received, timestamp, true);
3715 }
3716 /* other message types are purposefully ignored */
3717
3718 MemoryContextReset(ApplyMessageContext);
3719 }
3720
3721 len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
3722 }
3723 }
3724
3725 /* confirm all writes so far */
3726 send_feedback(last_received, false, false);
3727
3728 if (!in_remote_transaction && !in_streamed_transaction)
3729 {
3730 /*
3731 * If we didn't get any transactions for a while there might be
3732 * unconsumed invalidation messages in the queue, consume them
3733 * now.
3734 */
3735 AcceptInvalidationMessages();
3736 maybe_reread_subscription();
3737
3738 /* Process any table synchronization changes. */
3739 process_syncing_tables(last_received);
3740 }
3741
3742 /* Cleanup the memory. */
3743 MemoryContextReset(ApplyMessageContext);
3744 MemoryContextSwitchTo(TopMemoryContext);
3745
3746 /* Check if we need to exit the streaming loop. */
3747 if (endofstream)
3748 break;
3749
3750 /*
3751 * Wait for more data or latch. If we have unflushed transactions,
3752 * wake up after WalWriterDelay to see if they've been flushed yet (in
3753 * which case we should send a feedback message). Otherwise, there's
3754 * no particular urgency about waking up unless we get data or a
3755 * signal.
3756 */
3757 if (!dlist_is_empty(&lsn_mapping))
3758 wait_time = WalWriterDelay;
3759 else
3760 wait_time = NAPTIME_PER_CYCLE;
3761
3762 rc = WaitLatchOrSocket(MyLatch,
3763 WL_SOCKET_READABLE | WL_LATCH_SET |
3764 WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
3765 fd, wait_time,
3766 WAIT_EVENT_LOGICAL_APPLY_MAIN);
3767
3768 if (rc & WL_LATCH_SET)
3769 {
3770 ResetLatch(MyLatch);
3771 CHECK_FOR_INTERRUPTS();
3772 }
3773
3774 if (ConfigReloadPending)
3775 {
3776 ConfigReloadPending = false;
3777 ProcessConfigFile(PGC_SIGHUP);
3778 }
3779
3780 if (rc & WL_TIMEOUT)
3781 {
3782 /*
3783 * We didn't receive anything new. If we haven't heard anything
3784 * from the server for more than wal_receiver_timeout / 2, ping
3785 * the server. Also, if it's been longer than
3786 * wal_receiver_status_interval since the last update we sent,
3787 * send a status update to the primary anyway, to report any
3788 * progress in applying WAL.
3789 */
3790 bool requestReply = false;
3791
3792 /*
3793 * Check if time since last receive from primary has reached the
3794 * configured limit.
3795 */
3796 if (wal_receiver_timeout > 0)
3797 {
3798 TimestampTz now = GetCurrentTimestamp();
3799 TimestampTz timeout;
3800
3801 timeout =
3802 TimestampTzPlusMilliseconds(last_recv_timestamp,
3803 wal_receiver_timeout);
3804
3805 if (now >= timeout)
3806 ereport(ERROR,
3807 (errcode(ERRCODE_CONNECTION_FAILURE),
3808 errmsg("terminating logical replication worker due to timeout")));
3809
3810 /* Check to see if it's time for a ping. */
3811 if (!ping_sent)
3812 {
3813 timeout = TimestampTzPlusMilliseconds(last_recv_timestamp,
3814 (wal_receiver_timeout / 2));
3815 if (now >= timeout)
3816 {
3817 requestReply = true;
3818 ping_sent = true;
3819 }
3820 }
3821 }
3822
3823 send_feedback(last_received, requestReply, requestReply);
3824
3825 /*
3826 * Force reporting to ensure long idle periods don't lead to
3827 * arbitrarily delayed stats. Stats can only be reported outside
3828 * of (implicit or explicit) transactions. That shouldn't lead to
3829 * stats being delayed for long, because transactions are either
3830 * sent as a whole on commit or streamed. Streamed transactions
3831 * are spilled to disk and applied on commit.
3832 */
3833 if (!IsTransactionState())
3834 pgstat_report_stat(true);
3835 }
3836 }
3837
3838 /* Pop the error context stack */
3839 error_context_stack = errcallback.previous;
3840 apply_error_context_stack = error_context_stack;
3841
3842 /* All done */
3843 walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
3844 }
3845
3846 /*
3847 * Send a Standby Status Update message to server.
3848 *
3849 * 'recvpos' is the latest LSN we've received data to, force is set if we need
3850 * to send a response to avoid timeouts.
3851 */
3852 static void
3853 send_feedback(XLogRecPtr recvpos, bool force, bool requestReply)
3854 {
3855 static StringInfo reply_message = NULL;
3856 static TimestampTz send_time = 0;
3857
3858 static XLogRecPtr last_recvpos = InvalidXLogRecPtr;
3859 static XLogRecPtr last_writepos = InvalidXLogRecPtr;
3860 static XLogRecPtr last_flushpos = InvalidXLogRecPtr;
3861
3862 XLogRecPtr writepos;
3863 XLogRecPtr flushpos;
3864 TimestampTz now;
3865 bool have_pending_txes;
3866
3867 /*
3868 * If the user doesn't want status to be reported to the publisher, be
3869 * sure to exit before doing anything at all.
3870 */
3871 if (!force && wal_receiver_status_interval <= 0)
3872 return;
3873
3874 /* It's legal to not pass a recvpos */
3875 if (recvpos < last_recvpos)
3876 recvpos = last_recvpos;
3877
3878 get_flush_position(&writepos, &flushpos, &have_pending_txes);
3879
3880 /*
3881 * No outstanding transactions to flush, we can report the latest received
3882 * position. This is important for synchronous replication.
3883 */
3884 if (!have_pending_txes)
3885 flushpos = writepos = recvpos;
3886
3887 if (writepos < last_writepos)
3888 writepos = last_writepos;
3889
3890 if (flushpos < last_flushpos)
3891 flushpos = last_flushpos;
3892
3893 now = GetCurrentTimestamp();
3894
3895 /* if we've already reported everything we're good */
3896 if (!force &&
3897 writepos == last_writepos &&
3898 flushpos == last_flushpos &&
3899 !TimestampDifferenceExceeds(send_time, now,
3900 wal_receiver_status_interval * 1000))
3901 return;
3902 send_time = now;
3903
3904 if (!reply_message)
3905 {
3906 MemoryContext oldctx = MemoryContextSwitchTo(ApplyContext);
3907
3908 reply_message = makeStringInfo();
3909 MemoryContextSwitchTo(oldctx);
3910 }
3911 else
3912 resetStringInfo(reply_message);
3913
3914 pq_sendbyte(reply_message, 'r');
3915 pq_sendint64(reply_message, recvpos); /* write */
3916 pq_sendint64(reply_message, flushpos); /* flush */
3917 pq_sendint64(reply_message, writepos); /* apply */
3918 pq_sendint64(reply_message, now); /* sendTime */
3919 pq_sendbyte(reply_message, requestReply); /* replyRequested */
3920
3921 elog(DEBUG2, "sending feedback (force %d) to recv %X/%X, write %X/%X, flush %X/%X",
3922 force,
3923 LSN_FORMAT_ARGS(recvpos),
3924 LSN_FORMAT_ARGS(writepos),
3925 LSN_FORMAT_ARGS(flushpos));
3926
3927 walrcv_send(LogRepWorkerWalRcvConn,
3928 reply_message->data, reply_message->len);
3929
3930 if (recvpos > last_recvpos)
3931 last_recvpos = recvpos;
3932 if (writepos > last_writepos)
3933 last_writepos = writepos;
3934 if (flushpos > last_flushpos)
3935 last_flushpos = flushpos;
3936 }
3937
3938 /*
3939 * Exit routine for apply workers due to subscription parameter changes.
3940 */
3941 static void
3942 apply_worker_exit(void)
3943 {
3944 if (am_parallel_apply_worker())
3945 {
3946 /*
3947 * Don't stop the parallel apply worker as the leader will detect the
3948 * subscription parameter change and restart logical replication later
3949 * anyway. This also prevents the leader from reporting errors when
3950 * trying to communicate with a stopped parallel apply worker, which
3951 * would accidentally disable subscriptions if disable_on_error was
3952 * set.
3953 */
3954 return;
3955 }
3956
3957 /*
3958 * Reset the last-start time for this apply worker so that the launcher
3959 * will restart it without waiting for wal_retrieve_retry_interval if the
3960 * subscription is still active, and so that we won't leak that hash table
3961 * entry if it isn't.
3962 */
3963 if (am_leader_apply_worker())
3964 ApplyLauncherForgetWorkerStartTime(MyLogicalRepWorker->subid);
3965
3966 proc_exit(0);
3967 }
3968
3969 /*
3970 * Reread subscription info if needed.
3971 *
3972 * For significant changes, we react by exiting the current process; a new
3973 * one will be launched afterwards if needed.
3974 */
3975 void
3976 maybe_reread_subscription(void)
3977 {
3978 MemoryContext oldctx;
3979 Subscription *newsub;
3980 bool started_tx = false;
3981
3982 /* When cache state is valid there is nothing to do here. */
3983 if (MySubscriptionValid)
3984 return;
3985
3986 /* This function might be called inside or outside of transaction. */
3987 if (!IsTransactionState())
3988 {
3989 StartTransactionCommand();
3990 started_tx = true;
3991 }
3992
3993 /* Ensure allocations in permanent context. */
3994 oldctx = MemoryContextSwitchTo(ApplyContext);
3995
3996 newsub = GetSubscription(MyLogicalRepWorker->subid, true);
3997
3998 /*
3999 * Exit if the subscription was removed. This normally should not happen
4000 * as the worker gets killed during DROP SUBSCRIPTION.
4001 */
4002 if (!newsub)
4003 {
4004 ereport(LOG,
4005 (errmsg("logical replication worker for subscription \"%s\" will stop because the subscription was removed",
4006 MySubscription->name)));
4007
4008 /* Ensure we remove no-longer-useful entry for worker's start time */
4009 if (am_leader_apply_worker())
4010 ApplyLauncherForgetWorkerStartTime(MyLogicalRepWorker->subid);
4011
4012 proc_exit(0);
4013 }
4014
4015 /* Exit if the subscription was disabled. */
4016 if (!newsub->enabled)
4017 {
4018 ereport(LOG,
4019 (errmsg("logical replication worker for subscription \"%s\" will stop because the subscription was disabled",
4020 MySubscription->name)));
4021
4022 apply_worker_exit();
4023 }
4024
4025 /* !slotname should never happen when enabled is true. */
4026 Assert(newsub->slotname);
4027
4028 /* two-phase cannot be altered while the worker is running */
4029 Assert(newsub->twophasestate == MySubscription->twophasestate);
4030
4031 /*
4032 * Exit if any parameter that affects the remote connection was changed.
4033 * The launcher will start a new worker but note that the parallel apply
4034 * worker won't restart if the streaming option's value is changed from
4035 * 'parallel' to any other value or the server decides not to stream the
4036 * in-progress transaction.
4037 */
4038 if (strcmp(newsub->conninfo, MySubscription->conninfo) != 0 ||
4039 strcmp(newsub->name, MySubscription->name) != 0 ||
4040 strcmp(newsub->slotname, MySubscription->slotname) != 0 ||
4041 newsub->binary != MySubscription->binary ||
4042 newsub->stream != MySubscription->stream ||
4043 newsub->passwordrequired != MySubscription->passwordrequired ||
4044 strcmp(newsub->origin, MySubscription->origin) != 0 ||
4045 newsub->owner != MySubscription->owner ||
4046 !equal(newsub->publications, MySubscription->publications))
4047 {
4048 if (am_parallel_apply_worker())
4049 ereport(LOG,
4050 (errmsg("logical replication parallel apply worker for subscription \"%s\" will stop because of a parameter change",
4051 MySubscription->name)));
4052 else
4053 ereport(LOG,
4054 (errmsg("logical replication worker for subscription \"%s\" will restart because of a parameter change",
4055 MySubscription->name)));
4056
4057 apply_worker_exit();
4058 }
4059
4060 /*
4061 * Exit if the subscription owner's superuser privileges have been
4062 * revoked.
4063 */
4064 if (!newsub->ownersuperuser && MySubscription->ownersuperuser)
4065 {
4066 if (am_parallel_apply_worker())
4067 ereport(LOG,
4068 errmsg("logical replication parallel apply worker for subscription \"%s\" will stop because the subscription owner's superuser privileges have been revoked",
4069 MySubscription->name));
4070 else
4071 ereport(LOG,
4072 errmsg("logical replication worker for subscription \"%s\" will restart because the subscription owner's superuser privileges have been revoked",
4073 MySubscription->name));
4074
4075 apply_worker_exit();
4076 }
4077
4078 /* Check for other changes that should never happen too. */
4079 if (newsub->dbid != MySubscription->dbid)
4080 {
4081 elog(ERROR, "subscription %u changed unexpectedly",
4082 MyLogicalRepWorker->subid);
4083 }
4084
4085 /* Clean old subscription info and switch to new one. */
4086 FreeSubscription(MySubscription);
4087 MySubscription = newsub;
4088
4089 MemoryContextSwitchTo(oldctx);
4090
4091 /* Change synchronous commit according to the user's wishes */
4092 SetConfigOption("synchronous_commit", MySubscription->synccommit,
4093 PGC_BACKEND, PGC_S_OVERRIDE);
4094
4095 if (started_tx)
4096 CommitTransactionCommand();
4097
4098 MySubscriptionValid = true;
4099 }
4100
4101 /*
4102 * Callback from subscription syscache invalidation.
4103 */
4104 static void
4105 subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue)
4106 {
4107 MySubscriptionValid = false;
4108 }
4109
4110 /*
4111 * subxact_info_write
4112 * Store information about subxacts for a toplevel transaction.
4113 *
4114 * For each subxact we store offset of its first change in the main file.
4115 * The file is always over-written as a whole.
4116 *
4117 * XXX We should only store subxacts that were not aborted yet.
4118 */
4119 static void
4120 subxact_info_write(Oid subid, TransactionId xid)
4121 {
4122 char path[MAXPGPATH];
4123 Size len;
4124 BufFile *fd;
4125
4126 Assert(TransactionIdIsValid(xid));
4127
4128 /* construct the subxact filename */
4129 subxact_filename(path, subid, xid);
4130
4131 /* Delete the subxacts file, if exists. */
4132 if (subxact_data.nsubxacts == 0)
4133 {
4134 cleanup_subxact_info();
4135 BufFileDeleteFileSet(MyLogicalRepWorker->stream_fileset, path, true);
4136
4137 return;
4138 }
4139
4140 /*
4141 * Create the subxact file if it not already created, otherwise open the
4142 * existing file.
4143 */
4144 fd = BufFileOpenFileSet(MyLogicalRepWorker->stream_fileset, path, O_RDWR,
4145 true);
4146 if (fd == NULL)
4147 fd = BufFileCreateFileSet(MyLogicalRepWorker->stream_fileset, path);
4148
4149 len = sizeof(SubXactInfo) * subxact_data.nsubxacts;
4150
4151 /* Write the subxact count and subxact info */
4152 BufFileWrite(fd, &subxact_data.nsubxacts, sizeof(subxact_data.nsubxacts));
4153 BufFileWrite(fd, subxact_data.subxacts, len);
4154
4155 BufFileClose(fd);
4156
4157 /* free the memory allocated for subxact info */
4158 cleanup_subxact_info();
4159 }
4160
4161 /*
4162 * subxact_info_read
4163 * Restore information about subxacts of a streamed transaction.
4164 *
4165 * Read information about subxacts into the structure subxact_data that can be
4166 * used later.
4167 */
4168 static void
4169 subxact_info_read(Oid subid, TransactionId xid)
4170 {
4171 char path[MAXPGPATH];
4172 Size len;
4173 BufFile *fd;
4174 MemoryContext oldctx;
4175
4176 Assert(!subxact_data.subxacts);
4177 Assert(subxact_data.nsubxacts == 0);
4178 Assert(subxact_data.nsubxacts_max == 0);
4179
4180 /*
4181 * If the subxact file doesn't exist that means we don't have any subxact
4182 * info.
4183 */
4184 subxact_filename(path, subid, xid);
4185 fd = BufFileOpenFileSet(MyLogicalRepWorker->stream_fileset, path, O_RDONLY,
4186 true);
4187 if (fd == NULL)
4188 return;
4189
4190 /* read number of subxact items */
4191 BufFileReadExact(fd, &subxact_data.nsubxacts, sizeof(subxact_data.nsubxacts));
4192
4193 len = sizeof(SubXactInfo) * subxact_data.nsubxacts;
4194
4195 /* we keep the maximum as a power of 2 */
4196 subxact_data.nsubxacts_max = 1 << my_log2(subxact_data.nsubxacts);
4197
4198 /*
4199 * Allocate subxact information in the logical streaming context. We need
4200 * this information during the complete stream so that we can add the sub
4201 * transaction info to this. On stream stop we will flush this information
4202 * to the subxact file and reset the logical streaming context.
4203 */
4204 oldctx = MemoryContextSwitchTo(LogicalStreamingContext);
4205 subxact_data.subxacts = palloc(subxact_data.nsubxacts_max *
4206 sizeof(SubXactInfo));
4207 MemoryContextSwitchTo(oldctx);
4208
4209 if (len > 0)
4210 BufFileReadExact(fd, subxact_data.subxacts, len);
4211
4212 BufFileClose(fd);
4213 }
4214
4215 /*
4216 * subxact_info_add
4217 * Add information about a subxact (offset in the main file).
4218 */
4219 static void
4220 subxact_info_add(TransactionId xid)
4221 {
4222 SubXactInfo *subxacts = subxact_data.subxacts;
4223 int64 i;
4224
4225 /* We must have a valid top level stream xid and a stream fd. */
4226 Assert(TransactionIdIsValid(stream_xid));
4227 Assert(stream_fd != NULL);
4228
4229 /*
4230 * If the XID matches the toplevel transaction, we don't want to add it.
4231 */
4232 if (stream_xid == xid)
4233 return;
4234
4235 /*
4236 * In most cases we're checking the same subxact as we've already seen in
4237 * the last call, so make sure to ignore it (this change comes later).
4238 */
4239 if (subxact_data.subxact_last == xid)
4240 return;
4241
4242 /* OK, remember we're processing this XID. */
4243 subxact_data.subxact_last = xid;
4244
4245 /*
4246 * Check if the transaction is already present in the array of subxact. We
4247 * intentionally scan the array from the tail, because we're likely adding
4248 * a change for the most recent subtransactions.
4249 *
4250 * XXX Can we rely on the subxact XIDs arriving in sorted order? That
4251 * would allow us to use binary search here.
4252 */
4253 for (i = subxact_data.nsubxacts; i > 0; i--)
4254 {
4255 /* found, so we're done */
4256 if (subxacts[i - 1].xid == xid)
4257 return;
4258 }
4259
4260 /* This is a new subxact, so we need to add it to the array. */
4261 if (subxact_data.nsubxacts == 0)
4262 {
4263 MemoryContext oldctx;
4264
4265 subxact_data.nsubxacts_max = 128;
4266
4267 /*
4268 * Allocate this memory for subxacts in per-stream context, see
4269 * subxact_info_read.
4270 */
4271 oldctx = MemoryContextSwitchTo(LogicalStreamingContext);
4272 subxacts = palloc(subxact_data.nsubxacts_max * sizeof(SubXactInfo));
4273 MemoryContextSwitchTo(oldctx);
4274 }
4275 else if (subxact_data.nsubxacts == subxact_data.nsubxacts_max)
4276 {
4277 subxact_data.nsubxacts_max *= 2;
4278 subxacts = repalloc(subxacts,
4279 subxact_data.nsubxacts_max * sizeof(SubXactInfo));
4280 }
4281
4282 subxacts[subxact_data.nsubxacts].xid = xid;
4283
4284 /*
4285 * Get the current offset of the stream file and store it as offset of
4286 * this subxact.
4287 */
4288 BufFileTell(stream_fd,
4289 &subxacts[subxact_data.nsubxacts].fileno,
4290 &subxacts[subxact_data.nsubxacts].offset);
4291
4292 subxact_data.nsubxacts++;
4293 subxact_data.subxacts = subxacts;
4294 }
4295
4296 /* format filename for file containing the info about subxacts */
4297 static inline void
4298 subxact_filename(char *path, Oid subid, TransactionId xid)
4299 {
4300 snprintf(path, MAXPGPATH, "%u-%u.subxacts", subid, xid);
4301 }
4302
4303 /* format filename for file containing serialized changes */
4304 static inline void
4305 changes_filename(char *path, Oid subid, TransactionId xid)
4306 {
4307 snprintf(path, MAXPGPATH, "%u-%u.changes", subid, xid);
4308 }
4309
4310 /*
4311 * stream_cleanup_files
4312 * Cleanup files for a subscription / toplevel transaction.
4313 *
4314 * Remove files with serialized changes and subxact info for a particular
4315 * toplevel transaction. Each subscription has a separate set of files
4316 * for any toplevel transaction.
4317 */
4318 void
4319 stream_cleanup_files(Oid subid, TransactionId xid)
4320 {
4321 char path[MAXPGPATH];
4322
4323 /* Delete the changes file. */
4324 changes_filename(path, subid, xid);
4325 BufFileDeleteFileSet(MyLogicalRepWorker->stream_fileset, path, false);
4326
4327 /* Delete the subxact file, if it exists. */
4328 subxact_filename(path, subid, xid);
4329 BufFileDeleteFileSet(MyLogicalRepWorker->stream_fileset, path, true);
4330 }
4331
4332 /*
4333 * stream_open_file
4334 * Open a file that we'll use to serialize changes for a toplevel
4335 * transaction.
4336 *
4337 * Open a file for streamed changes from a toplevel transaction identified
4338 * by stream_xid (global variable). If it's the first chunk of streamed
4339 * changes for this transaction, create the buffile, otherwise open the
4340 * previously created file.
4341 */
4342 static void
4343 stream_open_file(Oid subid, TransactionId xid, bool first_segment)
4344 {
4345 char path[MAXPGPATH];
4346 MemoryContext oldcxt;
4347
4348 Assert(OidIsValid(subid));
4349 Assert(TransactionIdIsValid(xid));
4350 Assert(stream_fd == NULL);
4351
4352
4353 changes_filename(path, subid, xid);
4354 elog(DEBUG1, "opening file \"%s\" for streamed changes", path);
4355
4356 /*
4357 * Create/open the buffiles under the logical streaming context so that we
4358 * have those files until stream stop.
4359 */
4360 oldcxt = MemoryContextSwitchTo(LogicalStreamingContext);
4361
4362 /*
4363 * If this is the first streamed segment, create the changes file.
4364 * Otherwise, just open the file for writing, in append mode.
4365 */
4366 if (first_segment)
4367 stream_fd = BufFileCreateFileSet(MyLogicalRepWorker->stream_fileset,
4368 path);
4369 else
4370 {
4371 /*
4372 * Open the file and seek to the end of the file because we always
4373 * append the changes file.
4374 */
4375 stream_fd = BufFileOpenFileSet(MyLogicalRepWorker->stream_fileset,
4376 path, O_RDWR, false);
4377 BufFileSeek(stream_fd, 0, 0, SEEK_END);
4378 }
4379
4380 MemoryContextSwitchTo(oldcxt);
4381 }
4382
4383 /*
4384 * stream_close_file
4385 * Close the currently open file with streamed changes.
4386 */
4387 static void
4388 stream_close_file(void)
4389 {
4390 Assert(stream_fd != NULL);
4391
4392 BufFileClose(stream_fd);
4393
4394 stream_fd = NULL;
4395 }
4396
4397 /*
4398 * stream_write_change
4399 * Serialize a change to a file for the current toplevel transaction.
4400 *
4401 * The change is serialized in a simple format, with length (not including
4402 * the length), action code (identifying the message type) and message
4403 * contents (without the subxact TransactionId value).
4404 */
4405 static void
4406 stream_write_change(char action, StringInfo s)
4407 {
4408 int len;
4409
4410 Assert(stream_fd != NULL);
4411
4412 /* total on-disk size, including the action type character */
4413 len = (s->len - s->cursor) + sizeof(char);
4414
4415 /* first write the size */
4416 BufFileWrite(stream_fd, &len, sizeof(len));
4417
4418 /* then the action */
4419 BufFileWrite(stream_fd, &action, sizeof(action));
4420
4421 /* and finally the remaining part of the buffer (after the XID) */
4422 len = (s->len - s->cursor);
4423
4424 BufFileWrite(stream_fd, &s->data[s->cursor], len);
4425 }
4426
4427 /*
4428 * stream_open_and_write_change
4429 * Serialize a message to a file for the given transaction.
4430 *
4431 * This function is similar to stream_write_change except that it will open the
4432 * target file if not already before writing the message and close the file at
4433 * the end.
4434 */
4435 static void
4436 stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
4437 {
4438 Assert(!in_streamed_transaction);
4439
4440 if (!stream_fd)
4441 stream_start_internal(xid, false);
4442
4443 stream_write_change(action, s);
4444 stream_stop_internal(xid);
4445 }
4446
4447 /*
4448 * Sets streaming options including replication slot name and origin start
4449 * position. Workers need these options for logical replication.
4450 */
4451 void
4452 set_stream_options(WalRcvStreamOptions *options,
4453 char *slotname,
4454 XLogRecPtr *origin_startpos)
4455 {
4456 int server_version;
4457
4458 options->logical = true;
4459 options->startpoint = *origin_startpos;
4460 options->slotname = slotname;
4461
4462 server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
4463 options->proto.logical.proto_version =
4464 server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
4465 server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
4466 server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
4467 LOGICALREP_PROTO_VERSION_NUM;
4468
4469 options->proto.logical.publication_names = MySubscription->publications;
4470 options->proto.logical.binary = MySubscription->binary;
4471
4472 /*
4473 * Assign the appropriate option value for streaming option according to
4474 * the 'streaming' mode and the publisher's ability to support that mode.
4475 */
4476 if (server_version >= 160000 &&
4477 MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
4478 {
4479 options->proto.logical.streaming_str = "parallel";
4480 MyLogicalRepWorker->parallel_apply = true;
4481 }
4482 else if (server_version >= 140000 &&
4483 MySubscription->stream != LOGICALREP_STREAM_OFF)
4484 {
4485 options->proto.logical.streaming_str = "on";
4486 MyLogicalRepWorker->parallel_apply = false;
4487 }
4488 else
4489 {
4490 options->proto.logical.streaming_str = NULL;
4491 MyLogicalRepWorker->parallel_apply = false;
4492 }
4493
4494 options->proto.logical.twophase = false;
4495 options->proto.logical.origin = pstrdup(MySubscription->origin);
4496 }
4497
4498 /*
4499 * Cleanup the memory for subxacts and reset the related variables.
4500 */
4501 static inline void
4502 cleanup_subxact_info()
4503 {
4504 if (subxact_data.subxacts)
4505 pfree(subxact_data.subxacts);
4506
4507 subxact_data.subxacts = NULL;
4508 subxact_data.subxact_last = InvalidTransactionId;
4509 subxact_data.nsubxacts = 0;
4510 subxact_data.nsubxacts_max = 0;
4511 }
4512
4513 /*
4514 * Common function to run the apply loop with error handling. Disable the
4515 * subscription, if necessary.
4516 *
4517 * Note that we don't handle FATAL errors which are probably because
4518 * of system resource error and are not repeatable.
4519 */
4520 void
4521 start_apply(XLogRecPtr origin_startpos)
4522 {
4523 PG_TRY();
4524 {
4525 LogicalRepApplyLoop(origin_startpos);
4526 }
4527 PG_CATCH();
4528 {
4529 /*
4530 * Reset the origin state to prevent the advancement of origin
4531 * progress if we fail to apply. Otherwise, this will result in
4532 * transaction loss as that transaction won't be sent again by the
4533 * server.
4534 */
4535 replorigin_reset(0, (Datum) 0);
4536
4537 if (MySubscription->disableonerr)
4538 DisableSubscriptionAndExit();
4539 else
4540 {
4541 /*
4542 * Report the worker failed while applying changes. Abort the
4543 * current transaction so that the stats message is sent in an
4544 * idle state.
4545 */
4546 AbortOutOfAnyTransaction();
4547 pgstat_report_subscription_error(MySubscription->oid, !am_tablesync_worker());
4548
4549 PG_RE_THROW();
4550 }
4551 }
4552 PG_END_TRY();
4553 }
4554
4555 /*
4556 * Runs the leader apply worker.
4557 *
4558 * It sets up replication origin, streaming options and then starts streaming.
4559 */
4560 static void
4561 run_apply_worker()
4562 {
4563 char originname[NAMEDATALEN];
4564 XLogRecPtr origin_startpos = InvalidXLogRecPtr;
4565 char *slotname = NULL;
4566 WalRcvStreamOptions options;
4567 RepOriginId originid;
4568 TimeLineID startpointTLI;
4569 char *err;
4570 bool must_use_password;
4571
4572 slotname = MySubscription->slotname;
4573
4574 /*
4575 * This shouldn't happen if the subscription is enabled, but guard against
4576 * DDL bugs or manual catalog changes. (libpqwalreceiver will crash if
4577 * slot is NULL.)
4578 */
4579 if (!slotname)
4580 ereport(ERROR,
4581 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
4582 errmsg("subscription has no replication slot set")));
4583
4584 /* Setup replication origin tracking. */
4585 ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
4586 originname, sizeof(originname));
4587 StartTransactionCommand();
4588 originid = replorigin_by_name(originname, true);
4589 if (!OidIsValid(originid))
4590 originid = replorigin_create(originname);
4591 replorigin_session_setup(originid, 0);
4592 replorigin_session_origin = originid;
4593 origin_startpos = replorigin_session_get_progress(false);
4594 CommitTransactionCommand();
4595
4596 /* Is the use of a password mandatory? */
4597 must_use_password = MySubscription->passwordrequired &&
4598 !MySubscription->ownersuperuser;
4599
4600 LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
4601 true, must_use_password,
4602 MySubscription->name, &err);
4603
4604 if (LogRepWorkerWalRcvConn == NULL)
4605 ereport(ERROR,
4606 (errcode(ERRCODE_CONNECTION_FAILURE),
4607 errmsg("apply worker for subscription \"%s\" could not connect to the publisher: %s",
4608 MySubscription->name, err)));
4609
4610 /*
4611 * We don't really use the output identify_system for anything but it does
4612 * some initializations on the upstream so let's still call it.
4613 */
4614 (void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
4615
4616 set_apply_error_context_origin(originname);
4617
4618 set_stream_options(&options, slotname, &origin_startpos);
4619
4620 /*
4621 * Even when the two_phase mode is requested by the user, it remains as
4622 * the tri-state PENDING until all tablesyncs have reached READY state.
4623 * Only then, can it become ENABLED.
4624 *
4625 * Note: If the subscription has no tables then leave the state as
4626 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
4627 * work.
4628 */
4629 if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
4630 AllTablesyncsReady())
4631 {
4632 /* Start streaming with two_phase enabled */
4633 options.proto.logical.twophase = true;
4634 walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
4635
4636 StartTransactionCommand();
4637
4638 /*
4639 * Updating pg_subscription might involve TOAST table access, so
4640 * ensure we have a valid snapshot.
4641 */
4642 PushActiveSnapshot(GetTransactionSnapshot());
4643
4644 UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
4645 MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
4646 PopActiveSnapshot();
4647 CommitTransactionCommand();
4648 }
4649 else
4650 {
4651 walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
4652 }
4653
4654 ereport(DEBUG1,
4655 (errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
4656 MySubscription->name,
4657 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
4658 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
4659 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
4660 "?")));
4661
4662 /* Run the main loop. */
4663 start_apply(origin_startpos);
4664 }
4665
4666 /*
4667 * Common initialization for leader apply worker, parallel apply worker and
4668 * tablesync worker.
4669 *
4670 * Initialize the database connection, in-memory subscription and necessary
4671 * config options.
4672 */
4673 void
4674 InitializeLogRepWorker(void)
4675 {
4676 MemoryContext oldctx;
4677
4678 /* Run as replica session replication role. */
4679 SetConfigOption("session_replication_role", "replica",
4680 PGC_SUSET, PGC_S_OVERRIDE);
4681
4682 /* Connect to our database. */
4683 BackgroundWorkerInitializeConnectionByOid(MyLogicalRepWorker->dbid,
4684 MyLogicalRepWorker->userid,
4685 0);
4686
4687 /*
4688 * Set always-secure search path, so malicious users can't redirect user
4689 * code (e.g. pg_index.indexprs).
4690 */
4691 SetConfigOption("search_path", "", PGC_SUSET, PGC_S_OVERRIDE);
4692
4693 /* Load the subscription into persistent memory context. */
4694 ApplyContext = AllocSetContextCreate(TopMemoryContext,
4695 "ApplyContext",
4696 ALLOCSET_DEFAULT_SIZES);
4697 StartTransactionCommand();
4698 oldctx = MemoryContextSwitchTo(ApplyContext);
4699
4700 /*
4701 * Lock the subscription to prevent it from being concurrently dropped,
4702 * then re-verify its existence. After the initialization, the worker will
4703 * be terminated gracefully if the subscription is dropped.
4704 */
4705 LockSharedObject(SubscriptionRelationId, MyLogicalRepWorker->subid, 0,
4706 AccessShareLock);
4707 MySubscription = GetSubscription(MyLogicalRepWorker->subid, true);
4708 if (!MySubscription)
4709 {
4710 ereport(LOG,
4711 (errmsg("logical replication worker for subscription %u will not start because the subscription was removed during startup",
4712 MyLogicalRepWorker->subid)));
4713
4714 /* Ensure we remove no-longer-useful entry for worker's start time */
4715 if (am_leader_apply_worker())
4716 ApplyLauncherForgetWorkerStartTime(MyLogicalRepWorker->subid);
4717
4718 proc_exit(0);
4719 }
4720
4721 MySubscriptionValid = true;
4722 MemoryContextSwitchTo(oldctx);
4723
4724 if (!MySubscription->enabled)
4725 {
4726 ereport(LOG,
4727 (errmsg("logical replication worker for subscription \"%s\" will not start because the subscription was disabled during startup",
4728 MySubscription->name)));
4729
4730 apply_worker_exit();
4731 }
4732
4733 /* Setup synchronous commit according to the user's wishes */
4734 SetConfigOption("synchronous_commit", MySubscription->synccommit,
4735 PGC_BACKEND, PGC_S_OVERRIDE);
4736
4737 /*
4738 * Keep us informed about subscription or role changes. Note that the
4739 * role's superuser privilege can be revoked.
4740 */
4741 CacheRegisterSyscacheCallback(SUBSCRIPTIONOID,
4742 subscription_change_cb,
4743 (Datum) 0);
4744
4745 CacheRegisterSyscacheCallback(AUTHOID,
4746 subscription_change_cb,
4747 (Datum) 0);
4748
4749 if (am_tablesync_worker())
4750 ereport(LOG,
4751 (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
4752 MySubscription->name,
4753 get_rel_name(MyLogicalRepWorker->relid))));
4754 else
4755 ereport(LOG,
4756 (errmsg("logical replication apply worker for subscription \"%s\" has started",
4757 MySubscription->name)));
4758
4759 CommitTransactionCommand();
4760
4761 /*
4762 * Register a callback to reset the origin state before aborting any
4763 * pending transaction during shutdown (see ShutdownPostgres()). This will
4764 * avoid origin advancement for an incomplete transaction which could
4765 * otherwise lead to its loss as such a transaction won't be sent by the
4766 * server again.
4767 *
4768 * Note that even a LOG or DEBUG statement placed after setting the origin
4769 * state may process a shutdown signal before committing the current apply
4770 * operation. So, it is important to register such a callback here.
4771 *
4772 * Register this callback here to ensure that all types of logical
4773 * replication workers that set up origins and apply remote transactions
4774 * are protected.
4775 */
4776 before_shmem_exit(replorigin_reset, (Datum) 0);
4777 }
4778
4779 /*
4780 * Reset the origin state.
4781 */
4782 static void
4783 replorigin_reset(int code, Datum arg)
4784 {
4785 replorigin_session_origin = InvalidRepOriginId;
4786 replorigin_session_origin_lsn = InvalidXLogRecPtr;
4787 replorigin_session_origin_timestamp = 0;
4788 }
4789
4790 /* Common function to setup the leader apply or tablesync worker. */
4791 void
4792 SetupApplyOrSyncWorker(int worker_slot)
4793 {
4794 /* Attach to slot */
4795 logicalrep_worker_attach(worker_slot);
4796
4797 Assert(am_tablesync_worker() || am_leader_apply_worker());
4798
4799 /* Setup signal handling */
4800 pqsignal(SIGHUP, SignalHandlerForConfigReload);
4801 pqsignal(SIGTERM, die);
4802 BackgroundWorkerUnblockSignals();
4803
4804 /*
4805 * We don't currently need any ResourceOwner in a walreceiver process, but
4806 * if we did, we could call CreateAuxProcessResourceOwner here.
4807 */
4808
4809 /* Initialise stats to a sanish value */
4810 MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
4811 MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
4812
4813 /* Load the libpq-specific functions */
4814 load_file("libpqwalreceiver", false);
4815
4816 InitializeLogRepWorker();
4817
4818 /*
4819 * Setup callback for syscache so that we know when something changes in
4820 * the subscription relation state.
4821 */
4822 CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
4823 invalidate_syncing_table_states,
4824 (Datum) 0);
4825 }
4826
4827 /* Logical Replication Apply worker entry point */
4828 void
4829 ApplyWorkerMain(Datum main_arg)
4830 {
4831 int worker_slot = DatumGetInt32(main_arg);
4832
4833 InitializingApplyWorker = true;
4834
4835 SetupApplyOrSyncWorker(worker_slot);
4836
4837 InitializingApplyWorker = false;
4838
4839 run_apply_worker();
4840
4841 proc_exit(0);
4842 }
4843
4844 /*
4845 * After error recovery, disable the subscription in a new transaction
4846 * and exit cleanly.
4847 */
4848 void
4849 DisableSubscriptionAndExit(void)
4850 {
4851 /*
4852 * Emit the error message, and recover from the error state to an idle
4853 * state
4854 */
4855 HOLD_INTERRUPTS();
4856
4857 EmitErrorReport();
4858 AbortOutOfAnyTransaction();
4859 FlushErrorState();
4860
4861 RESUME_INTERRUPTS();
4862
4863 /* Report the worker failed during either table synchronization or apply */
4864 pgstat_report_subscription_error(MyLogicalRepWorker->subid,
4865 !am_tablesync_worker());
4866
4867 /* Disable the subscription */
4868 StartTransactionCommand();
4869
4870 /*
4871 * Updating pg_subscription might involve TOAST table access, so ensure we
4872 * have a valid snapshot.
4873 */
4874 PushActiveSnapshot(GetTransactionSnapshot());
4875
4876 DisableSubscription(MySubscription->oid);
4877 PopActiveSnapshot();
4878 CommitTransactionCommand();
4879
4880 /* Ensure we remove no-longer-useful entry for worker's start time */
4881 if (am_leader_apply_worker())
4882 ApplyLauncherForgetWorkerStartTime(MyLogicalRepWorker->subid);
4883
4884 /* Notify the subscription has been disabled and exit */
4885 ereport(LOG,
4886 errmsg("subscription \"%s\" has been disabled because of an error",
4887 MySubscription->name));
4888
4889 proc_exit(0);
4890 }
4891
4892 /*
4893 * Is current process a logical replication worker?
4894 */
4895 bool
4896 IsLogicalWorker(void)
4897 {
4898 return MyLogicalRepWorker != NULL;
4899 }
4900
4901 /*
4902 * Is current process a logical replication parallel apply worker?
4903 */
4904 bool
4905 IsLogicalParallelApplyWorker(void)
4906 {
4907 return IsLogicalWorker() && am_parallel_apply_worker();
4908 }
4909
4910 /*
4911 * Start skipping changes of the transaction if the given LSN matches the
4912 * LSN specified by subscription's skiplsn.
4913 */
4914 static void
4915 maybe_start_skipping_changes(XLogRecPtr finish_lsn)
4916 {
4917 Assert(!is_skipping_changes());
4918 Assert(!in_remote_transaction);
4919 Assert(!in_streamed_transaction);
4920
4921 /*
4922 * Quick return if it's not requested to skip this transaction. This
4923 * function is called for every remote transaction and we assume that
4924 * skipping the transaction is not used often.
4925 */
4926 if (likely(XLogRecPtrIsInvalid(MySubscription->skiplsn) ||
4927 MySubscription->skiplsn != finish_lsn))
4928 return;
4929
4930 /* Start skipping all changes of this transaction */
4931 skip_xact_finish_lsn = finish_lsn;
4932
4933 ereport(LOG,
4934 errmsg("logical replication starts skipping transaction at LSN %X/%X",
4935 LSN_FORMAT_ARGS(skip_xact_finish_lsn)));
4936 }
4937
4938 /*
4939 * Stop skipping changes by resetting skip_xact_finish_lsn if enabled.
4940 */
4941 static void
4942 stop_skipping_changes(void)
4943 {
4944 if (!is_skipping_changes())
4945 return;
4946
4947 ereport(LOG,
4948 (errmsg("logical replication completed skipping transaction at LSN %X/%X",
4949 LSN_FORMAT_ARGS(skip_xact_finish_lsn))));
4950
4951 /* Stop skipping changes */
4952 skip_xact_finish_lsn = InvalidXLogRecPtr;
4953 }
4954
4955 /*
4956 * Clear subskiplsn of pg_subscription catalog.
4957 *
4958 * finish_lsn is the transaction's finish LSN that is used to check if the
4959 * subskiplsn matches it. If not matched, we raise a warning when clearing the
4960 * subskiplsn in order to inform users for cases e.g., where the user mistakenly
4961 * specified the wrong subskiplsn.
4962 */
4963 static void
4964 clear_subscription_skip_lsn(XLogRecPtr finish_lsn)
4965 {
4966 Relation rel;
4967 Form_pg_subscription subform;
4968 HeapTuple tup;
4969 XLogRecPtr myskiplsn = MySubscription->skiplsn;
4970 bool started_tx = false;
4971
4972 if (likely(XLogRecPtrIsInvalid(myskiplsn)) || am_parallel_apply_worker())
4973 return;
4974
4975 if (!IsTransactionState())
4976 {
4977 StartTransactionCommand();
4978 started_tx = true;
4979 }
4980
4981 /*
4982 * Updating pg_subscription might involve TOAST table access, so ensure we
4983 * have a valid snapshot.
4984 */
4985 PushActiveSnapshot(GetTransactionSnapshot());
4986
4987 /*
4988 * Protect subskiplsn of pg_subscription from being concurrently updated
4989 * while clearing it.
4990 */
4991 LockSharedObject(SubscriptionRelationId, MySubscription->oid, 0,
4992 AccessShareLock);
4993
4994 rel = table_open(SubscriptionRelationId, RowExclusiveLock);
4995
4996 /* Fetch the existing tuple. */
4997 tup = SearchSysCacheCopy1(SUBSCRIPTIONOID,
4998 ObjectIdGetDatum(MySubscription->oid));
4999
5000 if (!HeapTupleIsValid(tup))
5001 elog(ERROR, "subscription \"%s\" does not exist", MySubscription->name);
5002
5003 subform = (Form_pg_subscription) GETSTRUCT(tup);
5004
5005 /*
5006 * Clear the subskiplsn. If the user has already changed subskiplsn before
5007 * clearing it we don't update the catalog and the replication origin
5008 * state won't get advanced. So in the worst case, if the server crashes
5009 * before sending an acknowledgment of the flush position the transaction
5010 * will be sent again and the user needs to set subskiplsn again. We can
5011 * reduce the possibility by logging a replication origin WAL record to
5012 * advance the origin LSN instead but there is no way to advance the
5013 * origin timestamp and it doesn't seem to be worth doing anything about
5014 * it since it's a very rare case.
5015 */
5016 if (subform->subskiplsn == myskiplsn)
5017 {
5018 bool nulls[Natts_pg_subscription];
5019 bool replaces[Natts_pg_subscription];
5020 Datum values[Natts_pg_subscription];
5021
5022 memset(values, 0, sizeof(values));
5023 memset(nulls, false, sizeof(nulls));
5024 memset(replaces, false, sizeof(replaces));
5025
5026 /* reset subskiplsn */
5027 values[Anum_pg_subscription_subskiplsn - 1] = LSNGetDatum(InvalidXLogRecPtr);
5028 replaces[Anum_pg_subscription_subskiplsn - 1] = true;
5029
5030 tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
5031 replaces);
5032 CatalogTupleUpdate(rel, &tup->t_self, tup);
5033
5034 if (myskiplsn != finish_lsn)
5035 ereport(WARNING,
5036 errmsg("skip-LSN of subscription \"%s\" cleared", MySubscription->name),
5037 errdetail("Remote transaction's finish WAL location (LSN) %X/%X did not match skip-LSN %X/%X.",
5038 LSN_FORMAT_ARGS(finish_lsn),
5039 LSN_FORMAT_ARGS(myskiplsn)));
5040 }
5041
5042 heap_freetuple(tup);
5043 table_close(rel, NoLock);
5044
5045 PopActiveSnapshot();
5046
5047 if (started_tx)
5048 CommitTransactionCommand();
5049 }
5050
5051 /* Error callback to give more context info about the change being applied */
5052 void
5053 apply_error_callback(void *arg)
5054 {
5055 ApplyErrorCallbackArg *errarg = &apply_error_callback_arg;
5056
5057 if (apply_error_callback_arg.command == 0)
5058 return;
5059
5060 Assert(errarg->origin_name);
5061
5062 if (errarg->rel == NULL)
5063 {
5064 if (!TransactionIdIsValid(errarg->remote_xid))
5065 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\"",
5066 errarg->origin_name,
5067 logicalrep_message_type(errarg->command));
5068 else if (XLogRecPtrIsInvalid(errarg->finish_lsn))
5069 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" in transaction %u",
5070 errarg->origin_name,
5071 logicalrep_message_type(errarg->command),
5072 errarg->remote_xid);
5073 else
5074 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" in transaction %u, finished at %X/%X",
5075 errarg->origin_name,
5076 logicalrep_message_type(errarg->command),
5077 errarg->remote_xid,
5078 LSN_FORMAT_ARGS(errarg->finish_lsn));
5079 }
5080 else
5081 {
5082 if (errarg->remote_attnum < 0)
5083 {
5084 if (XLogRecPtrIsInvalid(errarg->finish_lsn))
5085 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" in transaction %u",
5086 errarg->origin_name,
5087 logicalrep_message_type(errarg->command),
5088 errarg->rel->remoterel.nspname,
5089 errarg->rel->remoterel.relname,
5090 errarg->remote_xid);
5091 else
5092 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" in transaction %u, finished at %X/%X",
5093 errarg->origin_name,
5094 logicalrep_message_type(errarg->command),
5095 errarg->rel->remoterel.nspname,
5096 errarg->rel->remoterel.relname,
5097 errarg->remote_xid,
5098 LSN_FORMAT_ARGS(errarg->finish_lsn));
5099 }
5100 else
5101 {
5102 if (XLogRecPtrIsInvalid(errarg->finish_lsn))
5103 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u",
5104 errarg->origin_name,
5105 logicalrep_message_type(errarg->command),
5106 errarg->rel->remoterel.nspname,
5107 errarg->rel->remoterel.relname,
5108 errarg->rel->remoterel.attnames[errarg->remote_attnum],
5109 errarg->remote_xid);
5110 else
5111 errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u, finished at %X/%X",
5112 errarg->origin_name,
5113 logicalrep_message_type(errarg->command),
5114 errarg->rel->remoterel.nspname,
5115 errarg->rel->remoterel.relname,
5116 errarg->rel->remoterel.attnames[errarg->remote_attnum],
5117 errarg->remote_xid,
5118 LSN_FORMAT_ARGS(errarg->finish_lsn));
5119 }
5120 }
5121 }
5122
5123 /* Set transaction information of apply error callback */
5124 static inline void
5125 set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn)
5126 {
5127 apply_error_callback_arg.remote_xid = xid;
5128 apply_error_callback_arg.finish_lsn = lsn;
5129 }
5130
5131 /* Reset all information of apply error callback */
5132 static inline void
5133 reset_apply_error_context_info(void)
5134 {
5135 apply_error_callback_arg.command = 0;
5136 apply_error_callback_arg.rel = NULL;
5137 apply_error_callback_arg.remote_attnum = -1;
5138 set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr);
5139 }
5140
5141 /*
5142 * Request wakeup of the workers for the given subscription OID
5143 * at commit of the current transaction.
5144 *
5145 * This is used to ensure that the workers process assorted changes
5146 * as soon as possible.
5147 */
5148 void
5149 LogicalRepWorkersWakeupAtCommit(Oid subid)
5150 {
5151 MemoryContext oldcxt;
5152
5153 oldcxt = MemoryContextSwitchTo(TopTransactionContext);
5154 on_commit_wakeup_workers_subids =
5155 list_append_unique_oid(on_commit_wakeup_workers_subids, subid);
5156 MemoryContextSwitchTo(oldcxt);
5157 }
5158
5159 /*
5160 * Wake up the workers of any subscriptions that were changed in this xact.
5161 */
5162 void
5163 AtEOXact_LogicalRepWorkers(bool isCommit)
5164 {
5165 if (isCommit && on_commit_wakeup_workers_subids != NIL)
5166 {
5167 ListCell *lc;
5168
5169 LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
5170 foreach(lc, on_commit_wakeup_workers_subids)
5171 {
5172 Oid subid = lfirst_oid(lc);
5173 List *workers;
5174 ListCell *lc2;
5175
5176 workers = logicalrep_workers_find(subid, true, false);
5177 foreach(lc2, workers)
5178 {
5179 LogicalRepWorker *worker = (LogicalRepWorker *) lfirst(lc2);
5180
5181 logicalrep_worker_wakeup_ptr(worker);
5182 }
5183 }
5184 LWLockRelease(LogicalRepWorkerLock);
5185 }
5186
5187 /* The List storage will be reclaimed automatically in xact cleanup. */
5188 on_commit_wakeup_workers_subids = NIL;
5189 }
5190
5191 /*
5192 * Allocate the origin name in long-lived context for error context message.
5193 */
5194 void
5195 set_apply_error_context_origin(char *originname)
5196 {
5197 apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
5198 originname);
5199 }
5200
5201 /*
5202 * Return the action to be taken for the given transaction. See
5203 * TransApplyAction for information on each of the actions.
5204 *
5205 * *winfo is assigned to the destination parallel worker info when the leader
5206 * apply worker has to pass all the transaction's changes to the parallel
5207 * apply worker.
5208 */
5209 static TransApplyAction
5210 get_transaction_apply_action(TransactionId xid, ParallelApplyWorkerInfo **winfo)
5211 {
5212 *winfo = NULL;
5213
5214 if (am_parallel_apply_worker())
5215 {
5216 return TRANS_PARALLEL_APPLY;
5217 }
5218
5219 /*
5220 * If we are processing this transaction using a parallel apply worker
5221 * then either we send the changes to the parallel worker or if the worker
5222 * is busy then serialize the changes to the file which will later be
5223 * processed by the parallel worker.
5224 */
5225 *winfo = pa_find_worker(xid);
5226
5227 if (*winfo && (*winfo)->serialize_changes)
5228 {
5229 return TRANS_LEADER_PARTIAL_SERIALIZE;
5230 }
5231 else if (*winfo)
5232 {
5233 return TRANS_LEADER_SEND_TO_PARALLEL;
5234 }
5235
5236 /*
5237 * If there is no parallel worker involved to process this transaction
5238 * then we either directly apply the change or serialize it to a file
5239 * which will later be applied when the transaction finish message is
5240 * processed.
5241 */
5242 else if (in_streamed_transaction)
5243 {
5244 return TRANS_LEADER_SERIALIZE;
5245 }
5246 else
5247 {
5248 return TRANS_LEADER_APPLY;
5249 }
5250 }