Debian Codegraph @

src/backend/storage/buffer/bufmgr.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * bufmgr.c
4 * buffer manager interface routines
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 * Portions Copyright (c) 1994, Regents of the University of California
8 *
9 *
10 * IDENTIFICATION
11 * src/backend/storage/buffer/bufmgr.c
12 *
13 *-------------------------------------------------------------------------
14 */
15 /*
16 * Principal entry points:
17 *
18 * ReadBuffer() -- find or create a buffer holding the requested page,
19 * and pin it so that no one can destroy it while this process
20 * is using it.
21 *
22 * StartReadBuffer() -- as above, with separate wait step
23 * StartReadBuffers() -- multiple block version
24 * WaitReadBuffers() -- second step of above
25 *
26 * ReleaseBuffer() -- unpin a buffer
27 *
28 * MarkBufferDirty() -- mark a pinned buffer's contents as "dirty".
29 * The disk write is delayed until buffer replacement or checkpoint.
30 *
31 * See also these files:
32 * freelist.c -- chooses victim for buffer replacement
33 * buf_table.c -- manages the buffer lookup table
34 */
35 #include "postgres.h"
36
37 #include <sys/file.h>
38 #include <unistd.h>
39
40 #include "access/tableam.h"
41 #include "access/xloginsert.h"
42 #include "access/xlogutils.h"
43 #ifdef USE_ASSERT_CHECKING
44 #include "catalog/pg_tablespace_d.h"
45 #endif
46 #include "catalog/storage.h"
47 #include "catalog/storage_xlog.h"
48 #include "executor/instrument.h"
49 #include "lib/binaryheap.h"
50 #include "miscadmin.h"
51 #include "pg_trace.h"
52 #include "pgstat.h"
53 #include "postmaster/bgwriter.h"
54 #include "storage/aio.h"
55 #include "storage/buf_internals.h"
56 #include "storage/bufmgr.h"
57 #include "storage/fd.h"
58 #include "storage/ipc.h"
59 #include "storage/lmgr.h"
60 #include "storage/proc.h"
61 #include "storage/read_stream.h"
62 #include "storage/smgr.h"
63 #include "storage/standby.h"
64 #include "utils/memdebug.h"
65 #include "utils/ps_status.h"
66 #include "utils/rel.h"
67 #include "utils/resowner.h"
68 #include "utils/timestamp.h"
69
70
71 /* Note: these two macros only work on shared buffers, not local ones! */
72 #define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ))
73 #define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr)))
74
75 /* Note: this macro only works on local buffers, not shared ones! */
76 #define LocalBufHdrGetBlock(bufHdr) \
77 LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)]
78
79 /* Bits in SyncOneBuffer's return value */
80 #define BUF_WRITTEN 0x01
81 #define BUF_REUSABLE 0x02
82
83 #define RELS_BSEARCH_THRESHOLD 20
84
85 /*
86 * This is the size (in the number of blocks) above which we scan the
87 * entire buffer pool to remove the buffers for all the pages of relation
88 * being dropped. For the relations with size below this threshold, we find
89 * the buffers by doing lookups in BufMapping table.
90 */
91 #define BUF_DROP_FULL_SCAN_THRESHOLD (uint64) (NBuffers / 32)
92
93 typedef struct PrivateRefCountEntry
94 {
95 Buffer buffer;
96 int32 refcount;
97 } PrivateRefCountEntry;
98
99 /* 64 bytes, about the size of a cache line on common systems */
100 #define REFCOUNT_ARRAY_ENTRIES 8
101
102 /*
103 * Status of buffers to checkpoint for a particular tablespace, used
104 * internally in BufferSync.
105 */
106 typedef struct CkptTsStatus
107 {
108 /* oid of the tablespace */
109 Oid tsId;
110
111 /*
112 * Checkpoint progress for this tablespace. To make progress comparable
113 * between tablespaces the progress is, for each tablespace, measured as a
114 * number between 0 and the total number of to-be-checkpointed pages. Each
115 * page checkpointed in this tablespace increments this space's progress
116 * by progress_slice.
117 */
118 float8 progress;
119 float8 progress_slice;
120
121 /* number of to-be checkpointed pages in this tablespace */
122 int num_to_scan;
123 /* already processed pages in this tablespace */
124 int num_scanned;
125
126 /* current offset in CkptBufferIds for this tablespace */
127 int index;
128 } CkptTsStatus;
129
130 /*
131 * Type for array used to sort SMgrRelations
132 *
133 * FlushRelationsAllBuffers shares the same comparator function with
134 * DropRelationsAllBuffers. Pointer to this struct and RelFileLocator must be
135 * compatible.
136 */
137 typedef struct SMgrSortArray
138 {
139 RelFileLocator rlocator; /* This must be the first member */
140 SMgrRelation srel;
141 } SMgrSortArray;
142
143 /* GUC variables */
144 bool zero_damaged_pages = false;
145 int bgwriter_lru_maxpages = 100;
146 double bgwriter_lru_multiplier = 2.0;
147 bool track_io_timing = false;
148
149 /*
150 * How many buffers PrefetchBuffer callers should try to stay ahead of their
151 * ReadBuffer calls by. Zero means "never prefetch". This value is only used
152 * for buffers not belonging to tablespaces that have their
153 * effective_io_concurrency parameter set.
154 */
155 int effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY;
156
157 /*
158 * Like effective_io_concurrency, but used by maintenance code paths that might
159 * benefit from a higher setting because they work on behalf of many sessions.
160 * Overridden by the tablespace setting of the same name.
161 */
162 int maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY;
163
164 /*
165 * Limit on how many blocks should be handled in single I/O operations.
166 * StartReadBuffers() callers should respect it, as should other operations
167 * that call smgr APIs directly. It is computed as the minimum of underlying
168 * GUCs io_combine_limit_guc and io_max_combine_limit.
169 */
170 int io_combine_limit = DEFAULT_IO_COMBINE_LIMIT;
171 int io_combine_limit_guc = DEFAULT_IO_COMBINE_LIMIT;
172 int io_max_combine_limit = DEFAULT_IO_COMBINE_LIMIT;
173
174 /*
175 * GUC variables about triggering kernel writeback for buffers written; OS
176 * dependent defaults are set via the GUC mechanism.
177 */
178 int checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER;
179 int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER;
180 int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER;
181
182 /* local state for LockBufferForCleanup */
183 static BufferDesc *PinCountWaitBuf = NULL;
184
185 /*
186 * Backend-Private refcount management:
187 *
188 * Each buffer also has a private refcount that keeps track of the number of
189 * times the buffer is pinned in the current process. This is so that the
190 * shared refcount needs to be modified only once if a buffer is pinned more
191 * than once by an individual backend. It's also used to check that no buffers
192 * are still pinned at the end of transactions and when exiting.
193 *
194 *
195 * To avoid - as we used to - requiring an array with NBuffers entries to keep
196 * track of local buffers, we use a small sequentially searched array
197 * (PrivateRefCountArray) and an overflow hash table (PrivateRefCountHash) to
198 * keep track of backend local pins.
199 *
200 * Until no more than REFCOUNT_ARRAY_ENTRIES buffers are pinned at once, all
201 * refcounts are kept track of in the array; after that, new array entries
202 * displace old ones into the hash table. That way a frequently used entry
203 * can't get "stuck" in the hashtable while infrequent ones clog the array.
204 *
205 * Note that in most scenarios the number of pinned buffers will not exceed
206 * REFCOUNT_ARRAY_ENTRIES.
207 *
208 *
209 * To enter a buffer into the refcount tracking mechanism first reserve a free
210 * entry using ReservePrivateRefCountEntry() and then later, if necessary,
211 * fill it with NewPrivateRefCountEntry(). That split lets us avoid doing
212 * memory allocations in NewPrivateRefCountEntry() which can be important
213 * because in some scenarios it's called with a spinlock held...
214 */
215 static struct PrivateRefCountEntry PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES];
216 static HTAB *PrivateRefCountHash = NULL;
217 static int32 PrivateRefCountOverflowed = 0;
218 static uint32 PrivateRefCountClock = 0;
219 static PrivateRefCountEntry *ReservedRefCountEntry = NULL;
220
221 static uint32 MaxProportionalPins;
222
223 static void ReservePrivateRefCountEntry(void);
224 static PrivateRefCountEntry *NewPrivateRefCountEntry(Buffer buffer);
225 static PrivateRefCountEntry *GetPrivateRefCountEntry(Buffer buffer, bool do_move);
226 static inline int32 GetPrivateRefCount(Buffer buffer);
227 static void ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref);
228
229 /* ResourceOwner callbacks to hold in-progress I/Os and buffer pins */
230 static void ResOwnerReleaseBufferIO(Datum res);
231 static char *ResOwnerPrintBufferIO(Datum res);
232 static void ResOwnerReleaseBufferPin(Datum res);
233 static char *ResOwnerPrintBufferPin(Datum res);
234
235 const ResourceOwnerDesc buffer_io_resowner_desc =
236 {
237 .name = "buffer io",
238 .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
239 .release_priority = RELEASE_PRIO_BUFFER_IOS,
240 .ReleaseResource = ResOwnerReleaseBufferIO,
241 .DebugPrint = ResOwnerPrintBufferIO
242 };
243
244 const ResourceOwnerDesc buffer_pin_resowner_desc =
245 {
246 .name = "buffer pin",
247 .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
248 .release_priority = RELEASE_PRIO_BUFFER_PINS,
249 .ReleaseResource = ResOwnerReleaseBufferPin,
250 .DebugPrint = ResOwnerPrintBufferPin
251 };
252
253 /*
254 * Ensure that the PrivateRefCountArray has sufficient space to store one more
255 * entry. This has to be called before using NewPrivateRefCountEntry() to fill
256 * a new entry - but it's perfectly fine to not use a reserved entry.
257 */
258 static void
259 ReservePrivateRefCountEntry(void)
260 {
261 /* Already reserved (or freed), nothing to do */
262 if (ReservedRefCountEntry != NULL)
263 return;
264
265 /*
266 * First search for a free entry the array, that'll be sufficient in the
267 * majority of cases.
268 */
269 {
270 int i;
271
272 for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
273 {
274 PrivateRefCountEntry *res;
275
276 res = &PrivateRefCountArray[i];
277
278 if (res->buffer == InvalidBuffer)
279 {
280 ReservedRefCountEntry = res;
281 return;
282 }
283 }
284 }
285
286 /*
287 * No luck. All array entries are full. Move one array entry into the hash
288 * table.
289 */
290 {
291 /*
292 * Move entry from the current clock position in the array into the
293 * hashtable. Use that slot.
294 */
295 PrivateRefCountEntry *hashent;
296 bool found;
297
298 /* select victim slot */
299 ReservedRefCountEntry =
300 &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES];
301
302 /* Better be used, otherwise we shouldn't get here. */
303 Assert(ReservedRefCountEntry->buffer != InvalidBuffer);
304
305 /* enter victim array entry into hashtable */
306 hashent = hash_search(PrivateRefCountHash,
307 &(ReservedRefCountEntry->buffer),
308 HASH_ENTER,
309 &found);
310 Assert(!found);
311 hashent->refcount = ReservedRefCountEntry->refcount;
312
313 /* clear the now free array slot */
314 ReservedRefCountEntry->buffer = InvalidBuffer;
315 ReservedRefCountEntry->refcount = 0;
316
317 PrivateRefCountOverflowed++;
318 }
319 }
320
321 /*
322 * Fill a previously reserved refcount entry.
323 */
324 static PrivateRefCountEntry *
325 NewPrivateRefCountEntry(Buffer buffer)
326 {
327 PrivateRefCountEntry *res;
328
329 /* only allowed to be called when a reservation has been made */
330 Assert(ReservedRefCountEntry != NULL);
331
332 /* use up the reserved entry */
333 res = ReservedRefCountEntry;
334 ReservedRefCountEntry = NULL;
335
336 /* and fill it */
337 res->buffer = buffer;
338 res->refcount = 0;
339
340 return res;
341 }
342
343 /*
344 * Return the PrivateRefCount entry for the passed buffer.
345 *
346 * Returns NULL if a buffer doesn't have a refcount entry. Otherwise, if
347 * do_move is true, and the entry resides in the hashtable the entry is
348 * optimized for frequent access by moving it to the array.
349 */
350 static PrivateRefCountEntry *
351 GetPrivateRefCountEntry(Buffer buffer, bool do_move)
352 {
353 PrivateRefCountEntry *res;
354 int i;
355
356 Assert(BufferIsValid(buffer));
357 Assert(!BufferIsLocal(buffer));
358
359 /*
360 * First search for references in the array, that'll be sufficient in the
361 * majority of cases.
362 */
363 for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
364 {
365 res = &PrivateRefCountArray[i];
366
367 if (res->buffer == buffer)
368 return res;
369 }
370
371 /*
372 * By here we know that the buffer, if already pinned, isn't residing in
373 * the array.
374 *
375 * Only look up the buffer in the hashtable if we've previously overflowed
376 * into it.
377 */
378 if (PrivateRefCountOverflowed == 0)
379 return NULL;
380
381 res = hash_search(PrivateRefCountHash, &buffer, HASH_FIND, NULL);
382
383 if (res == NULL)
384 return NULL;
385 else if (!do_move)
386 {
387 /* caller doesn't want us to move the hash entry into the array */
388 return res;
389 }
390 else
391 {
392 /* move buffer from hashtable into the free array slot */
393 bool found;
394 PrivateRefCountEntry *free;
395
396 /* Ensure there's a free array slot */
397 ReservePrivateRefCountEntry();
398
399 /* Use up the reserved slot */
400 Assert(ReservedRefCountEntry != NULL);
401 free = ReservedRefCountEntry;
402 ReservedRefCountEntry = NULL;
403 Assert(free->buffer == InvalidBuffer);
404
405 /* and fill it */
406 free->buffer = buffer;
407 free->refcount = res->refcount;
408
409 /* delete from hashtable */
410 hash_search(PrivateRefCountHash, &buffer, HASH_REMOVE, &found);
411 Assert(found);
412 Assert(PrivateRefCountOverflowed > 0);
413 PrivateRefCountOverflowed--;
414
415 return free;
416 }
417 }
418
419 /*
420 * Returns how many times the passed buffer is pinned by this backend.
421 *
422 * Only works for shared memory buffers!
423 */
424 static inline int32
425 GetPrivateRefCount(Buffer buffer)
426 {
427 PrivateRefCountEntry *ref;
428
429 Assert(BufferIsValid(buffer));
430 Assert(!BufferIsLocal(buffer));
431
432 /*
433 * Not moving the entry - that's ok for the current users, but we might
434 * want to change this one day.
435 */
436 ref = GetPrivateRefCountEntry(buffer, false);
437
438 if (ref == NULL)
439 return 0;
440 return ref->refcount;
441 }
442
443 /*
444 * Release resources used to track the reference count of a buffer which we no
445 * longer have pinned and don't want to pin again immediately.
446 */
447 static void
448 ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref)
449 {
450 Assert(ref->refcount == 0);
451
452 if (ref >= &PrivateRefCountArray[0] &&
453 ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES])
454 {
455 ref->buffer = InvalidBuffer;
456
457 /*
458 * Mark the just used entry as reserved - in many scenarios that
459 * allows us to avoid ever having to search the array/hash for free
460 * entries.
461 */
462 ReservedRefCountEntry = ref;
463 }
464 else
465 {
466 bool found;
467 Buffer buffer = ref->buffer;
468
469 hash_search(PrivateRefCountHash, &buffer, HASH_REMOVE, &found);
470 Assert(found);
471 Assert(PrivateRefCountOverflowed > 0);
472 PrivateRefCountOverflowed--;
473 }
474 }
475
476 /*
477 * BufferIsPinned
478 * True iff the buffer is pinned (also checks for valid buffer number).
479 *
480 * NOTE: what we check here is that *this* backend holds a pin on
481 * the buffer. We do not care whether some other backend does.
482 */
483 #define BufferIsPinned(bufnum) \
484 ( \
485 !BufferIsValid(bufnum) ? \
486 false \
487 : \
488 BufferIsLocal(bufnum) ? \
489 (LocalRefCount[-(bufnum) - 1] > 0) \
490 : \
491 (GetPrivateRefCount(bufnum) > 0) \
492 )
493
494
495 static Buffer ReadBuffer_common(Relation rel,
496 SMgrRelation smgr, char smgr_persistence,
497 ForkNumber forkNum, BlockNumber blockNum,
498 ReadBufferMode mode, BufferAccessStrategy strategy);
499 static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr,
500 ForkNumber fork,
501 BufferAccessStrategy strategy,
502 uint32 flags,
503 uint32 extend_by,
504 BlockNumber extend_upto,
505 Buffer *buffers,
506 uint32 *extended_by);
507 static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr,
508 ForkNumber fork,
509 BufferAccessStrategy strategy,
510 uint32 flags,
511 uint32 extend_by,
512 BlockNumber extend_upto,
513 Buffer *buffers,
514 uint32 *extended_by);
515 static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy);
516 static void PinBuffer_Locked(BufferDesc *buf);
517 static void UnpinBuffer(BufferDesc *buf);
518 static void UnpinBufferNoOwner(BufferDesc *buf);
519 static void BufferSync(int flags);
520 static uint32 WaitBufHdrUnlocked(BufferDesc *buf);
521 static int SyncOneBuffer(int buf_id, bool skip_recently_used,
522 WritebackContext *wb_context);
523 static void WaitIO(BufferDesc *buf);
524 static void AbortBufferIO(Buffer buffer);
525 static void shared_buffer_write_error_callback(void *arg);
526 static void local_buffer_write_error_callback(void *arg);
527 static inline BufferDesc *BufferAlloc(SMgrRelation smgr,
528 char relpersistence,
529 ForkNumber forkNum,
530 BlockNumber blockNum,
531 BufferAccessStrategy strategy,
532 bool *foundPtr, IOContext io_context);
533 static bool AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress);
534 static void CheckReadBuffersOperation(ReadBuffersOperation *operation, bool is_complete);
535 static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context);
536 static void FlushBuffer(BufferDesc *buf, SMgrRelation reln,
537 IOObject io_object, IOContext io_context);
538 static void FindAndDropRelationBuffers(RelFileLocator rlocator,
539 ForkNumber forkNum,
540 BlockNumber nForkBlock,
541 BlockNumber firstDelBlock);
542 static void RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
543 RelFileLocator dstlocator,
544 ForkNumber forkNum, bool permanent);
545 static void AtProcExit_Buffers(int code, Datum arg);
546 static void CheckForBufferLeaks(void);
547 #ifdef USE_ASSERT_CHECKING
548 static void AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode,
549 void *unused_context);
550 #endif
551 static int rlocator_comparator(const void *p1, const void *p2);
552 static inline int buffertag_comparator(const BufferTag *ba, const BufferTag *bb);
553 static inline int ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b);
554 static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg);
555
556
557 /*
558 * Implementation of PrefetchBuffer() for shared buffers.
559 */
560 PrefetchBufferResult
561 PrefetchSharedBuffer(SMgrRelation smgr_reln,
562 ForkNumber forkNum,
563 BlockNumber blockNum)
564 {
565 PrefetchBufferResult result = {InvalidBuffer, false};
566 BufferTag newTag; /* identity of requested block */
567 uint32 newHash; /* hash value for newTag */
568 LWLock *newPartitionLock; /* buffer partition lock for it */
569 int buf_id;
570
571 Assert(BlockNumberIsValid(blockNum));
572
573 /* create a tag so we can lookup the buffer */
574 InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator,
575 forkNum, blockNum);
576
577 /* determine its hash code and partition lock ID */
578 newHash = BufTableHashCode(&newTag);
579 newPartitionLock = BufMappingPartitionLock(newHash);
580
581 /* see if the block is in the buffer pool already */
582 LWLockAcquire(newPartitionLock, LW_SHARED);
583 buf_id = BufTableLookup(&newTag, newHash);
584 LWLockRelease(newPartitionLock);
585
586 /* If not in buffers, initiate prefetch */
587 if (buf_id < 0)
588 {
589 #ifdef USE_PREFETCH
590 /*
591 * Try to initiate an asynchronous read. This returns false in
592 * recovery if the relation file doesn't exist.
593 */
594 if ((io_direct_flags & IO_DIRECT_DATA) == 0 &&
595 smgrprefetch(smgr_reln, forkNum, blockNum, 1))
596 {
597 result.initiated_io = true;
598 }
599 #endif /* USE_PREFETCH */
600 }
601 else
602 {
603 /*
604 * Report the buffer it was in at that time. The caller may be able
605 * to avoid a buffer table lookup, but it's not pinned and it must be
606 * rechecked!
607 */
608 result.recent_buffer = buf_id + 1;
609 }
610
611 /*
612 * If the block *is* in buffers, we do nothing. This is not really ideal:
613 * the block might be just about to be evicted, which would be stupid
614 * since we know we are going to need it soon. But the only easy answer
615 * is to bump the usage_count, which does not seem like a great solution:
616 * when the caller does ultimately touch the block, usage_count would get
617 * bumped again, resulting in too much favoritism for blocks that are
618 * involved in a prefetch sequence. A real fix would involve some
619 * additional per-buffer state, and it's not clear that there's enough of
620 * a problem to justify that.
621 */
622
623 return result;
624 }
625
626 /*
627 * PrefetchBuffer -- initiate asynchronous read of a block of a relation
628 *
629 * This is named by analogy to ReadBuffer but doesn't actually allocate a
630 * buffer. Instead it tries to ensure that a future ReadBuffer for the given
631 * block will not be delayed by the I/O. Prefetching is optional.
632 *
633 * There are three possible outcomes:
634 *
635 * 1. If the block is already cached, the result includes a valid buffer that
636 * could be used by the caller to avoid the need for a later buffer lookup, but
637 * it's not pinned, so the caller must recheck it.
638 *
639 * 2. If the kernel has been asked to initiate I/O, the initiated_io member is
640 * true. Currently there is no way to know if the data was already cached by
641 * the kernel and therefore didn't really initiate I/O, and no way to know when
642 * the I/O completes other than using synchronous ReadBuffer().
643 *
644 * 3. Otherwise, the buffer wasn't already cached by PostgreSQL, and
645 * USE_PREFETCH is not defined (this build doesn't support prefetching due to
646 * lack of a kernel facility), direct I/O is enabled, or the underlying
647 * relation file wasn't found and we are in recovery. (If the relation file
648 * wasn't found and we are not in recovery, an error is raised).
649 */
650 PrefetchBufferResult
651 PrefetchBuffer(Relation reln, ForkNumber forkNum, BlockNumber blockNum)
652 {
653 Assert(RelationIsValid(reln));
654 Assert(BlockNumberIsValid(blockNum));
655
656 if (RelationUsesLocalBuffers(reln))
657 {
658 /* see comments in ReadBuffer_common */
659 if (RELATION_IS_OTHER_TEMP(reln))
660 ereport(ERROR,
661 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
662 errmsg("cannot access temporary tables of other sessions")));
663
664 /* pass it off to localbuf.c */
665 return PrefetchLocalBuffer(RelationGetSmgr(reln), forkNum, blockNum);
666 }
667 else
668 {
669 /* pass it to the shared buffer version */
670 return PrefetchSharedBuffer(RelationGetSmgr(reln), forkNum, blockNum);
671 }
672 }
673
674 /*
675 * ReadRecentBuffer -- try to pin a block in a recently observed buffer
676 *
677 * Compared to ReadBuffer(), this avoids a buffer mapping lookup when it's
678 * successful. Return true if the buffer is valid and still has the expected
679 * tag. In that case, the buffer is pinned and the usage count is bumped.
680 */
681 bool
682 ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum,
683 Buffer recent_buffer)
684 {
685 BufferDesc *bufHdr;
686 BufferTag tag;
687 uint32 buf_state;
688 bool have_private_ref;
689
690 Assert(BufferIsValid(recent_buffer));
691
692 ResourceOwnerEnlarge(CurrentResourceOwner);
693 ReservePrivateRefCountEntry();
694 InitBufferTag(&tag, &rlocator, forkNum, blockNum);
695
696 if (BufferIsLocal(recent_buffer))
697 {
698 int b = -recent_buffer - 1;
699
700 bufHdr = GetLocalBufferDescriptor(b);
701 buf_state = pg_atomic_read_u32(&bufHdr->state);
702
703 /* Is it still valid and holding the right tag? */
704 if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag))
705 {
706 PinLocalBuffer(bufHdr, true);
707
708 pgBufferUsage.local_blks_hit++;
709
710 return true;
711 }
712 }
713 else
714 {
715 bufHdr = GetBufferDescriptor(recent_buffer - 1);
716 have_private_ref = GetPrivateRefCount(recent_buffer) > 0;
717
718 /*
719 * Do we already have this buffer pinned with a private reference? If
720 * so, it must be valid and it is safe to check the tag without
721 * locking. If not, we have to lock the header first and then check.
722 */
723 if (have_private_ref)
724 buf_state = pg_atomic_read_u32(&bufHdr->state);
725 else
726 buf_state = LockBufHdr(bufHdr);
727
728 if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag))
729 {
730 /*
731 * It's now safe to pin the buffer. We can't pin first and ask
732 * questions later, because it might confuse code paths like
733 * InvalidateBuffer() if we pinned a random non-matching buffer.
734 */
735 if (have_private_ref)
736 PinBuffer(bufHdr, NULL); /* bump pin count */
737 else
738 PinBuffer_Locked(bufHdr); /* pin for first time */
739
740 pgBufferUsage.shared_blks_hit++;
741
742 return true;
743 }
744
745 /* If we locked the header above, now unlock. */
746 if (!have_private_ref)
747 UnlockBufHdr(bufHdr, buf_state);
748 }
749
750 return false;
751 }
752
753 /*
754 * ReadBuffer -- a shorthand for ReadBufferExtended, for reading from main
755 * fork with RBM_NORMAL mode and default strategy.
756 */
757 Buffer
758 ReadBuffer(Relation reln, BlockNumber blockNum)
759 {
760 return ReadBufferExtended(reln, MAIN_FORKNUM, blockNum, RBM_NORMAL, NULL);
761 }
762
763 /*
764 * ReadBufferExtended -- returns a buffer containing the requested
765 * block of the requested relation. If the blknum
766 * requested is P_NEW, extend the relation file and
767 * allocate a new block. (Caller is responsible for
768 * ensuring that only one backend tries to extend a
769 * relation at the same time!)
770 *
771 * Returns: the buffer number for the buffer containing
772 * the block read. The returned buffer has been pinned.
773 * Does not return on error --- elog's instead.
774 *
775 * Assume when this function is called, that reln has been opened already.
776 *
777 * In RBM_NORMAL mode, the page is read from disk, and the page header is
778 * validated. An error is thrown if the page header is not valid. (But
779 * note that an all-zero page is considered "valid"; see
780 * PageIsVerified().)
781 *
782 * RBM_ZERO_ON_ERROR is like the normal mode, but if the page header is not
783 * valid, the page is zeroed instead of throwing an error. This is intended
784 * for non-critical data, where the caller is prepared to repair errors.
785 *
786 * In RBM_ZERO_AND_LOCK mode, if the page isn't in buffer cache already, it's
787 * filled with zeros instead of reading it from disk. Useful when the caller
788 * is going to fill the page from scratch, since this saves I/O and avoids
789 * unnecessary failure if the page-on-disk has corrupt page headers.
790 * The page is returned locked to ensure that the caller has a chance to
791 * initialize the page before it's made visible to others.
792 * Caution: do not use this mode to read a page that is beyond the relation's
793 * current physical EOF; that is likely to cause problems in md.c when
794 * the page is modified and written out. P_NEW is OK, though.
795 *
796 * RBM_ZERO_AND_CLEANUP_LOCK is the same as RBM_ZERO_AND_LOCK, but acquires
797 * a cleanup-strength lock on the page.
798 *
799 * RBM_NORMAL_NO_LOG mode is treated the same as RBM_NORMAL here.
800 *
801 * If strategy is not NULL, a nondefault buffer access strategy is used.
802 * See buffer/README for details.
803 */
804 inline Buffer
805 ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum,
806 ReadBufferMode mode, BufferAccessStrategy strategy)
807 {
808 Buffer buf;
809
810 /*
811 * Read the buffer, and update pgstat counters to reflect a cache hit or
812 * miss. The other-session temp-relation check is enforced by
813 * ReadBuffer_common().
814 */
815 buf = ReadBuffer_common(reln, RelationGetSmgr(reln), 0,
816 forkNum, blockNum, mode, strategy);
817
818 return buf;
819 }
820
821
822 /*
823 * ReadBufferWithoutRelcache -- like ReadBufferExtended, but doesn't require
824 * a relcache entry for the relation.
825 *
826 * Pass permanent = true for a RELPERSISTENCE_PERMANENT relation, and
827 * permanent = false for a RELPERSISTENCE_UNLOGGED relation. This function
828 * cannot be used for temporary relations (and making that work might be
829 * difficult, unless we only want to read temporary relations for our own
830 * ProcNumber).
831 */
832 Buffer
833 ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum,
834 BlockNumber blockNum, ReadBufferMode mode,
835 BufferAccessStrategy strategy, bool permanent)
836 {
837 SMgrRelation smgr = smgropen(rlocator, INVALID_PROC_NUMBER);
838
839 return ReadBuffer_common(NULL, smgr,
840 permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED,
841 forkNum, blockNum,
842 mode, strategy);
843 }
844
845 /*
846 * Convenience wrapper around ExtendBufferedRelBy() extending by one block.
847 */
848 Buffer
849 ExtendBufferedRel(BufferManagerRelation bmr,
850 ForkNumber forkNum,
851 BufferAccessStrategy strategy,
852 uint32 flags)
853 {
854 Buffer buf;
855 uint32 extend_by = 1;
856
857 ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by,
858 &buf, &extend_by);
859
860 return buf;
861 }
862
863 /*
864 * Extend relation by multiple blocks.
865 *
866 * Tries to extend the relation by extend_by blocks. Depending on the
867 * availability of resources the relation may end up being extended by a
868 * smaller number of pages (unless an error is thrown, always by at least one
869 * page). *extended_by is updated to the number of pages the relation has been
870 * extended to.
871 *
872 * buffers needs to be an array that is at least extend_by long. Upon
873 * completion, the first extend_by array elements will point to a pinned
874 * buffer.
875 *
876 * If EB_LOCK_FIRST is part of flags, the first returned buffer is
877 * locked. This is useful for callers that want a buffer that is guaranteed to
878 * be empty.
879 */
880 BlockNumber
881 ExtendBufferedRelBy(BufferManagerRelation bmr,
882 ForkNumber fork,
883 BufferAccessStrategy strategy,
884 uint32 flags,
885 uint32 extend_by,
886 Buffer *buffers,
887 uint32 *extended_by)
888 {
889 Assert((bmr.rel != NULL) != (bmr.smgr != NULL));
890 Assert(bmr.smgr == NULL || bmr.relpersistence != 0);
891 Assert(extend_by > 0);
892
893 if (bmr.smgr == NULL)
894 {
895 bmr.smgr = RelationGetSmgr(bmr.rel);
896 bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
897 }
898
899 return ExtendBufferedRelCommon(bmr, fork, strategy, flags,
900 extend_by, InvalidBlockNumber,
901 buffers, extended_by);
902 }
903
904 /*
905 * Extend the relation so it is at least extend_to blocks large, return buffer
906 * (extend_to - 1).
907 *
908 * This is useful for callers that want to write a specific page, regardless
909 * of the current size of the relation (e.g. useful for visibilitymap and for
910 * crash recovery).
911 */
912 Buffer
913 ExtendBufferedRelTo(BufferManagerRelation bmr,
914 ForkNumber fork,
915 BufferAccessStrategy strategy,
916 uint32 flags,
917 BlockNumber extend_to,
918 ReadBufferMode mode)
919 {
920 BlockNumber current_size;
921 uint32 extended_by = 0;
922 Buffer buffer = InvalidBuffer;
923 Buffer buffers[64];
924
925 Assert((bmr.rel != NULL) != (bmr.smgr != NULL));
926 Assert(bmr.smgr == NULL || bmr.relpersistence != 0);
927 Assert(extend_to != InvalidBlockNumber && extend_to > 0);
928
929 if (bmr.smgr == NULL)
930 {
931 bmr.smgr = RelationGetSmgr(bmr.rel);
932 bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
933 }
934
935 /*
936 * If desired, create the file if it doesn't exist. If
937 * smgr_cached_nblocks[fork] is positive then it must exist, no need for
938 * an smgrexists call.
939 */
940 if ((flags & EB_CREATE_FORK_IF_NEEDED) &&
941 (bmr.smgr->smgr_cached_nblocks[fork] == 0 ||
942 bmr.smgr->smgr_cached_nblocks[fork] == InvalidBlockNumber) &&
943 !smgrexists(bmr.smgr, fork))
944 {
945 LockRelationForExtension(bmr.rel, ExclusiveLock);
946
947 /* recheck, fork might have been created concurrently */
948 if (!smgrexists(bmr.smgr, fork))
949 smgrcreate(bmr.smgr, fork, flags & EB_PERFORMING_RECOVERY);
950
951 UnlockRelationForExtension(bmr.rel, ExclusiveLock);
952 }
953
954 /*
955 * If requested, invalidate size cache, so that smgrnblocks asks the
956 * kernel.
957 */
958 if (flags & EB_CLEAR_SIZE_CACHE)
959 bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber;
960
961 /*
962 * Estimate how many pages we'll need to extend by. This avoids acquiring
963 * unnecessarily many victim buffers.
964 */
965 current_size = smgrnblocks(bmr.smgr, fork);
966
967 /*
968 * Since no-one else can be looking at the page contents yet, there is no
969 * difference between an exclusive lock and a cleanup-strength lock. Note
970 * that we pass the original mode to ReadBuffer_common() below, when
971 * falling back to reading the buffer to a concurrent relation extension.
972 */
973 if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
974 flags |= EB_LOCK_TARGET;
975
976 while (current_size < extend_to)
977 {
978 uint32 num_pages = lengthof(buffers);
979 BlockNumber first_block;
980
981 if ((uint64) current_size + num_pages > extend_to)
982 num_pages = extend_to - current_size;
983
984 first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags,
985 num_pages, extend_to,
986 buffers, &extended_by);
987
988 current_size = first_block + extended_by;
989 Assert(num_pages != 0 || current_size >= extend_to);
990
991 for (uint32 i = 0; i < extended_by; i++)
992 {
993 if (first_block + i != extend_to - 1)
994 ReleaseBuffer(buffers[i]);
995 else
996 buffer = buffers[i];
997 }
998 }
999
1000 /*
1001 * It's possible that another backend concurrently extended the relation.
1002 * In that case read the buffer.
1003 *
1004 * XXX: Should we control this via a flag?
1005 */
1006 if (buffer == InvalidBuffer)
1007 {
1008 Assert(extended_by == 0);
1009 buffer = ReadBuffer_common(bmr.rel, bmr.smgr, bmr.relpersistence,
1010 fork, extend_to - 1, mode, strategy);
1011 }
1012
1013 return buffer;
1014 }
1015
1016 /*
1017 * Lock and optionally zero a buffer, as part of the implementation of
1018 * RBM_ZERO_AND_LOCK or RBM_ZERO_AND_CLEANUP_LOCK. The buffer must be already
1019 * pinned. If the buffer is not already valid, it is zeroed and made valid.
1020 */
1021 static void
1022 ZeroAndLockBuffer(Buffer buffer, ReadBufferMode mode, bool already_valid)
1023 {
1024 BufferDesc *bufHdr;
1025 bool need_to_zero;
1026 bool isLocalBuf = BufferIsLocal(buffer);
1027
1028 Assert(mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK);
1029
1030 if (already_valid)
1031 {
1032 /*
1033 * If the caller already knew the buffer was valid, we can skip some
1034 * header interaction. The caller just wants to lock the buffer.
1035 */
1036 need_to_zero = false;
1037 }
1038 else if (isLocalBuf)
1039 {
1040 /* Simple case for non-shared buffers. */
1041 bufHdr = GetLocalBufferDescriptor(-buffer - 1);
1042 need_to_zero = StartLocalBufferIO(bufHdr, true, false);
1043 }
1044 else
1045 {
1046 /*
1047 * Take BM_IO_IN_PROGRESS, or discover that BM_VALID has been set
1048 * concurrently. Even though we aren't doing I/O, that ensures that
1049 * we don't zero a page that someone else has pinned. An exclusive
1050 * content lock wouldn't be enough, because readers are allowed to
1051 * drop the content lock after determining that a tuple is visible
1052 * (see buffer access rules in README).
1053 */
1054 bufHdr = GetBufferDescriptor(buffer - 1);
1055 need_to_zero = StartBufferIO(bufHdr, true, false);
1056 }
1057
1058 if (need_to_zero)
1059 {
1060 memset(BufferGetPage(buffer), 0, BLCKSZ);
1061
1062 /*
1063 * Grab the buffer content lock before marking the page as valid, to
1064 * make sure that no other backend sees the zeroed page before the
1065 * caller has had a chance to initialize it.
1066 *
1067 * Since no-one else can be looking at the page contents yet, there is
1068 * no difference between an exclusive lock and a cleanup-strength
1069 * lock. (Note that we cannot use LockBuffer() or
1070 * LockBufferForCleanup() here, because they assert that the buffer is
1071 * already valid.)
1072 */
1073 if (!isLocalBuf)
1074 LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE);
1075
1076 /* Set BM_VALID, terminate IO, and wake up any waiters */
1077 if (isLocalBuf)
1078 TerminateLocalBufferIO(bufHdr, false, BM_VALID, false);
1079 else
1080 TerminateBufferIO(bufHdr, false, BM_VALID, true, false);
1081 }
1082 else if (!isLocalBuf)
1083 {
1084 /*
1085 * The buffer is valid, so we can't zero it. The caller still expects
1086 * the page to be locked on return.
1087 */
1088 if (mode == RBM_ZERO_AND_LOCK)
1089 LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
1090 else
1091 LockBufferForCleanup(buffer);
1092 }
1093 }
1094
1095 /*
1096 * Pin a buffer for a given block. *foundPtr is set to true if the block was
1097 * already present, or false if more work is required to either read it in or
1098 * zero it.
1099 */
1100 static pg_always_inline Buffer
1101 PinBufferForBlock(Relation rel,
1102 SMgrRelation smgr,
1103 char persistence,
1104 ForkNumber forkNum,
1105 BlockNumber blockNum,
1106 BufferAccessStrategy strategy,
1107 bool *foundPtr)
1108 {
1109 BufferDesc *bufHdr;
1110 IOContext io_context;
1111 IOObject io_object;
1112
1113 Assert(blockNum != P_NEW);
1114
1115 /* Persistence should be set before */
1116 Assert((persistence == RELPERSISTENCE_TEMP ||
1117 persistence == RELPERSISTENCE_PERMANENT ||
1118 persistence == RELPERSISTENCE_UNLOGGED));
1119
1120 if (persistence == RELPERSISTENCE_TEMP)
1121 {
1122 io_context = IOCONTEXT_NORMAL;
1123 io_object = IOOBJECT_TEMP_RELATION;
1124 }
1125 else
1126 {
1127 io_context = IOContextForStrategy(strategy);
1128 io_object = IOOBJECT_RELATION;
1129 }
1130
1131 TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum,
1132 smgr->smgr_rlocator.locator.spcOid,
1133 smgr->smgr_rlocator.locator.dbOid,
1134 smgr->smgr_rlocator.locator.relNumber,
1135 smgr->smgr_rlocator.backend);
1136
1137 if (persistence == RELPERSISTENCE_TEMP)
1138 {
1139 bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, foundPtr);
1140 if (*foundPtr)
1141 pgBufferUsage.local_blks_hit++;
1142 }
1143 else
1144 {
1145 bufHdr = BufferAlloc(smgr, persistence, forkNum, blockNum,
1146 strategy, foundPtr, io_context);
1147 if (*foundPtr)
1148 pgBufferUsage.shared_blks_hit++;
1149 }
1150 if (rel)
1151 {
1152 /*
1153 * While pgBufferUsage's "read" counter isn't bumped unless we reach
1154 * WaitReadBuffers() (so, not for hits, and not for buffers that are
1155 * zeroed instead), the per-relation stats always count them.
1156 */
1157 pgstat_count_buffer_read(rel);
1158 if (*foundPtr)
1159 pgstat_count_buffer_hit(rel);
1160 }
1161 if (*foundPtr)
1162 {
1163 pgstat_count_io_op(io_object, io_context, IOOP_HIT, 1, 0);
1164 if (VacuumCostActive)
1165 VacuumCostBalance += VacuumCostPageHit;
1166
1167 TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum,
1168 smgr->smgr_rlocator.locator.spcOid,
1169 smgr->smgr_rlocator.locator.dbOid,
1170 smgr->smgr_rlocator.locator.relNumber,
1171 smgr->smgr_rlocator.backend,
1172 true);
1173 }
1174
1175 return BufferDescriptorGetBuffer(bufHdr);
1176 }
1177
1178 /*
1179 * ReadBuffer_common -- common logic for all ReadBuffer variants
1180 *
1181 * smgr is required, rel is optional unless using P_NEW.
1182 */
1183 static pg_always_inline Buffer
1184 ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence,
1185 ForkNumber forkNum,
1186 BlockNumber blockNum, ReadBufferMode mode,
1187 BufferAccessStrategy strategy)
1188 {
1189 ReadBuffersOperation operation;
1190 Buffer buffer;
1191 int flags;
1192 char persistence;
1193
1194 /*
1195 * Reject attempts to read non-local temporary relations; we would be
1196 * likely to get wrong data since we have no visibility into the owning
1197 * session's local buffers. This is the canonical place for the check,
1198 * covering the ReadBufferExtended() entry point and any other caller that
1199 * supplies a Relation.
1200 */
1201 if (rel && RELATION_IS_OTHER_TEMP(rel))
1202 ereport(ERROR,
1203 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1204 errmsg("cannot access temporary tables of other sessions")));
1205
1206 /*
1207 * Backward compatibility path, most code should use ExtendBufferedRel()
1208 * instead, as acquiring the extension lock inside ExtendBufferedRel()
1209 * scales a lot better.
1210 */
1211 if (unlikely(blockNum == P_NEW))
1212 {
1213 uint32 flags = EB_SKIP_EXTENSION_LOCK;
1214
1215 /*
1216 * Since no-one else can be looking at the page contents yet, there is
1217 * no difference between an exclusive lock and a cleanup-strength
1218 * lock.
1219 */
1220 if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
1221 flags |= EB_LOCK_FIRST;
1222
1223 return ExtendBufferedRel(BMR_REL(rel), forkNum, strategy, flags);
1224 }
1225
1226 if (rel)
1227 persistence = rel->rd_rel->relpersistence;
1228 else
1229 persistence = smgr_persistence;
1230
1231 if (unlikely(mode == RBM_ZERO_AND_CLEANUP_LOCK ||
1232 mode == RBM_ZERO_AND_LOCK))
1233 {
1234 bool found;
1235
1236 buffer = PinBufferForBlock(rel, smgr, persistence,
1237 forkNum, blockNum, strategy, &found);
1238 ZeroAndLockBuffer(buffer, mode, found);
1239 return buffer;
1240 }
1241
1242 /*
1243 * Signal that we are going to immediately wait. If we're immediately
1244 * waiting, there is no benefit in actually executing the IO
1245 * asynchronously, it would just add dispatch overhead.
1246 */
1247 flags = READ_BUFFERS_SYNCHRONOUSLY;
1248 if (mode == RBM_ZERO_ON_ERROR)
1249 flags |= READ_BUFFERS_ZERO_ON_ERROR;
1250 operation.smgr = smgr;
1251 operation.rel = rel;
1252 operation.persistence = persistence;
1253 operation.forknum = forkNum;
1254 operation.strategy = strategy;
1255 if (StartReadBuffer(&operation,
1256 &buffer,
1257 blockNum,
1258 flags))
1259 WaitReadBuffers(&operation);
1260
1261 return buffer;
1262 }
1263
1264 static pg_always_inline bool
1265 StartReadBuffersImpl(ReadBuffersOperation *operation,
1266 Buffer *buffers,
1267 BlockNumber blockNum,
1268 int *nblocks,
1269 int flags,
1270 bool allow_forwarding)
1271 {
1272 int actual_nblocks = *nblocks;
1273 int maxcombine = 0;
1274 bool did_start_io;
1275
1276 Assert(*nblocks == 1 || allow_forwarding);
1277 Assert(*nblocks > 0);
1278 Assert(*nblocks <= MAX_IO_COMBINE_LIMIT);
1279
1280 /* see comments in ReadBuffer_common */
1281 if (operation->rel && RELATION_IS_OTHER_TEMP(operation->rel))
1282 ereport(ERROR,
1283 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1284 errmsg("cannot access temporary tables of other sessions")));
1285
1286 for (int i = 0; i < actual_nblocks; ++i)
1287 {
1288 bool found;
1289
1290 if (allow_forwarding && buffers[i] != InvalidBuffer)
1291 {
1292 BufferDesc *bufHdr;
1293
1294 /*
1295 * This is a buffer that was pinned by an earlier call to
1296 * StartReadBuffers(), but couldn't be handled in one operation at
1297 * that time. The operation was split, and the caller has passed
1298 * an already pinned buffer back to us to handle the rest of the
1299 * operation. It must continue at the expected block number.
1300 */
1301 Assert(BufferGetBlockNumber(buffers[i]) == blockNum + i);
1302
1303 /*
1304 * It might be an already valid buffer (a hit) that followed the
1305 * final contiguous block of an earlier I/O (a miss) marking the
1306 * end of it, or a buffer that some other backend has since made
1307 * valid by performing the I/O for us, in which case we can handle
1308 * it as a hit now. It is safe to check for a BM_VALID flag with
1309 * a relaxed load, because we got a fresh view of it while pinning
1310 * it in the previous call.
1311 *
1312 * On the other hand if we don't see BM_VALID yet, it must be an
1313 * I/O that was split by the previous call and we need to try to
1314 * start a new I/O from this block. We're also racing against any
1315 * other backend that might start the I/O or even manage to mark
1316 * it BM_VALID after this check, but StartBufferIO() will handle
1317 * those cases.
1318 */
1319 if (BufferIsLocal(buffers[i]))
1320 bufHdr = GetLocalBufferDescriptor(-buffers[i] - 1);
1321 else
1322 bufHdr = GetBufferDescriptor(buffers[i] - 1);
1323 Assert(pg_atomic_read_u32(&bufHdr->state) & BM_TAG_VALID);
1324 found = pg_atomic_read_u32(&bufHdr->state) & BM_VALID;
1325 }
1326 else
1327 {
1328 buffers[i] = PinBufferForBlock(operation->rel,
1329 operation->smgr,
1330 operation->persistence,
1331 operation->forknum,
1332 blockNum + i,
1333 operation->strategy,
1334 &found);
1335 }
1336
1337 if (found)
1338 {
1339 /*
1340 * We have a hit. If it's the first block in the requested range,
1341 * we can return it immediately and report that WaitReadBuffers()
1342 * does not need to be called. If the initial value of *nblocks
1343 * was larger, the caller will have to call again for the rest.
1344 */
1345 if (i == 0)
1346 {
1347 *nblocks = 1;
1348
1349 #ifdef USE_ASSERT_CHECKING
1350
1351 /*
1352 * Initialize enough of ReadBuffersOperation to make
1353 * CheckReadBuffersOperation() work. Outside of assertions
1354 * that's not necessary when no IO is issued.
1355 */
1356 operation->buffers = buffers;
1357 operation->blocknum = blockNum;
1358 operation->nblocks = 1;
1359 operation->nblocks_done = 1;
1360 CheckReadBuffersOperation(operation, true);
1361 #endif
1362 return false;
1363 }
1364
1365 /*
1366 * Otherwise we already have an I/O to perform, but this block
1367 * can't be included as it is already valid. Split the I/O here.
1368 * There may or may not be more blocks requiring I/O after this
1369 * one, we haven't checked, but they can't be contiguous with this
1370 * one in the way. We'll leave this buffer pinned, forwarding it
1371 * to the next call, avoiding the need to unpin it here and re-pin
1372 * it in the next call.
1373 */
1374 actual_nblocks = i;
1375 break;
1376 }
1377 else
1378 {
1379 /*
1380 * Check how many blocks we can cover with the same IO. The smgr
1381 * implementation might e.g. be limited due to a segment boundary.
1382 */
1383 if (i == 0 && actual_nblocks > 1)
1384 {
1385 maxcombine = smgrmaxcombine(operation->smgr,
1386 operation->forknum,
1387 blockNum);
1388 if (unlikely(maxcombine < actual_nblocks))
1389 {
1390 elog(DEBUG2, "limiting nblocks at %u from %u to %u",
1391 blockNum, actual_nblocks, maxcombine);
1392 actual_nblocks = maxcombine;
1393 }
1394 }
1395 }
1396 }
1397 *nblocks = actual_nblocks;
1398
1399 /* Populate information needed for I/O. */
1400 operation->buffers = buffers;
1401 operation->blocknum = blockNum;
1402 operation->flags = flags;
1403 operation->nblocks = actual_nblocks;
1404 operation->nblocks_done = 0;
1405 pgaio_wref_clear(&operation->io_wref);
1406
1407 /*
1408 * When using AIO, start the IO in the background. If not, issue prefetch
1409 * requests if desired by the caller.
1410 *
1411 * The reason we have a dedicated path for IOMETHOD_SYNC here is to
1412 * de-risk the introduction of AIO somewhat. It's a large architectural
1413 * change, with lots of chances for unanticipated performance effects.
1414 *
1415 * Use of IOMETHOD_SYNC already leads to not actually performing IO
1416 * asynchronously, but without the check here we'd execute IO earlier than
1417 * we used to. Eventually this IOMETHOD_SYNC specific path should go away.
1418 */
1419 if (io_method != IOMETHOD_SYNC)
1420 {
1421 /*
1422 * Try to start IO asynchronously. It's possible that no IO needs to
1423 * be started, if another backend already performed the IO.
1424 *
1425 * Note that if an IO is started, it might not cover the entire
1426 * requested range, e.g. because an intermediary block has been read
1427 * in by another backend. In that case any "trailing" buffers we
1428 * already pinned above will be "forwarded" by read_stream.c to the
1429 * next call to StartReadBuffers().
1430 *
1431 * This is signalled to the caller by decrementing *nblocks *and*
1432 * reducing operation->nblocks. The latter is done here, but not below
1433 * WaitReadBuffers(), as in WaitReadBuffers() we can't "shorten" the
1434 * overall read size anymore, we need to retry until done in its
1435 * entirety or until failed.
1436 */
1437 did_start_io = AsyncReadBuffers(operation, nblocks);
1438
1439 operation->nblocks = *nblocks;
1440 }
1441 else
1442 {
1443 operation->flags |= READ_BUFFERS_SYNCHRONOUSLY;
1444
1445 if (flags & READ_BUFFERS_ISSUE_ADVICE)
1446 {
1447 /*
1448 * In theory we should only do this if PinBufferForBlock() had to
1449 * allocate new buffers above. That way, if two calls to
1450 * StartReadBuffers() were made for the same blocks before
1451 * WaitReadBuffers(), only the first would issue the advice.
1452 * That'd be a better simulation of true asynchronous I/O, which
1453 * would only start the I/O once, but isn't done here for
1454 * simplicity.
1455 */
1456 smgrprefetch(operation->smgr,
1457 operation->forknum,
1458 blockNum,
1459 actual_nblocks);
1460 }
1461
1462 /*
1463 * Indicate that WaitReadBuffers() should be called. WaitReadBuffers()
1464 * will initiate the necessary IO.
1465 */
1466 did_start_io = true;
1467 }
1468
1469 CheckReadBuffersOperation(operation, !did_start_io);
1470
1471 return did_start_io;
1472 }
1473
1474 /*
1475 * Begin reading a range of blocks beginning at blockNum and extending for
1476 * *nblocks. *nblocks and the buffers array are in/out parameters. On entry,
1477 * the buffers elements covered by *nblocks must hold either InvalidBuffer or
1478 * buffers forwarded by an earlier call to StartReadBuffers() that was split
1479 * and is now being continued. On return, *nblocks holds the number of blocks
1480 * accepted by this operation. If it is less than the original number then
1481 * this operation has been split, but buffer elements up to the original
1482 * requested size may hold forwarded buffers to be used for a continuing
1483 * operation. The caller must either start a new I/O beginning at the block
1484 * immediately following the blocks accepted by this call and pass those
1485 * buffers back in, or release them if it chooses not to. It shouldn't make
1486 * any other use of or assumptions about forwarded buffers.
1487 *
1488 * If false is returned, no I/O is necessary and the buffers covered by
1489 * *nblocks on exit are valid and ready to be accessed. If true is returned,
1490 * an I/O has been started, and WaitReadBuffers() must be called with the same
1491 * operation object before the buffers covered by *nblocks on exit can be
1492 * accessed. Along with the operation object, the caller-supplied array of
1493 * buffers must remain valid until WaitReadBuffers() is called, and any
1494 * forwarded buffers must also be preserved for a continuing call unless
1495 * they are explicitly released.
1496 */
1497 bool
1498 StartReadBuffers(ReadBuffersOperation *operation,
1499 Buffer *buffers,
1500 BlockNumber blockNum,
1501 int *nblocks,
1502 int flags)
1503 {
1504 return StartReadBuffersImpl(operation, buffers, blockNum, nblocks, flags,
1505 true /* expect forwarded buffers */ );
1506 }
1507
1508 /*
1509 * Single block version of the StartReadBuffers(). This might save a few
1510 * instructions when called from another translation unit, because it is
1511 * specialized for nblocks == 1.
1512 *
1513 * This version does not support "forwarded" buffers: they cannot be created
1514 * by reading only one block and *buffer is ignored on entry.
1515 */
1516 bool
1517 StartReadBuffer(ReadBuffersOperation *operation,
1518 Buffer *buffer,
1519 BlockNumber blocknum,
1520 int flags)
1521 {
1522 int nblocks = 1;
1523 bool result;
1524
1525 result = StartReadBuffersImpl(operation, buffer, blocknum, &nblocks, flags,
1526 false /* single block, no forwarding */ );
1527 Assert(nblocks == 1); /* single block can't be short */
1528
1529 return result;
1530 }
1531
1532 /*
1533 * Perform sanity checks on the ReadBuffersOperation.
1534 */
1535 static void
1536 CheckReadBuffersOperation(ReadBuffersOperation *operation, bool is_complete)
1537 {
1538 #ifdef USE_ASSERT_CHECKING
1539 Assert(operation->nblocks_done <= operation->nblocks);
1540 Assert(!is_complete || operation->nblocks == operation->nblocks_done);
1541
1542 for (int i = 0; i < operation->nblocks; i++)
1543 {
1544 Buffer buffer = operation->buffers[i];
1545 BufferDesc *buf_hdr = BufferIsLocal(buffer) ?
1546 GetLocalBufferDescriptor(-buffer - 1) :
1547 GetBufferDescriptor(buffer - 1);
1548
1549 Assert(BufferGetBlockNumber(buffer) == operation->blocknum + i);
1550 Assert(pg_atomic_read_u32(&buf_hdr->state) & BM_TAG_VALID);
1551
1552 if (i < operation->nblocks_done)
1553 Assert(pg_atomic_read_u32(&buf_hdr->state) & BM_VALID);
1554 }
1555 #endif
1556 }
1557
1558 /* helper for ReadBuffersCanStartIO(), to avoid repetition */
1559 static inline bool
1560 ReadBuffersCanStartIOOnce(Buffer buffer, bool nowait)
1561 {
1562 if (BufferIsLocal(buffer))
1563 return StartLocalBufferIO(GetLocalBufferDescriptor(-buffer - 1),
1564 true, nowait);
1565 else
1566 return StartBufferIO(GetBufferDescriptor(buffer - 1), true, nowait);
1567 }
1568
1569 /*
1570 * Helper for AsyncReadBuffers that tries to get the buffer ready for IO.
1571 */
1572 static inline bool
1573 ReadBuffersCanStartIO(Buffer buffer, bool nowait)
1574 {
1575 /*
1576 * If this backend currently has staged IO, we need to submit the pending
1577 * IO before waiting for the right to issue IO, to avoid the potential for
1578 * deadlocks (and, more commonly, unnecessary delays for other backends).
1579 */
1580 if (!nowait && pgaio_have_staged())
1581 {
1582 if (ReadBuffersCanStartIOOnce(buffer, true))
1583 return true;
1584
1585 /*
1586 * Unfortunately StartBufferIO() returning false doesn't allow to
1587 * distinguish between the buffer already being valid and IO already
1588 * being in progress. Since IO already being in progress is quite
1589 * rare, this approach seems fine.
1590 */
1591 pgaio_submit_staged();
1592 }
1593
1594 return ReadBuffersCanStartIOOnce(buffer, nowait);
1595 }
1596
1597 /*
1598 * Helper for WaitReadBuffers() that processes the results of a readv
1599 * operation, raising an error if necessary.
1600 */
1601 static void
1602 ProcessReadBuffersResult(ReadBuffersOperation *operation)
1603 {
1604 PgAioReturn *aio_ret = &operation->io_return;
1605 PgAioResultStatus rs = aio_ret->result.status;
1606 int newly_read_blocks = 0;
1607
1608 Assert(pgaio_wref_valid(&operation->io_wref));
1609 Assert(aio_ret->result.status != PGAIO_RS_UNKNOWN);
1610
1611 /*
1612 * SMGR reports the number of blocks successfully read as the result of
1613 * the IO operation. Thus we can simply add that to ->nblocks_done.
1614 */
1615
1616 if (likely(rs != PGAIO_RS_ERROR))
1617 newly_read_blocks = aio_ret->result.result;
1618
1619 if (rs == PGAIO_RS_ERROR || rs == PGAIO_RS_WARNING)
1620 pgaio_result_report(aio_ret->result, &aio_ret->target_data,
1621 rs == PGAIO_RS_ERROR ? ERROR : WARNING);
1622 else if (aio_ret->result.status == PGAIO_RS_PARTIAL)
1623 {
1624 /*
1625 * We'll retry, so we just emit a debug message to the server log (or
1626 * not even that in prod scenarios).
1627 */
1628 pgaio_result_report(aio_ret->result, &aio_ret->target_data, DEBUG1);
1629 elog(DEBUG3, "partial read, will retry");
1630 }
1631
1632 Assert(newly_read_blocks > 0);
1633 Assert(newly_read_blocks <= MAX_IO_COMBINE_LIMIT);
1634
1635 operation->nblocks_done += newly_read_blocks;
1636
1637 Assert(operation->nblocks_done <= operation->nblocks);
1638 }
1639
1640 void
1641 WaitReadBuffers(ReadBuffersOperation *operation)
1642 {
1643 PgAioReturn *aio_ret = &operation->io_return;
1644 IOContext io_context;
1645 IOObject io_object;
1646
1647 if (operation->persistence == RELPERSISTENCE_TEMP)
1648 {
1649 io_context = IOCONTEXT_NORMAL;
1650 io_object = IOOBJECT_TEMP_RELATION;
1651 }
1652 else
1653 {
1654 io_context = IOContextForStrategy(operation->strategy);
1655 io_object = IOOBJECT_RELATION;
1656 }
1657
1658 /*
1659 * If we get here without an IO operation having been issued, the
1660 * io_method == IOMETHOD_SYNC path must have been used. Otherwise the
1661 * caller should not have called WaitReadBuffers().
1662 *
1663 * In the case of IOMETHOD_SYNC, we start - as we used to before the
1664 * introducing of AIO - the IO in WaitReadBuffers(). This is done as part
1665 * of the retry logic below, no extra code is required.
1666 *
1667 * This path is expected to eventually go away.
1668 */
1669 if (!pgaio_wref_valid(&operation->io_wref) && io_method != IOMETHOD_SYNC)
1670 elog(ERROR, "waiting for read operation that didn't read");
1671
1672 /*
1673 * To handle partial reads, and IOMETHOD_SYNC, we re-issue IO until we're
1674 * done. We may need multiple retries, not just because we could get
1675 * multiple partial reads, but also because some of the remaining
1676 * to-be-read buffers may have been read in by other backends, limiting
1677 * the IO size.
1678 */
1679 while (true)
1680 {
1681 int ignored_nblocks_progress;
1682
1683 CheckReadBuffersOperation(operation, false);
1684
1685 /*
1686 * If there is an IO associated with the operation, we may need to
1687 * wait for it.
1688 */
1689 if (pgaio_wref_valid(&operation->io_wref))
1690 {
1691 /*
1692 * Track the time spent waiting for the IO to complete. As
1693 * tracking a wait even if we don't actually need to wait
1694 *
1695 * a) is not cheap, due to the timestamping overhead
1696 *
1697 * b) reports some time as waiting, even if we never waited
1698 *
1699 * we first check if we already know the IO is complete.
1700 */
1701 if (aio_ret->result.status == PGAIO_RS_UNKNOWN &&
1702 !pgaio_wref_check_done(&operation->io_wref))
1703 {
1704 instr_time io_start = pgstat_prepare_io_time(track_io_timing);
1705
1706 pgaio_wref_wait(&operation->io_wref);
1707
1708 /*
1709 * The IO operation itself was already counted earlier, in
1710 * AsyncReadBuffers(), this just accounts for the wait time.
1711 */
1712 pgstat_count_io_op_time(io_object, io_context, IOOP_READ,
1713 io_start, 0, 0);
1714 }
1715 else
1716 {
1717 Assert(pgaio_wref_check_done(&operation->io_wref));
1718 }
1719
1720 /*
1721 * We now are sure the IO completed. Check the results. This
1722 * includes reporting on errors if there were any.
1723 */
1724 ProcessReadBuffersResult(operation);
1725 }
1726
1727 /*
1728 * Most of the time, the one IO we already started, will read in
1729 * everything. But we need to deal with partial reads and buffers not
1730 * needing IO anymore.
1731 */
1732 if (operation->nblocks_done == operation->nblocks)
1733 break;
1734
1735 CHECK_FOR_INTERRUPTS();
1736
1737 /*
1738 * This may only complete the IO partially, either because some
1739 * buffers were already valid, or because of a partial read.
1740 *
1741 * NB: In contrast to after the AsyncReadBuffers() call in
1742 * StartReadBuffers(), we do *not* reduce
1743 * ReadBuffersOperation->nblocks here, callers expect the full
1744 * operation to be completed at this point (as more operations may
1745 * have been queued).
1746 */
1747 AsyncReadBuffers(operation, &ignored_nblocks_progress);
1748 }
1749
1750 CheckReadBuffersOperation(operation, true);
1751
1752 /* NB: READ_DONE tracepoint was already executed in completion callback */
1753 }
1754
1755 /*
1756 * Initiate IO for the ReadBuffersOperation
1757 *
1758 * This function only starts a single IO at a time. The size of the IO may be
1759 * limited to below the to-be-read blocks, if one of the buffers has
1760 * concurrently been read in. If the first to-be-read buffer is already valid,
1761 * no IO will be issued.
1762 *
1763 * To support retries after partial reads, the first operation->nblocks_done
1764 * buffers are skipped.
1765 *
1766 * On return *nblocks_progress is updated to reflect the number of buffers
1767 * affected by the call. If the first buffer is valid, *nblocks_progress is
1768 * set to 1 and operation->nblocks_done is incremented.
1769 *
1770 * Returns true if IO was initiated, false if no IO was necessary.
1771 */
1772 static bool
1773 AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress)
1774 {
1775 Buffer *buffers = &operation->buffers[0];
1776 int flags = operation->flags;
1777 ForkNumber forknum = operation->forknum;
1778 char persistence = operation->persistence;
1779 int16 nblocks_done = operation->nblocks_done;
1780 BlockNumber blocknum = operation->blocknum + nblocks_done;
1781 Buffer *io_buffers = &operation->buffers[nblocks_done];
1782 int io_buffers_len = 0;
1783 PgAioHandle *ioh;
1784 uint32 ioh_flags = 0;
1785 void *io_pages[MAX_IO_COMBINE_LIMIT];
1786 IOContext io_context;
1787 IOObject io_object;
1788 bool did_start_io;
1789
1790 /*
1791 * When this IO is executed synchronously, either because the caller will
1792 * immediately block waiting for the IO or because IOMETHOD_SYNC is used,
1793 * the AIO subsystem needs to know.
1794 */
1795 if (flags & READ_BUFFERS_SYNCHRONOUSLY)
1796 ioh_flags |= PGAIO_HF_SYNCHRONOUS;
1797
1798 if (persistence == RELPERSISTENCE_TEMP)
1799 {
1800 io_context = IOCONTEXT_NORMAL;
1801 io_object = IOOBJECT_TEMP_RELATION;
1802 ioh_flags |= PGAIO_HF_REFERENCES_LOCAL;
1803 }
1804 else
1805 {
1806 io_context = IOContextForStrategy(operation->strategy);
1807 io_object = IOOBJECT_RELATION;
1808 }
1809
1810 /*
1811 * If zero_damaged_pages is enabled, add the READ_BUFFERS_ZERO_ON_ERROR
1812 * flag. The reason for that is that, hopefully, zero_damaged_pages isn't
1813 * set globally, but on a per-session basis. The completion callback,
1814 * which may be run in other processes, e.g. in IO workers, may have a
1815 * different value of the zero_damaged_pages GUC.
1816 *
1817 * XXX: We probably should eventually use a different flag for
1818 * zero_damaged_pages, so we can report different log levels / error codes
1819 * for zero_damaged_pages and ZERO_ON_ERROR.
1820 */
1821 if (zero_damaged_pages)
1822 flags |= READ_BUFFERS_ZERO_ON_ERROR;
1823
1824 /*
1825 * For the same reason as with zero_damaged_pages we need to use this
1826 * backend's ignore_checksum_failure value.
1827 */
1828 if (ignore_checksum_failure)
1829 flags |= READ_BUFFERS_IGNORE_CHECKSUM_FAILURES;
1830
1831
1832 /*
1833 * To be allowed to report stats in the local completion callback we need
1834 * to prepare to report stats now. This ensures we can safely report the
1835 * checksum failure even in a critical section.
1836 */
1837 pgstat_prepare_report_checksum_failure(operation->smgr->smgr_rlocator.locator.dbOid);
1838
1839 /*
1840 * Get IO handle before ReadBuffersCanStartIO(), as pgaio_io_acquire()
1841 * might block, which we don't want after setting IO_IN_PROGRESS.
1842 *
1843 * If we need to wait for IO before we can get a handle, submit
1844 * already-staged IO first, so that other backends don't need to wait.
1845 * There wouldn't be a deadlock risk, as pgaio_io_acquire() just needs to
1846 * wait for already submitted IO, which doesn't require additional locks,
1847 * but it could still cause undesirable waits.
1848 *
1849 * A secondary benefit is that this would allow us to measure the time in
1850 * pgaio_io_acquire() without causing undue timer overhead in the common,
1851 * non-blocking, case. However, currently the pgstats infrastructure
1852 * doesn't really allow that, as it a) asserts that an operation can't
1853 * have time without operations b) doesn't have an API to report
1854 * "accumulated" time.
1855 */
1856 ioh = pgaio_io_acquire_nb(CurrentResourceOwner, &operation->io_return);
1857 if (unlikely(!ioh))
1858 {
1859 pgaio_submit_staged();
1860
1861 ioh = pgaio_io_acquire(CurrentResourceOwner, &operation->io_return);
1862 }
1863
1864 /*
1865 * Check if we can start IO on the first to-be-read buffer.
1866 *
1867 * If an I/O is already in progress in another backend, we want to wait
1868 * for the outcome: either done, or something went wrong and we will
1869 * retry.
1870 */
1871 if (!ReadBuffersCanStartIO(buffers[nblocks_done], false))
1872 {
1873 /*
1874 * Someone else has already completed this block, we're done.
1875 *
1876 * When IO is necessary, ->nblocks_done is updated in
1877 * ProcessReadBuffersResult(), but that is not called if no IO is
1878 * necessary. Thus update here.
1879 */
1880 operation->nblocks_done += 1;
1881 *nblocks_progress = 1;
1882
1883 pgaio_io_release(ioh);
1884 pgaio_wref_clear(&operation->io_wref);
1885 did_start_io = false;
1886
1887 /*
1888 * Report and track this as a 'hit' for this backend, even though it
1889 * must have started out as a miss in PinBufferForBlock(). The other
1890 * backend will track this as a 'read'.
1891 */
1892 TRACE_POSTGRESQL_BUFFER_READ_DONE(forknum, blocknum,
1893 operation->smgr->smgr_rlocator.locator.spcOid,
1894 operation->smgr->smgr_rlocator.locator.dbOid,
1895 operation->smgr->smgr_rlocator.locator.relNumber,
1896 operation->smgr->smgr_rlocator.backend,
1897 true);
1898
1899 if (persistence == RELPERSISTENCE_TEMP)
1900 pgBufferUsage.local_blks_hit += 1;
1901 else
1902 pgBufferUsage.shared_blks_hit += 1;
1903
1904 if (operation->rel)
1905 pgstat_count_buffer_hit(operation->rel);
1906
1907 pgstat_count_io_op(io_object, io_context, IOOP_HIT, 1, 0);
1908
1909 if (VacuumCostActive)
1910 VacuumCostBalance += VacuumCostPageHit;
1911 }
1912 else
1913 {
1914 instr_time io_start;
1915
1916 /* We found a buffer that we need to read in. */
1917 Assert(io_buffers[0] == buffers[nblocks_done]);
1918 io_pages[0] = BufferGetBlock(buffers[nblocks_done]);
1919 io_buffers_len = 1;
1920
1921 /*
1922 * How many neighboring-on-disk blocks can we scatter-read into other
1923 * buffers at the same time? In this case we don't wait if we see an
1924 * I/O already in progress. We already set BM_IO_IN_PROGRESS for the
1925 * head block, so we should get on with that I/O as soon as possible.
1926 */
1927 for (int i = nblocks_done + 1; i < operation->nblocks; i++)
1928 {
1929 if (!ReadBuffersCanStartIO(buffers[i], true))
1930 break;
1931 /* Must be consecutive block numbers. */
1932 Assert(BufferGetBlockNumber(buffers[i - 1]) ==
1933 BufferGetBlockNumber(buffers[i]) - 1);
1934 Assert(io_buffers[io_buffers_len] == buffers[i]);
1935
1936 io_pages[io_buffers_len++] = BufferGetBlock(buffers[i]);
1937 }
1938
1939 /* get a reference to wait for in WaitReadBuffers() */
1940 pgaio_io_get_wref(ioh, &operation->io_wref);
1941
1942 /* provide the list of buffers to the completion callbacks */
1943 pgaio_io_set_handle_data_32(ioh, (uint32 *) io_buffers, io_buffers_len);
1944
1945 pgaio_io_register_callbacks(ioh,
1946 persistence == RELPERSISTENCE_TEMP ?
1947 PGAIO_HCB_LOCAL_BUFFER_READV :
1948 PGAIO_HCB_SHARED_BUFFER_READV,
1949 flags);
1950
1951 pgaio_io_set_flag(ioh, ioh_flags);
1952
1953 /* ---
1954 * Even though we're trying to issue IO asynchronously, track the time
1955 * in smgrstartreadv():
1956 * - if io_method == IOMETHOD_SYNC, we will always perform the IO
1957 * immediately
1958 * - the io method might not support the IO (e.g. worker IO for a temp
1959 * table)
1960 * ---
1961 */
1962 io_start = pgstat_prepare_io_time(track_io_timing);
1963 smgrstartreadv(ioh, operation->smgr, forknum,
1964 blocknum,
1965 io_pages, io_buffers_len);
1966 pgstat_count_io_op_time(io_object, io_context, IOOP_READ,
1967 io_start, 1, io_buffers_len * BLCKSZ);
1968
1969 if (persistence == RELPERSISTENCE_TEMP)
1970 pgBufferUsage.local_blks_read += io_buffers_len;
1971 else
1972 pgBufferUsage.shared_blks_read += io_buffers_len;
1973
1974 /*
1975 * Track vacuum cost when issuing IO, not after waiting for it.
1976 * Otherwise we could end up issuing a lot of IO in a short timespan,
1977 * despite a low cost limit.
1978 */
1979 if (VacuumCostActive)
1980 VacuumCostBalance += VacuumCostPageMiss * io_buffers_len;
1981
1982 *nblocks_progress = io_buffers_len;
1983 did_start_io = true;
1984 }
1985
1986 return did_start_io;
1987 }
1988
1989 /*
1990 * BufferAlloc -- subroutine for PinBufferForBlock. Handles lookup of a shared
1991 * buffer. If no buffer exists already, selects a replacement victim and
1992 * evicts the old page, but does NOT read in new page.
1993 *
1994 * "strategy" can be a buffer replacement strategy object, or NULL for
1995 * the default strategy. The selected buffer's usage_count is advanced when
1996 * using the default strategy, but otherwise possibly not (see PinBuffer).
1997 *
1998 * The returned buffer is pinned and is already marked as holding the
1999 * desired page. If it already did have the desired page, *foundPtr is
2000 * set true. Otherwise, *foundPtr is set false.
2001 *
2002 * io_context is passed as an output parameter to avoid calling
2003 * IOContextForStrategy() when there is a shared buffers hit and no IO
2004 * statistics need be captured.
2005 *
2006 * No locks are held either at entry or exit.
2007 */
2008 static pg_always_inline BufferDesc *
2009 BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
2010 BlockNumber blockNum,
2011 BufferAccessStrategy strategy,
2012 bool *foundPtr, IOContext io_context)
2013 {
2014 BufferTag newTag; /* identity of requested block */
2015 uint32 newHash; /* hash value for newTag */
2016 LWLock *newPartitionLock; /* buffer partition lock for it */
2017 int existing_buf_id;
2018 Buffer victim_buffer;
2019 BufferDesc *victim_buf_hdr;
2020 uint32 victim_buf_state;
2021
2022 /* Make sure we will have room to remember the buffer pin */
2023 ResourceOwnerEnlarge(CurrentResourceOwner);
2024 ReservePrivateRefCountEntry();
2025
2026 /* create a tag so we can lookup the buffer */
2027 InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
2028
2029 /* determine its hash code and partition lock ID */
2030 newHash = BufTableHashCode(&newTag);
2031 newPartitionLock = BufMappingPartitionLock(newHash);
2032
2033 /* see if the block is in the buffer pool already */
2034 LWLockAcquire(newPartitionLock, LW_SHARED);
2035 existing_buf_id = BufTableLookup(&newTag, newHash);
2036 if (existing_buf_id >= 0)
2037 {
2038 BufferDesc *buf;
2039 bool valid;
2040
2041 /*
2042 * Found it. Now, pin the buffer so no one can steal it from the
2043 * buffer pool, and check to see if the correct data has been loaded
2044 * into the buffer.
2045 */
2046 buf = GetBufferDescriptor(existing_buf_id);
2047
2048 valid = PinBuffer(buf, strategy);
2049
2050 /* Can release the mapping lock as soon as we've pinned it */
2051 LWLockRelease(newPartitionLock);
2052
2053 *foundPtr = true;
2054
2055 if (!valid)
2056 {
2057 /*
2058 * We can only get here if (a) someone else is still reading in
2059 * the page, (b) a previous read attempt failed, or (c) someone
2060 * called StartReadBuffers() but not yet WaitReadBuffers().
2061 */
2062 *foundPtr = false;
2063 }
2064
2065 return buf;
2066 }
2067
2068 /*
2069 * Didn't find it in the buffer pool. We'll have to initialize a new
2070 * buffer. Remember to unlock the mapping lock while doing the work.
2071 */
2072 LWLockRelease(newPartitionLock);
2073
2074 /*
2075 * Acquire a victim buffer. Somebody else might try to do the same, we
2076 * don't hold any conflicting locks. If so we'll have to undo our work
2077 * later.
2078 */
2079 victim_buffer = GetVictimBuffer(strategy, io_context);
2080 victim_buf_hdr = GetBufferDescriptor(victim_buffer - 1);
2081
2082 /*
2083 * Try to make a hashtable entry for the buffer under its new tag. If
2084 * somebody else inserted another buffer for the tag, we'll release the
2085 * victim buffer we acquired and use the already inserted one.
2086 */
2087 LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
2088 existing_buf_id = BufTableInsert(&newTag, newHash, victim_buf_hdr->buf_id);
2089 if (existing_buf_id >= 0)
2090 {
2091 BufferDesc *existing_buf_hdr;
2092 bool valid;
2093
2094 /*
2095 * Got a collision. Someone has already done what we were about to do.
2096 * We'll just handle this as if it were found in the buffer pool in
2097 * the first place. First, give up the buffer we were planning to
2098 * use.
2099 *
2100 * We could do this after releasing the partition lock, but then we'd
2101 * have to call ResourceOwnerEnlarge() & ReservePrivateRefCountEntry()
2102 * before acquiring the lock, for the rare case of such a collision.
2103 */
2104 UnpinBuffer(victim_buf_hdr);
2105
2106 /*
2107 * The victim buffer we acquired previously is clean and unused, let
2108 * it be found again quickly
2109 */
2110 StrategyFreeBuffer(victim_buf_hdr);
2111
2112 /* remaining code should match code at top of routine */
2113
2114 existing_buf_hdr = GetBufferDescriptor(existing_buf_id);
2115
2116 valid = PinBuffer(existing_buf_hdr, strategy);
2117
2118 /* Can release the mapping lock as soon as we've pinned it */
2119 LWLockRelease(newPartitionLock);
2120
2121 *foundPtr = true;
2122
2123 if (!valid)
2124 {
2125 /*
2126 * We can only get here if (a) someone else is still reading in
2127 * the page, (b) a previous read attempt failed, or (c) someone
2128 * called StartReadBuffers() but not yet WaitReadBuffers().
2129 */
2130 *foundPtr = false;
2131 }
2132
2133 return existing_buf_hdr;
2134 }
2135
2136 /*
2137 * Need to lock the buffer header too in order to change its tag.
2138 */
2139 victim_buf_state = LockBufHdr(victim_buf_hdr);
2140
2141 /* some sanity checks while we hold the buffer header lock */
2142 Assert(BUF_STATE_GET_REFCOUNT(victim_buf_state) == 1);
2143 Assert(!(victim_buf_state & (BM_TAG_VALID | BM_VALID | BM_DIRTY | BM_IO_IN_PROGRESS)));
2144
2145 victim_buf_hdr->tag = newTag;
2146
2147 /*
2148 * Make sure BM_PERMANENT is set for buffers that must be written at every
2149 * checkpoint. Unlogged buffers only need to be written at shutdown
2150 * checkpoints, except for their "init" forks, which need to be treated
2151 * just like permanent relations.
2152 */
2153 victim_buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
2154 if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM)
2155 victim_buf_state |= BM_PERMANENT;
2156
2157 UnlockBufHdr(victim_buf_hdr, victim_buf_state);
2158
2159 LWLockRelease(newPartitionLock);
2160
2161 /*
2162 * Buffer contents are currently invalid.
2163 */
2164 *foundPtr = false;
2165
2166 return victim_buf_hdr;
2167 }
2168
2169 /*
2170 * InvalidateBuffer -- mark a shared buffer invalid and return it to the
2171 * freelist.
2172 *
2173 * The buffer header spinlock must be held at entry. We drop it before
2174 * returning. (This is sane because the caller must have locked the
2175 * buffer in order to be sure it should be dropped.)
2176 *
2177 * This is used only in contexts such as dropping a relation. We assume
2178 * that no other backend could possibly be interested in using the page,
2179 * so the only reason the buffer might be pinned is if someone else is
2180 * trying to write it out. We have to let them finish before we can
2181 * reclaim the buffer.
2182 *
2183 * The buffer could get reclaimed by someone else while we are waiting
2184 * to acquire the necessary locks; if so, don't mess it up.
2185 */
2186 static void
2187 InvalidateBuffer(BufferDesc *buf)
2188 {
2189 BufferTag oldTag;
2190 uint32 oldHash; /* hash value for oldTag */
2191 LWLock *oldPartitionLock; /* buffer partition lock for it */
2192 uint32 oldFlags;
2193 uint32 buf_state;
2194
2195 /* Save the original buffer tag before dropping the spinlock */
2196 oldTag = buf->tag;
2197
2198 buf_state = pg_atomic_read_u32(&buf->state);
2199 Assert(buf_state & BM_LOCKED);
2200 UnlockBufHdr(buf, buf_state);
2201
2202 /*
2203 * Need to compute the old tag's hashcode and partition lock ID. XXX is it
2204 * worth storing the hashcode in BufferDesc so we need not recompute it
2205 * here? Probably not.
2206 */
2207 oldHash = BufTableHashCode(&oldTag);
2208 oldPartitionLock = BufMappingPartitionLock(oldHash);
2209
2210 retry:
2211
2212 /*
2213 * Acquire exclusive mapping lock in preparation for changing the buffer's
2214 * association.
2215 */
2216 LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
2217
2218 /* Re-lock the buffer header */
2219 buf_state = LockBufHdr(buf);
2220
2221 /* If it's changed while we were waiting for lock, do nothing */
2222 if (!BufferTagsEqual(&buf->tag, &oldTag))
2223 {
2224 UnlockBufHdr(buf, buf_state);
2225 LWLockRelease(oldPartitionLock);
2226 return;
2227 }
2228
2229 /*
2230 * We assume the reason for it to be pinned is that either we were
2231 * asynchronously reading the page in before erroring out or someone else
2232 * is flushing the page out. Wait for the IO to finish. (This could be
2233 * an infinite loop if the refcount is messed up... it would be nice to
2234 * time out after awhile, but there seems no way to be sure how many loops
2235 * may be needed. Note that if the other guy has pinned the buffer but
2236 * not yet done StartBufferIO, WaitIO will fall through and we'll
2237 * effectively be busy-looping here.)
2238 */
2239 if (BUF_STATE_GET_REFCOUNT(buf_state) != 0)
2240 {
2241 UnlockBufHdr(buf, buf_state);
2242 LWLockRelease(oldPartitionLock);
2243 /* safety check: should definitely not be our *own* pin */
2244 if (GetPrivateRefCount(BufferDescriptorGetBuffer(buf)) > 0)
2245 elog(ERROR, "buffer is pinned in InvalidateBuffer");
2246 WaitIO(buf);
2247 goto retry;
2248 }
2249
2250 /*
2251 * Clear out the buffer's tag and flags. We must do this to ensure that
2252 * linear scans of the buffer array don't think the buffer is valid.
2253 */
2254 oldFlags = buf_state & BUF_FLAG_MASK;
2255 ClearBufferTag(&buf->tag);
2256 buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
2257 UnlockBufHdr(buf, buf_state);
2258
2259 /*
2260 * Remove the buffer from the lookup hashtable, if it was in there.
2261 */
2262 if (oldFlags & BM_TAG_VALID)
2263 BufTableDelete(&oldTag, oldHash);
2264
2265 /*
2266 * Done with mapping lock.
2267 */
2268 LWLockRelease(oldPartitionLock);
2269
2270 /*
2271 * Insert the buffer at the head of the list of free buffers.
2272 */
2273 StrategyFreeBuffer(buf);
2274 }
2275
2276 /*
2277 * Helper routine for GetVictimBuffer()
2278 *
2279 * Needs to be called on a buffer with a valid tag, pinned, but without the
2280 * buffer header spinlock held.
2281 *
2282 * Returns true if the buffer can be reused, in which case the buffer is only
2283 * pinned by this backend and marked as invalid, false otherwise.
2284 */
2285 static bool
2286 InvalidateVictimBuffer(BufferDesc *buf_hdr)
2287 {
2288 uint32 buf_state;
2289 uint32 hash;
2290 LWLock *partition_lock;
2291 BufferTag tag;
2292
2293 Assert(GetPrivateRefCount(BufferDescriptorGetBuffer(buf_hdr)) == 1);
2294
2295 /* have buffer pinned, so it's safe to read tag without lock */
2296 tag = buf_hdr->tag;
2297
2298 hash = BufTableHashCode(&tag);
2299 partition_lock = BufMappingPartitionLock(hash);
2300
2301 LWLockAcquire(partition_lock, LW_EXCLUSIVE);
2302
2303 /* lock the buffer header */
2304 buf_state = LockBufHdr(buf_hdr);
2305
2306 /*
2307 * We have the buffer pinned nobody else should have been able to unset
2308 * this concurrently.
2309 */
2310 Assert(buf_state & BM_TAG_VALID);
2311 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
2312 Assert(BufferTagsEqual(&buf_hdr->tag, &tag));
2313
2314 /*
2315 * If somebody else pinned the buffer since, or even worse, dirtied it,
2316 * give up on this buffer: It's clearly in use.
2317 */
2318 if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY))
2319 {
2320 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
2321
2322 UnlockBufHdr(buf_hdr, buf_state);
2323 LWLockRelease(partition_lock);
2324
2325 return false;
2326 }
2327
2328 /*
2329 * Clear out the buffer's tag and flags and usagecount. This is not
2330 * strictly required, as BM_TAG_VALID/BM_VALID needs to be checked before
2331 * doing anything with the buffer. But currently it's beneficial, as the
2332 * cheaper pre-check for several linear scans of shared buffers use the
2333 * tag (see e.g. FlushDatabaseBuffers()).
2334 */
2335 ClearBufferTag(&buf_hdr->tag);
2336 buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
2337 UnlockBufHdr(buf_hdr, buf_state);
2338
2339 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
2340
2341 /* finally delete buffer from the buffer mapping table */
2342 BufTableDelete(&tag, hash);
2343
2344 LWLockRelease(partition_lock);
2345
2346 Assert(!(buf_state & (BM_DIRTY | BM_VALID | BM_TAG_VALID)));
2347 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
2348 Assert(BUF_STATE_GET_REFCOUNT(pg_atomic_read_u32(&buf_hdr->state)) > 0);
2349
2350 return true;
2351 }
2352
2353 static Buffer
2354 GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context)
2355 {
2356 BufferDesc *buf_hdr;
2357 Buffer buf;
2358 uint32 buf_state;
2359 bool from_ring;
2360
2361 /*
2362 * Ensure, while the spinlock's not yet held, that there's a free refcount
2363 * entry, and a resource owner slot for the pin.
2364 */
2365 ReservePrivateRefCountEntry();
2366 ResourceOwnerEnlarge(CurrentResourceOwner);
2367
2368 /* we return here if a prospective victim buffer gets used concurrently */
2369 again:
2370
2371 /*
2372 * Select a victim buffer. The buffer is returned with its header
2373 * spinlock still held!
2374 */
2375 buf_hdr = StrategyGetBuffer(strategy, &buf_state, &from_ring);
2376 buf = BufferDescriptorGetBuffer(buf_hdr);
2377
2378 Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 0);
2379
2380 /* Pin the buffer and then release the buffer spinlock */
2381 PinBuffer_Locked(buf_hdr);
2382
2383 /*
2384 * We shouldn't have any other pins for this buffer.
2385 */
2386 CheckBufferIsPinnedOnce(buf);
2387
2388 /*
2389 * If the buffer was dirty, try to write it out. There is a race
2390 * condition here, in that someone might dirty it after we released the
2391 * buffer header lock above, or even while we are writing it out (since
2392 * our share-lock won't prevent hint-bit updates). We will recheck the
2393 * dirty bit after re-locking the buffer header.
2394 */
2395 if (buf_state & BM_DIRTY)
2396 {
2397 LWLock *content_lock;
2398
2399 Assert(buf_state & BM_TAG_VALID);
2400 Assert(buf_state & BM_VALID);
2401
2402 /*
2403 * We need a share-lock on the buffer contents to write it out (else
2404 * we might write invalid data, eg because someone else is compacting
2405 * the page contents while we write). We must use a conditional lock
2406 * acquisition here to avoid deadlock. Even though the buffer was not
2407 * pinned (and therefore surely not locked) when StrategyGetBuffer
2408 * returned it, someone else could have pinned and exclusive-locked it
2409 * by the time we get here. If we try to get the lock unconditionally,
2410 * we'd block waiting for them; if they later block waiting for us,
2411 * deadlock ensues. (This has been observed to happen when two
2412 * backends are both trying to split btree index pages, and the second
2413 * one just happens to be trying to split the page the first one got
2414 * from StrategyGetBuffer.)
2415 */
2416 content_lock = BufferDescriptorGetContentLock(buf_hdr);
2417 if (!LWLockConditionalAcquire(content_lock, LW_SHARED))
2418 {
2419 /*
2420 * Someone else has locked the buffer, so give it up and loop back
2421 * to get another one.
2422 */
2423 UnpinBuffer(buf_hdr);
2424 goto again;
2425 }
2426
2427 /*
2428 * If using a nondefault strategy, and writing the buffer would
2429 * require a WAL flush, let the strategy decide whether to go ahead
2430 * and write/reuse the buffer or to choose another victim. We need a
2431 * lock to inspect the page LSN, so this can't be done inside
2432 * StrategyGetBuffer.
2433 */
2434 if (strategy != NULL)
2435 {
2436 XLogRecPtr lsn;
2437
2438 /* Read the LSN while holding buffer header lock */
2439 buf_state = LockBufHdr(buf_hdr);
2440 lsn = BufferGetLSN(buf_hdr);
2441 UnlockBufHdr(buf_hdr, buf_state);
2442
2443 if (XLogNeedsFlush(lsn)
2444 && StrategyRejectBuffer(strategy, buf_hdr, from_ring))
2445 {
2446 LWLockRelease(content_lock);
2447 UnpinBuffer(buf_hdr);
2448 goto again;
2449 }
2450 }
2451
2452 /* OK, do the I/O */
2453 FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context);
2454 LWLockRelease(content_lock);
2455
2456 ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context,
2457 &buf_hdr->tag);
2458 }
2459
2460
2461 if (buf_state & BM_VALID)
2462 {
2463 /*
2464 * When a BufferAccessStrategy is in use, blocks evicted from shared
2465 * buffers are counted as IOOP_EVICT in the corresponding context
2466 * (e.g. IOCONTEXT_BULKWRITE). Shared buffers are evicted by a
2467 * strategy in two cases: 1) while initially claiming buffers for the
2468 * strategy ring 2) to replace an existing strategy ring buffer
2469 * because it is pinned or in use and cannot be reused.
2470 *
2471 * Blocks evicted from buffers already in the strategy ring are
2472 * counted as IOOP_REUSE in the corresponding strategy context.
2473 *
2474 * At this point, we can accurately count evictions and reuses,
2475 * because we have successfully claimed the valid buffer. Previously,
2476 * we may have been forced to release the buffer due to concurrent
2477 * pinners or erroring out.
2478 */
2479 pgstat_count_io_op(IOOBJECT_RELATION, io_context,
2480 from_ring ? IOOP_REUSE : IOOP_EVICT, 1, 0);
2481 }
2482
2483 /*
2484 * If the buffer has an entry in the buffer mapping table, delete it. This
2485 * can fail because another backend could have pinned or dirtied the
2486 * buffer.
2487 */
2488 if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr))
2489 {
2490 UnpinBuffer(buf_hdr);
2491 goto again;
2492 }
2493
2494 /* a final set of sanity checks */
2495 #ifdef USE_ASSERT_CHECKING
2496 buf_state = pg_atomic_read_u32(&buf_hdr->state);
2497
2498 Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 1);
2499 Assert(!(buf_state & (BM_TAG_VALID | BM_VALID | BM_DIRTY)));
2500
2501 CheckBufferIsPinnedOnce(buf);
2502 #endif
2503
2504 return buf;
2505 }
2506
2507 /*
2508 * Return the maximum number of buffers that a backend should try to pin once,
2509 * to avoid exceeding its fair share. This is the highest value that
2510 * GetAdditionalPinLimit() could ever return. Note that it may be zero on a
2511 * system with a very small buffer pool relative to max_connections.
2512 */
2513 uint32
2514 GetPinLimit(void)
2515 {
2516 return MaxProportionalPins;
2517 }
2518
2519 /*
2520 * Return the maximum number of additional buffers that this backend should
2521 * pin if it wants to stay under the per-backend limit, considering the number
2522 * of buffers it has already pinned. Unlike LimitAdditionalPins(), the limit
2523 * return by this function can be zero.
2524 */
2525 uint32
2526 GetAdditionalPinLimit(void)
2527 {
2528 uint32 estimated_pins_held;
2529
2530 /*
2531 * We get the number of "overflowed" pins for free, but don't know the
2532 * number of pins in PrivateRefCountArray. The cost of calculating that
2533 * exactly doesn't seem worth it, so just assume the max.
2534 */
2535 estimated_pins_held = PrivateRefCountOverflowed + REFCOUNT_ARRAY_ENTRIES;
2536
2537 /* Is this backend already holding more than its fair share? */
2538 if (estimated_pins_held > MaxProportionalPins)
2539 return 0;
2540
2541 return MaxProportionalPins - estimated_pins_held;
2542 }
2543
2544 /*
2545 * Limit the number of pins a batch operation may additionally acquire, to
2546 * avoid running out of pinnable buffers.
2547 *
2548 * One additional pin is always allowed, on the assumption that the operation
2549 * requires at least one to make progress.
2550 */
2551 void
2552 LimitAdditionalPins(uint32 *additional_pins)
2553 {
2554 uint32 limit;
2555
2556 if (*additional_pins <= 1)
2557 return;
2558
2559 limit = GetAdditionalPinLimit();
2560 limit = Max(limit, 1);
2561 if (limit < *additional_pins)
2562 *additional_pins = limit;
2563 }
2564
2565 /*
2566 * Logic shared between ExtendBufferedRelBy(), ExtendBufferedRelTo(). Just to
2567 * avoid duplicating the tracing and relpersistence related logic.
2568 */
2569 static BlockNumber
2570 ExtendBufferedRelCommon(BufferManagerRelation bmr,
2571 ForkNumber fork,
2572 BufferAccessStrategy strategy,
2573 uint32 flags,
2574 uint32 extend_by,
2575 BlockNumber extend_upto,
2576 Buffer *buffers,
2577 uint32 *extended_by)
2578 {
2579 BlockNumber first_block;
2580
2581 TRACE_POSTGRESQL_BUFFER_EXTEND_START(fork,
2582 bmr.smgr->smgr_rlocator.locator.spcOid,
2583 bmr.smgr->smgr_rlocator.locator.dbOid,
2584 bmr.smgr->smgr_rlocator.locator.relNumber,
2585 bmr.smgr->smgr_rlocator.backend,
2586 extend_by);
2587
2588 if (bmr.relpersistence == RELPERSISTENCE_TEMP)
2589 {
2590 /*
2591 * Reject attempts to extend non-local temporary relations; we have no
2592 * ability to transfer about-to-be-created local buffers into the
2593 * owning session's local buffers. This is the canonical place for
2594 * the check, covering any attempt to extend a non-local temporary
2595 * relation.
2596 */
2597 if (bmr.rel && RELATION_IS_OTHER_TEMP(bmr.rel))
2598 ereport(ERROR,
2599 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
2600 errmsg("cannot access temporary tables of other sessions")));
2601
2602 first_block = ExtendBufferedRelLocal(bmr, fork, flags,
2603 extend_by, extend_upto,
2604 buffers, &extend_by);
2605 }
2606 else
2607 first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags,
2608 extend_by, extend_upto,
2609 buffers, &extend_by);
2610 *extended_by = extend_by;
2611
2612 TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork,
2613 bmr.smgr->smgr_rlocator.locator.spcOid,
2614 bmr.smgr->smgr_rlocator.locator.dbOid,
2615 bmr.smgr->smgr_rlocator.locator.relNumber,
2616 bmr.smgr->smgr_rlocator.backend,
2617 *extended_by,
2618 first_block);
2619
2620 return first_block;
2621 }
2622
2623 /*
2624 * Implementation of ExtendBufferedRelBy() and ExtendBufferedRelTo() for
2625 * shared buffers.
2626 */
2627 static BlockNumber
2628 ExtendBufferedRelShared(BufferManagerRelation bmr,
2629 ForkNumber fork,
2630 BufferAccessStrategy strategy,
2631 uint32 flags,
2632 uint32 extend_by,
2633 BlockNumber extend_upto,
2634 Buffer *buffers,
2635 uint32 *extended_by)
2636 {
2637 BlockNumber first_block;
2638 IOContext io_context = IOContextForStrategy(strategy);
2639 instr_time io_start;
2640
2641 LimitAdditionalPins(&extend_by);
2642
2643 /*
2644 * Acquire victim buffers for extension without holding extension lock.
2645 * Writing out victim buffers is the most expensive part of extending the
2646 * relation, particularly when doing so requires WAL flushes. Zeroing out
2647 * the buffers is also quite expensive, so do that before holding the
2648 * extension lock as well.
2649 *
2650 * These pages are pinned by us and not valid. While we hold the pin they
2651 * can't be acquired as victim buffers by another backend.
2652 */
2653 for (uint32 i = 0; i < extend_by; i++)
2654 {
2655 Block buf_block;
2656
2657 buffers[i] = GetVictimBuffer(strategy, io_context);
2658 buf_block = BufHdrGetBlock(GetBufferDescriptor(buffers[i] - 1));
2659
2660 /* new buffers are zero-filled */
2661 MemSet(buf_block, 0, BLCKSZ);
2662 }
2663
2664 /*
2665 * Lock relation against concurrent extensions, unless requested not to.
2666 *
2667 * We use the same extension lock for all forks. That's unnecessarily
2668 * restrictive, but currently extensions for forks don't happen often
2669 * enough to make it worth locking more granularly.
2670 *
2671 * Note that another backend might have extended the relation by the time
2672 * we get the lock.
2673 */
2674 if (!(flags & EB_SKIP_EXTENSION_LOCK))
2675 LockRelationForExtension(bmr.rel, ExclusiveLock);
2676
2677 /*
2678 * If requested, invalidate size cache, so that smgrnblocks asks the
2679 * kernel.
2680 */
2681 if (flags & EB_CLEAR_SIZE_CACHE)
2682 bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber;
2683
2684 first_block = smgrnblocks(bmr.smgr, fork);
2685
2686 /*
2687 * Now that we have the accurate relation size, check if the caller wants
2688 * us to extend to only up to a specific size. If there were concurrent
2689 * extensions, we might have acquired too many buffers and need to release
2690 * them.
2691 */
2692 if (extend_upto != InvalidBlockNumber)
2693 {
2694 uint32 orig_extend_by = extend_by;
2695
2696 if (first_block > extend_upto)
2697 extend_by = 0;
2698 else if ((uint64) first_block + extend_by > extend_upto)
2699 extend_by = extend_upto - first_block;
2700
2701 for (uint32 i = extend_by; i < orig_extend_by; i++)
2702 {
2703 BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1);
2704
2705 /*
2706 * The victim buffer we acquired previously is clean and unused,
2707 * let it be found again quickly
2708 */
2709 StrategyFreeBuffer(buf_hdr);
2710 UnpinBuffer(buf_hdr);
2711 }
2712
2713 if (extend_by == 0)
2714 {
2715 if (!(flags & EB_SKIP_EXTENSION_LOCK))
2716 UnlockRelationForExtension(bmr.rel, ExclusiveLock);
2717 *extended_by = extend_by;
2718 return first_block;
2719 }
2720 }
2721
2722 /* Fail if relation is already at maximum possible length */
2723 if ((uint64) first_block + extend_by >= MaxBlockNumber)
2724 ereport(ERROR,
2725 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2726 errmsg("cannot extend relation %s beyond %u blocks",
2727 relpath(bmr.smgr->smgr_rlocator, fork).str,
2728 MaxBlockNumber)));
2729
2730 /*
2731 * Insert buffers into buffer table, mark as IO_IN_PROGRESS.
2732 *
2733 * This needs to happen before we extend the relation, because as soon as
2734 * we do, other backends can start to read in those pages.
2735 */
2736 for (uint32 i = 0; i < extend_by; i++)
2737 {
2738 Buffer victim_buf = buffers[i];
2739 BufferDesc *victim_buf_hdr = GetBufferDescriptor(victim_buf - 1);
2740 BufferTag tag;
2741 uint32 hash;
2742 LWLock *partition_lock;
2743 int existing_id;
2744
2745 /* in case we need to pin an existing buffer below */
2746 ResourceOwnerEnlarge(CurrentResourceOwner);
2747 ReservePrivateRefCountEntry();
2748
2749 InitBufferTag(&tag, &bmr.smgr->smgr_rlocator.locator, fork, first_block + i);
2750 hash = BufTableHashCode(&tag);
2751 partition_lock = BufMappingPartitionLock(hash);
2752
2753 LWLockAcquire(partition_lock, LW_EXCLUSIVE);
2754
2755 existing_id = BufTableInsert(&tag, hash, victim_buf_hdr->buf_id);
2756
2757 /*
2758 * We get here only in the corner case where we are trying to extend
2759 * the relation but we found a pre-existing buffer. This can happen
2760 * because a prior attempt at extending the relation failed, and
2761 * because mdread doesn't complain about reads beyond EOF (when
2762 * zero_damaged_pages is ON) and so a previous attempt to read a block
2763 * beyond EOF could have left a "valid" zero-filled buffer.
2764 * Unfortunately, we have also seen this case occurring because of
2765 * buggy Linux kernels that sometimes return an lseek(SEEK_END) result
2766 * that doesn't account for a recent write. In that situation, the
2767 * pre-existing buffer would contain valid data that we don't want to
2768 * overwrite. Since the legitimate cases should always have left a
2769 * zero-filled buffer, complain if not PageIsNew.
2770 */
2771 if (existing_id >= 0)
2772 {
2773 BufferDesc *existing_hdr = GetBufferDescriptor(existing_id);
2774 Block buf_block;
2775 bool valid;
2776
2777 /*
2778 * Pin the existing buffer before releasing the partition lock,
2779 * preventing it from being evicted.
2780 */
2781 valid = PinBuffer(existing_hdr, strategy);
2782
2783 LWLockRelease(partition_lock);
2784
2785 /*
2786 * The victim buffer we acquired previously is clean and unused,
2787 * let it be found again quickly
2788 */
2789 StrategyFreeBuffer(victim_buf_hdr);
2790 UnpinBuffer(victim_buf_hdr);
2791
2792 buffers[i] = BufferDescriptorGetBuffer(existing_hdr);
2793 buf_block = BufHdrGetBlock(existing_hdr);
2794
2795 if (valid && !PageIsNew((Page) buf_block))
2796 ereport(ERROR,
2797 (errmsg("unexpected data beyond EOF in block %u of relation \"%s\"",
2798 existing_hdr->tag.blockNum,
2799 relpath(bmr.smgr->smgr_rlocator, fork).str),
2800 errhint("This has been seen to occur with buggy kernels; consider updating your system.")));
2801
2802 /*
2803 * We *must* do smgr[zero]extend before succeeding, else the page
2804 * will not be reserved by the kernel, and the next P_NEW call
2805 * will decide to return the same page. Clear the BM_VALID bit,
2806 * do StartBufferIO() and proceed.
2807 *
2808 * Loop to handle the very small possibility that someone re-sets
2809 * BM_VALID between our clearing it and StartBufferIO inspecting
2810 * it.
2811 */
2812 do
2813 {
2814 uint32 buf_state = LockBufHdr(existing_hdr);
2815
2816 buf_state &= ~BM_VALID;
2817 UnlockBufHdr(existing_hdr, buf_state);
2818 } while (!StartBufferIO(existing_hdr, true, false));
2819 }
2820 else
2821 {
2822 uint32 buf_state;
2823
2824 buf_state = LockBufHdr(victim_buf_hdr);
2825
2826 /* some sanity checks while we hold the buffer header lock */
2827 Assert(!(buf_state & (BM_VALID | BM_TAG_VALID | BM_DIRTY | BM_JUST_DIRTIED)));
2828 Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 1);
2829
2830 victim_buf_hdr->tag = tag;
2831
2832 buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
2833 if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM)
2834 buf_state |= BM_PERMANENT;
2835
2836 UnlockBufHdr(victim_buf_hdr, buf_state);
2837
2838 LWLockRelease(partition_lock);
2839
2840 /* XXX: could combine the locked operations in it with the above */
2841 StartBufferIO(victim_buf_hdr, true, false);
2842 }
2843 }
2844
2845 io_start = pgstat_prepare_io_time(track_io_timing);
2846
2847 /*
2848 * Note: if smgrzeroextend fails, we will end up with buffers that are
2849 * allocated but not marked BM_VALID. The next relation extension will
2850 * still select the same block number (because the relation didn't get any
2851 * longer on disk) and so future attempts to extend the relation will find
2852 * the same buffers (if they have not been recycled) but come right back
2853 * here to try smgrzeroextend again.
2854 *
2855 * We don't need to set checksum for all-zero pages.
2856 */
2857 smgrzeroextend(bmr.smgr, fork, first_block, extend_by, false);
2858
2859 /*
2860 * Release the file-extension lock; it's now OK for someone else to extend
2861 * the relation some more.
2862 *
2863 * We remove IO_IN_PROGRESS after this, as waking up waiting backends can
2864 * take noticeable time.
2865 */
2866 if (!(flags & EB_SKIP_EXTENSION_LOCK))
2867 UnlockRelationForExtension(bmr.rel, ExclusiveLock);
2868
2869 pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND,
2870 io_start, 1, extend_by * BLCKSZ);
2871
2872 /* Set BM_VALID, terminate IO, and wake up any waiters */
2873 for (uint32 i = 0; i < extend_by; i++)
2874 {
2875 Buffer buf = buffers[i];
2876 BufferDesc *buf_hdr = GetBufferDescriptor(buf - 1);
2877 bool lock = false;
2878
2879 if (flags & EB_LOCK_FIRST && i == 0)
2880 lock = true;
2881 else if (flags & EB_LOCK_TARGET)
2882 {
2883 Assert(extend_upto != InvalidBlockNumber);
2884 if (first_block + i + 1 == extend_upto)
2885 lock = true;
2886 }
2887
2888 if (lock)
2889 LWLockAcquire(BufferDescriptorGetContentLock(buf_hdr), LW_EXCLUSIVE);
2890
2891 TerminateBufferIO(buf_hdr, false, BM_VALID, true, false);
2892 }
2893
2894 pgBufferUsage.shared_blks_written += extend_by;
2895
2896 *extended_by = extend_by;
2897
2898 return first_block;
2899 }
2900
2901 /*
2902 * BufferIsExclusiveLocked
2903 *
2904 * Checks if buffer is exclusive-locked.
2905 *
2906 * Buffer must be pinned.
2907 */
2908 bool
2909 BufferIsExclusiveLocked(Buffer buffer)
2910 {
2911 BufferDesc *bufHdr;
2912
2913 Assert(BufferIsPinned(buffer));
2914
2915 if (BufferIsLocal(buffer))
2916 {
2917 /* Content locks are not maintained for local buffers. */
2918 return true;
2919 }
2920 else
2921 {
2922 bufHdr = GetBufferDescriptor(buffer - 1);
2923 return LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
2924 LW_EXCLUSIVE);
2925 }
2926 }
2927
2928 /*
2929 * BufferIsDirty
2930 *
2931 * Checks if buffer is already dirty.
2932 *
2933 * Buffer must be pinned and exclusive-locked. (Without an exclusive lock,
2934 * the result may be stale before it's returned.)
2935 */
2936 bool
2937 BufferIsDirty(Buffer buffer)
2938 {
2939 BufferDesc *bufHdr;
2940
2941 Assert(BufferIsPinned(buffer));
2942
2943 if (BufferIsLocal(buffer))
2944 {
2945 int bufid = -buffer - 1;
2946
2947 bufHdr = GetLocalBufferDescriptor(bufid);
2948 /* Content locks are not maintained for local buffers. */
2949 }
2950 else
2951 {
2952 bufHdr = GetBufferDescriptor(buffer - 1);
2953 Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
2954 LW_EXCLUSIVE));
2955 }
2956
2957 return pg_atomic_read_u32(&bufHdr->state) & BM_DIRTY;
2958 }
2959
2960 /*
2961 * MarkBufferDirty
2962 *
2963 * Marks buffer contents as dirty (actual write happens later).
2964 *
2965 * Buffer must be pinned and exclusive-locked. (If caller does not hold
2966 * exclusive lock, then somebody could be in process of writing the buffer,
2967 * leading to risk of bad data written to disk.)
2968 */
2969 void
2970 MarkBufferDirty(Buffer buffer)
2971 {
2972 BufferDesc *bufHdr;
2973 uint32 buf_state;
2974 uint32 old_buf_state;
2975
2976 if (!BufferIsValid(buffer))
2977 elog(ERROR, "bad buffer ID: %d", buffer);
2978
2979 if (BufferIsLocal(buffer))
2980 {
2981 MarkLocalBufferDirty(buffer);
2982 return;
2983 }
2984
2985 bufHdr = GetBufferDescriptor(buffer - 1);
2986
2987 Assert(BufferIsPinned(buffer));
2988 Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
2989 LW_EXCLUSIVE));
2990
2991 old_buf_state = pg_atomic_read_u32(&bufHdr->state);
2992 for (;;)
2993 {
2994 if (old_buf_state & BM_LOCKED)
2995 old_buf_state = WaitBufHdrUnlocked(bufHdr);
2996
2997 buf_state = old_buf_state;
2998
2999 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
3000 buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
3001
3002 if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state,
3003 buf_state))
3004 break;
3005 }
3006
3007 /*
3008 * If the buffer was not dirty already, do vacuum accounting.
3009 */
3010 if (!(old_buf_state & BM_DIRTY))
3011 {
3012 pgBufferUsage.shared_blks_dirtied++;
3013 if (VacuumCostActive)
3014 VacuumCostBalance += VacuumCostPageDirty;
3015 }
3016 }
3017
3018 /*
3019 * ReleaseAndReadBuffer -- combine ReleaseBuffer() and ReadBuffer()
3020 *
3021 * Formerly, this saved one cycle of acquiring/releasing the BufMgrLock
3022 * compared to calling the two routines separately. Now it's mainly just
3023 * a convenience function. However, if the passed buffer is valid and
3024 * already contains the desired block, we just return it as-is; and that
3025 * does save considerable work compared to a full release and reacquire.
3026 *
3027 * Note: it is OK to pass buffer == InvalidBuffer, indicating that no old
3028 * buffer actually needs to be released. This case is the same as ReadBuffer,
3029 * but can save some tests in the caller.
3030 */
3031 Buffer
3032 ReleaseAndReadBuffer(Buffer buffer,
3033 Relation relation,
3034 BlockNumber blockNum)
3035 {
3036 ForkNumber forkNum = MAIN_FORKNUM;
3037 BufferDesc *bufHdr;
3038
3039 if (BufferIsValid(buffer))
3040 {
3041 Assert(BufferIsPinned(buffer));
3042 if (BufferIsLocal(buffer))
3043 {
3044 bufHdr = GetLocalBufferDescriptor(-buffer - 1);
3045 if (bufHdr->tag.blockNum == blockNum &&
3046 BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) &&
3047 BufTagGetForkNum(&bufHdr->tag) == forkNum)
3048 return buffer;
3049 UnpinLocalBuffer(buffer);
3050 }
3051 else
3052 {
3053 bufHdr = GetBufferDescriptor(buffer - 1);
3054 /* we have pin, so it's ok to examine tag without spinlock */
3055 if (bufHdr->tag.blockNum == blockNum &&
3056 BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) &&
3057 BufTagGetForkNum(&bufHdr->tag) == forkNum)
3058 return buffer;
3059 UnpinBuffer(bufHdr);
3060 }
3061 }
3062
3063 return ReadBuffer(relation, blockNum);
3064 }
3065
3066 /*
3067 * PinBuffer -- make buffer unavailable for replacement.
3068 *
3069 * For the default access strategy, the buffer's usage_count is incremented
3070 * when we first pin it; for other strategies we just make sure the usage_count
3071 * isn't zero. (The idea of the latter is that we don't want synchronized
3072 * heap scans to inflate the count, but we need it to not be zero to discourage
3073 * other backends from stealing buffers from our ring. As long as we cycle
3074 * through the ring faster than the global clock-sweep cycles, buffers in
3075 * our ring won't be chosen as victims for replacement by other backends.)
3076 *
3077 * This should be applied only to shared buffers, never local ones.
3078 *
3079 * Since buffers are pinned/unpinned very frequently, pin buffers without
3080 * taking the buffer header lock; instead update the state variable in loop of
3081 * CAS operations. Hopefully it's just a single CAS.
3082 *
3083 * Note that ResourceOwnerEnlarge() and ReservePrivateRefCountEntry()
3084 * must have been done already.
3085 *
3086 * Returns true if buffer is BM_VALID, else false. This provision allows
3087 * some callers to avoid an extra spinlock cycle.
3088 */
3089 static bool
3090 PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
3091 {
3092 Buffer b = BufferDescriptorGetBuffer(buf);
3093 bool result;
3094 PrivateRefCountEntry *ref;
3095
3096 Assert(!BufferIsLocal(b));
3097 Assert(ReservedRefCountEntry != NULL);
3098
3099 ref = GetPrivateRefCountEntry(b, true);
3100
3101 if (ref == NULL)
3102 {
3103 uint32 buf_state;
3104 uint32 old_buf_state;
3105
3106 ref = NewPrivateRefCountEntry(b);
3107
3108 old_buf_state = pg_atomic_read_u32(&buf->state);
3109 for (;;)
3110 {
3111 if (old_buf_state & BM_LOCKED)
3112 old_buf_state = WaitBufHdrUnlocked(buf);
3113
3114 buf_state = old_buf_state;
3115
3116 /* increase refcount */
3117 buf_state += BUF_REFCOUNT_ONE;
3118
3119 if (strategy == NULL)
3120 {
3121 /* Default case: increase usagecount unless already max. */
3122 if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
3123 buf_state += BUF_USAGECOUNT_ONE;
3124 }
3125 else
3126 {
3127 /*
3128 * Ring buffers shouldn't evict others from pool. Thus we
3129 * don't make usagecount more than 1.
3130 */
3131 if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
3132 buf_state += BUF_USAGECOUNT_ONE;
3133 }
3134
3135 if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
3136 buf_state))
3137 {
3138 result = (buf_state & BM_VALID) != 0;
3139
3140 /*
3141 * Assume that we acquired a buffer pin for the purposes of
3142 * Valgrind buffer client checks (even in !result case) to
3143 * keep things simple. Buffers that are unsafe to access are
3144 * not generally guaranteed to be marked undefined or
3145 * non-accessible in any case.
3146 */
3147 VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(buf), BLCKSZ);
3148 break;
3149 }
3150 }
3151 }
3152 else
3153 {
3154 /*
3155 * If we previously pinned the buffer, it is likely to be valid, but
3156 * it may not be if StartReadBuffers() was called and
3157 * WaitReadBuffers() hasn't been called yet. We'll check by loading
3158 * the flags without locking. This is racy, but it's OK to return
3159 * false spuriously: when WaitReadBuffers() calls StartBufferIO(),
3160 * it'll see that it's now valid.
3161 *
3162 * Note: We deliberately avoid a Valgrind client request here.
3163 * Individual access methods can optionally superimpose buffer page
3164 * client requests on top of our client requests to enforce that
3165 * buffers are only accessed while locked (and pinned). It's possible
3166 * that the buffer page is legitimately non-accessible here. We
3167 * cannot meddle with that.
3168 */
3169 result = (pg_atomic_read_u32(&buf->state) & BM_VALID) != 0;
3170 }
3171
3172 ref->refcount++;
3173 Assert(ref->refcount > 0);
3174 ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
3175 return result;
3176 }
3177
3178 /*
3179 * PinBuffer_Locked -- as above, but caller already locked the buffer header.
3180 * The spinlock is released before return.
3181 *
3182 * As this function is called with the spinlock held, the caller has to
3183 * previously call ReservePrivateRefCountEntry() and
3184 * ResourceOwnerEnlarge(CurrentResourceOwner);
3185 *
3186 * Currently, no callers of this function want to modify the buffer's
3187 * usage_count at all, so there's no need for a strategy parameter.
3188 * Also we don't bother with a BM_VALID test (the caller could check that for
3189 * itself).
3190 *
3191 * Also all callers only ever use this function when it's known that the
3192 * buffer can't have a preexisting pin by this backend. That allows us to skip
3193 * searching the private refcount array & hash, which is a boon, because the
3194 * spinlock is still held.
3195 *
3196 * Note: use of this routine is frequently mandatory, not just an optimization
3197 * to save a spin lock/unlock cycle, because we need to pin a buffer before
3198 * its state can change under us.
3199 */
3200 static void
3201 PinBuffer_Locked(BufferDesc *buf)
3202 {
3203 Buffer b;
3204 PrivateRefCountEntry *ref;
3205 uint32 buf_state;
3206
3207 /*
3208 * As explained, We don't expect any preexisting pins. That allows us to
3209 * manipulate the PrivateRefCount after releasing the spinlock
3210 */
3211 Assert(GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf), false) == NULL);
3212
3213 /*
3214 * Buffer can't have a preexisting pin, so mark its page as defined to
3215 * Valgrind (this is similar to the PinBuffer() case where the backend
3216 * doesn't already have a buffer pin)
3217 */
3218 VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(buf), BLCKSZ);
3219
3220 /*
3221 * Since we hold the buffer spinlock, we can update the buffer state and
3222 * release the lock in one operation.
3223 */
3224 buf_state = pg_atomic_read_u32(&buf->state);
3225 Assert(buf_state & BM_LOCKED);
3226 buf_state += BUF_REFCOUNT_ONE;
3227 UnlockBufHdr(buf, buf_state);
3228
3229 b = BufferDescriptorGetBuffer(buf);
3230
3231 ref = NewPrivateRefCountEntry(b);
3232 ref->refcount++;
3233
3234 ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
3235 }
3236
3237 /*
3238 * Support for waking up another backend that is waiting for the cleanup lock
3239 * to be released using BM_PIN_COUNT_WAITER.
3240 *
3241 * See LockBufferForCleanup().
3242 *
3243 * Expected to be called just after releasing a buffer pin (in a BufferDesc,
3244 * not just reducing the backend-local pincount for the buffer).
3245 */
3246 static void
3247 WakePinCountWaiter(BufferDesc *buf)
3248 {
3249 /*
3250 * Acquire the buffer header lock, re-check that there's a waiter. Another
3251 * backend could have unpinned this buffer, and already woken up the
3252 * waiter.
3253 *
3254 * There's no danger of the buffer being replaced after we unpinned it
3255 * above, as it's pinned by the waiter. The waiter removes
3256 * BM_PIN_COUNT_WAITER if it stops waiting for a reason other than this
3257 * backend waking it up.
3258 */
3259 uint32 buf_state = LockBufHdr(buf);
3260
3261 if ((buf_state & BM_PIN_COUNT_WAITER) &&
3262 BUF_STATE_GET_REFCOUNT(buf_state) == 1)
3263 {
3264 /* we just released the last pin other than the waiter's */
3265 int wait_backend_pgprocno = buf->wait_backend_pgprocno;
3266
3267 buf_state &= ~BM_PIN_COUNT_WAITER;
3268 UnlockBufHdr(buf, buf_state);
3269 ProcSendSignal(wait_backend_pgprocno);
3270 }
3271 else
3272 UnlockBufHdr(buf, buf_state);
3273 }
3274
3275 /*
3276 * UnpinBuffer -- make buffer available for replacement.
3277 *
3278 * This should be applied only to shared buffers, never local ones. This
3279 * always adjusts CurrentResourceOwner.
3280 */
3281 static void
3282 UnpinBuffer(BufferDesc *buf)
3283 {
3284 Buffer b = BufferDescriptorGetBuffer(buf);
3285
3286 ResourceOwnerForgetBuffer(CurrentResourceOwner, b);
3287 UnpinBufferNoOwner(buf);
3288 }
3289
3290 static void
3291 UnpinBufferNoOwner(BufferDesc *buf)
3292 {
3293 PrivateRefCountEntry *ref;
3294 Buffer b = BufferDescriptorGetBuffer(buf);
3295
3296 Assert(!BufferIsLocal(b));
3297
3298 /* not moving as we're likely deleting it soon anyway */
3299 ref = GetPrivateRefCountEntry(b, false);
3300 Assert(ref != NULL);
3301 Assert(ref->refcount > 0);
3302 ref->refcount--;
3303 if (ref->refcount == 0)
3304 {
3305 uint32 buf_state;
3306 uint32 old_buf_state;
3307
3308 /*
3309 * Mark buffer non-accessible to Valgrind.
3310 *
3311 * Note that the buffer may have already been marked non-accessible
3312 * within access method code that enforces that buffers are only
3313 * accessed while a buffer lock is held.
3314 */
3315 VALGRIND_MAKE_MEM_NOACCESS(BufHdrGetBlock(buf), BLCKSZ);
3316
3317 /* I'd better not still hold the buffer content lock */
3318 Assert(!LWLockHeldByMe(BufferDescriptorGetContentLock(buf)));
3319
3320 /*
3321 * Decrement the shared reference count.
3322 *
3323 * Since buffer spinlock holder can update status using just write,
3324 * it's not safe to use atomic decrement here; thus use a CAS loop.
3325 */
3326 old_buf_state = pg_atomic_read_u32(&buf->state);
3327 for (;;)
3328 {
3329 if (old_buf_state & BM_LOCKED)
3330 old_buf_state = WaitBufHdrUnlocked(buf);
3331
3332 buf_state = old_buf_state;
3333
3334 buf_state -= BUF_REFCOUNT_ONE;
3335
3336 if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
3337 buf_state))
3338 break;
3339 }
3340
3341 /* Support LockBufferForCleanup() */
3342 if (buf_state & BM_PIN_COUNT_WAITER)
3343 WakePinCountWaiter(buf);
3344
3345 ForgetPrivateRefCountEntry(ref);
3346 }
3347 }
3348
3349 #define ST_SORT sort_checkpoint_bufferids
3350 #define ST_ELEMENT_TYPE CkptSortItem
3351 #define ST_COMPARE(a, b) ckpt_buforder_comparator(a, b)
3352 #define ST_SCOPE static
3353 #define ST_DEFINE
3354 #include "lib/sort_template.h"
3355
3356 /*
3357 * BufferSync -- Write out all dirty buffers in the pool.
3358 *
3359 * This is called at checkpoint time to write out all dirty shared buffers.
3360 * The checkpoint request flags should be passed in. If CHECKPOINT_IMMEDIATE
3361 * is set, we disable delays between writes; if CHECKPOINT_IS_SHUTDOWN,
3362 * CHECKPOINT_END_OF_RECOVERY or CHECKPOINT_FLUSH_ALL is set, we write even
3363 * unlogged buffers, which are otherwise skipped. The remaining flags
3364 * currently have no effect here.
3365 */
3366 static void
3367 BufferSync(int flags)
3368 {
3369 uint32 buf_state;
3370 int buf_id;
3371 int num_to_scan;
3372 int num_spaces;
3373 int num_processed;
3374 int num_written;
3375 CkptTsStatus *per_ts_stat = NULL;
3376 Oid last_tsid;
3377 binaryheap *ts_heap;
3378 int i;
3379 int mask = BM_DIRTY;
3380 WritebackContext wb_context;
3381
3382 /*
3383 * Unless this is a shutdown checkpoint or we have been explicitly told,
3384 * we write only permanent, dirty buffers. But at shutdown or end of
3385 * recovery, we write all dirty buffers.
3386 */
3387 if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY |
3388 CHECKPOINT_FLUSH_ALL))))
3389 mask |= BM_PERMANENT;
3390
3391 /*
3392 * Loop over all buffers, and mark the ones that need to be written with
3393 * BM_CHECKPOINT_NEEDED. Count them as we go (num_to_scan), so that we
3394 * can estimate how much work needs to be done.
3395 *
3396 * This allows us to write only those pages that were dirty when the
3397 * checkpoint began, and not those that get dirtied while it proceeds.
3398 * Whenever a page with BM_CHECKPOINT_NEEDED is written out, either by us
3399 * later in this function, or by normal backends or the bgwriter cleaning
3400 * scan, the flag is cleared. Any buffer dirtied after this point won't
3401 * have the flag set.
3402 *
3403 * Note that if we fail to write some buffer, we may leave buffers with
3404 * BM_CHECKPOINT_NEEDED still set. This is OK since any such buffer would
3405 * certainly need to be written for the next checkpoint attempt, too.
3406 */
3407 num_to_scan = 0;
3408 for (buf_id = 0; buf_id < NBuffers; buf_id++)
3409 {
3410 BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
3411
3412 /*
3413 * Header spinlock is enough to examine BM_DIRTY, see comment in
3414 * SyncOneBuffer.
3415 */
3416 buf_state = LockBufHdr(bufHdr);
3417
3418 if ((buf_state & mask) == mask)
3419 {
3420 CkptSortItem *item;
3421
3422 buf_state |= BM_CHECKPOINT_NEEDED;
3423
3424 item = &CkptBufferIds[num_to_scan++];
3425 item->buf_id = buf_id;
3426 item->tsId = bufHdr->tag.spcOid;
3427 item->relNumber = BufTagGetRelNumber(&bufHdr->tag);
3428 item->forkNum = BufTagGetForkNum(&bufHdr->tag);
3429 item->blockNum = bufHdr->tag.blockNum;
3430 }
3431
3432 UnlockBufHdr(bufHdr, buf_state);
3433
3434 /* Check for barrier events in case NBuffers is large. */
3435 if (ProcSignalBarrierPending)
3436 ProcessProcSignalBarrier();
3437 }
3438
3439 if (num_to_scan == 0)
3440 return; /* nothing to do */
3441
3442 WritebackContextInit(&wb_context, &checkpoint_flush_after);
3443
3444 TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
3445
3446 /*
3447 * Sort buffers that need to be written to reduce the likelihood of random
3448 * IO. The sorting is also important for the implementation of balancing
3449 * writes between tablespaces. Without balancing writes we'd potentially
3450 * end up writing to the tablespaces one-by-one; possibly overloading the
3451 * underlying system.
3452 */
3453 sort_checkpoint_bufferids(CkptBufferIds, num_to_scan);
3454
3455 num_spaces = 0;
3456
3457 /*
3458 * Allocate progress status for each tablespace with buffers that need to
3459 * be flushed. This requires the to-be-flushed array to be sorted.
3460 */
3461 last_tsid = InvalidOid;
3462 for (i = 0; i < num_to_scan; i++)
3463 {
3464 CkptTsStatus *s;
3465 Oid cur_tsid;
3466
3467 cur_tsid = CkptBufferIds[i].tsId;
3468
3469 /*
3470 * Grow array of per-tablespace status structs, every time a new
3471 * tablespace is found.
3472 */
3473 if (last_tsid == InvalidOid || last_tsid != cur_tsid)
3474 {
3475 Size sz;
3476
3477 num_spaces++;
3478
3479 /*
3480 * Not worth adding grow-by-power-of-2 logic here - even with a
3481 * few hundred tablespaces this should be fine.
3482 */
3483 sz = sizeof(CkptTsStatus) * num_spaces;
3484
3485 if (per_ts_stat == NULL)
3486 per_ts_stat = (CkptTsStatus *) palloc(sz);
3487 else
3488 per_ts_stat = (CkptTsStatus *) repalloc(per_ts_stat, sz);
3489
3490 s = &per_ts_stat[num_spaces - 1];
3491 memset(s, 0, sizeof(*s));
3492 s->tsId = cur_tsid;
3493
3494 /*
3495 * The first buffer in this tablespace. As CkptBufferIds is sorted
3496 * by tablespace all (s->num_to_scan) buffers in this tablespace
3497 * will follow afterwards.
3498 */
3499 s->index = i;
3500
3501 /*
3502 * progress_slice will be determined once we know how many buffers
3503 * are in each tablespace, i.e. after this loop.
3504 */
3505
3506 last_tsid = cur_tsid;
3507 }
3508 else
3509 {
3510 s = &per_ts_stat[num_spaces - 1];
3511 }
3512
3513 s->num_to_scan++;
3514
3515 /* Check for barrier events. */
3516 if (ProcSignalBarrierPending)
3517 ProcessProcSignalBarrier();
3518 }
3519
3520 Assert(num_spaces > 0);
3521
3522 /*
3523 * Build a min-heap over the write-progress in the individual tablespaces,
3524 * and compute how large a portion of the total progress a single
3525 * processed buffer is.
3526 */
3527 ts_heap = binaryheap_allocate(num_spaces,
3528 ts_ckpt_progress_comparator,
3529 NULL);
3530
3531 for (i = 0; i < num_spaces; i++)
3532 {
3533 CkptTsStatus *ts_stat = &per_ts_stat[i];
3534
3535 ts_stat->progress_slice = (float8) num_to_scan / ts_stat->num_to_scan;
3536
3537 binaryheap_add_unordered(ts_heap, PointerGetDatum(ts_stat));
3538 }
3539
3540 binaryheap_build(ts_heap);
3541
3542 /*
3543 * Iterate through to-be-checkpointed buffers and write the ones (still)
3544 * marked with BM_CHECKPOINT_NEEDED. The writes are balanced between
3545 * tablespaces; otherwise the sorting would lead to only one tablespace
3546 * receiving writes at a time, making inefficient use of the hardware.
3547 */
3548 num_processed = 0;
3549 num_written = 0;
3550 while (!binaryheap_empty(ts_heap))
3551 {
3552 BufferDesc *bufHdr = NULL;
3553 CkptTsStatus *ts_stat = (CkptTsStatus *)
3554 DatumGetPointer(binaryheap_first(ts_heap));
3555
3556 buf_id = CkptBufferIds[ts_stat->index].buf_id;
3557 Assert(buf_id != -1);
3558
3559 bufHdr = GetBufferDescriptor(buf_id);
3560
3561 num_processed++;
3562
3563 /*
3564 * We don't need to acquire the lock here, because we're only looking
3565 * at a single bit. It's possible that someone else writes the buffer
3566 * and clears the flag right after we check, but that doesn't matter
3567 * since SyncOneBuffer will then do nothing. However, there is a
3568 * further race condition: it's conceivable that between the time we
3569 * examine the bit here and the time SyncOneBuffer acquires the lock,
3570 * someone else not only wrote the buffer but replaced it with another
3571 * page and dirtied it. In that improbable case, SyncOneBuffer will
3572 * write the buffer though we didn't need to. It doesn't seem worth
3573 * guarding against this, though.
3574 */
3575 if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED)
3576 {
3577 if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN)
3578 {
3579 TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
3580 PendingCheckpointerStats.buffers_written++;
3581 num_written++;
3582 }
3583 }
3584
3585 /*
3586 * Measure progress independent of actually having to flush the buffer
3587 * - otherwise writing become unbalanced.
3588 */
3589 ts_stat->progress += ts_stat->progress_slice;
3590 ts_stat->num_scanned++;
3591 ts_stat->index++;
3592
3593 /* Have all the buffers from the tablespace been processed? */
3594 if (ts_stat->num_scanned == ts_stat->num_to_scan)
3595 {
3596 binaryheap_remove_first(ts_heap);
3597 }
3598 else
3599 {
3600 /* update heap with the new progress */
3601 binaryheap_replace_first(ts_heap, PointerGetDatum(ts_stat));
3602 }
3603
3604 /*
3605 * Sleep to throttle our I/O rate.
3606 *
3607 * (This will check for barrier events even if it doesn't sleep.)
3608 */
3609 CheckpointWriteDelay(flags, (double) num_processed / num_to_scan);
3610 }
3611
3612 /*
3613 * Issue all pending flushes. Only checkpointer calls BufferSync(), so
3614 * IOContext will always be IOCONTEXT_NORMAL.
3615 */
3616 IssuePendingWritebacks(&wb_context, IOCONTEXT_NORMAL);
3617
3618 pfree(per_ts_stat);
3619 per_ts_stat = NULL;
3620 binaryheap_free(ts_heap);
3621
3622 /*
3623 * Update checkpoint statistics. As noted above, this doesn't include
3624 * buffers written by other backends or bgwriter scan.
3625 */
3626 CheckpointStats.ckpt_bufs_written += num_written;
3627
3628 TRACE_POSTGRESQL_BUFFER_SYNC_DONE(NBuffers, num_written, num_to_scan);
3629 }
3630
3631 /*
3632 * BgBufferSync -- Write out some dirty buffers in the pool.
3633 *
3634 * This is called periodically by the background writer process.
3635 *
3636 * Returns true if it's appropriate for the bgwriter process to go into
3637 * low-power hibernation mode. (This happens if the strategy clock sweep
3638 * has been "lapped" and no buffer allocations have occurred recently,
3639 * or if the bgwriter has been effectively disabled by setting
3640 * bgwriter_lru_maxpages to 0.)
3641 */
3642 bool
3643 BgBufferSync(WritebackContext *wb_context)
3644 {
3645 /* info obtained from freelist.c */
3646 int strategy_buf_id;
3647 uint32 strategy_passes;
3648 uint32 recent_alloc;
3649
3650 /*
3651 * Information saved between calls so we can determine the strategy
3652 * point's advance rate and avoid scanning already-cleaned buffers.
3653 */
3654 static bool saved_info_valid = false;
3655 static int prev_strategy_buf_id;
3656 static uint32 prev_strategy_passes;
3657 static int next_to_clean;
3658 static uint32 next_passes;
3659
3660 /* Moving averages of allocation rate and clean-buffer density */
3661 static float smoothed_alloc = 0;
3662 static float smoothed_density = 10.0;
3663
3664 /* Potentially these could be tunables, but for now, not */
3665 float smoothing_samples = 16;
3666 float scan_whole_pool_milliseconds = 120000.0;
3667
3668 /* Used to compute how far we scan ahead */
3669 long strategy_delta;
3670 int bufs_to_lap;
3671 int bufs_ahead;
3672 float scans_per_alloc;
3673 int reusable_buffers_est;
3674 int upcoming_alloc_est;
3675 int min_scan_buffers;
3676
3677 /* Variables for the scanning loop proper */
3678 int num_to_scan;
3679 int num_written;
3680 int reusable_buffers;
3681
3682 /* Variables for final smoothed_density update */
3683 long new_strategy_delta;
3684 uint32 new_recent_alloc;
3685
3686 /*
3687 * Find out where the freelist clock sweep currently is, and how many
3688 * buffer allocations have happened since our last call.
3689 */
3690 strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc);
3691
3692 /* Report buffer alloc counts to pgstat */
3693 PendingBgWriterStats.buf_alloc += recent_alloc;
3694
3695 /*
3696 * If we're not running the LRU scan, just stop after doing the stats
3697 * stuff. We mark the saved state invalid so that we can recover sanely
3698 * if LRU scan is turned back on later.
3699 */
3700 if (bgwriter_lru_maxpages <= 0)
3701 {
3702 saved_info_valid = false;
3703 return true;
3704 }
3705
3706 /*
3707 * Compute strategy_delta = how many buffers have been scanned by the
3708 * clock sweep since last time. If first time through, assume none. Then
3709 * see if we are still ahead of the clock sweep, and if so, how many
3710 * buffers we could scan before we'd catch up with it and "lap" it. Note:
3711 * weird-looking coding of xxx_passes comparisons are to avoid bogus
3712 * behavior when the passes counts wrap around.
3713 */
3714 if (saved_info_valid)
3715 {
3716 int32 passes_delta = strategy_passes - prev_strategy_passes;
3717
3718 strategy_delta = strategy_buf_id - prev_strategy_buf_id;
3719 strategy_delta += (long) passes_delta * NBuffers;
3720
3721 Assert(strategy_delta >= 0);
3722
3723 if ((int32) (next_passes - strategy_passes) > 0)
3724 {
3725 /* we're one pass ahead of the strategy point */
3726 bufs_to_lap = strategy_buf_id - next_to_clean;
3727 #ifdef BGW_DEBUG
3728 elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
3729 next_passes, next_to_clean,
3730 strategy_passes, strategy_buf_id,
3731 strategy_delta, bufs_to_lap);
3732 #endif
3733 }
3734 else if (next_passes == strategy_passes &&
3735 next_to_clean >= strategy_buf_id)
3736 {
3737 /* on same pass, but ahead or at least not behind */
3738 bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id);
3739 #ifdef BGW_DEBUG
3740 elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
3741 next_passes, next_to_clean,
3742 strategy_passes, strategy_buf_id,
3743 strategy_delta, bufs_to_lap);
3744 #endif
3745 }
3746 else
3747 {
3748 /*
3749 * We're behind, so skip forward to the strategy point and start
3750 * cleaning from there.
3751 */
3752 #ifdef BGW_DEBUG
3753 elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld",
3754 next_passes, next_to_clean,
3755 strategy_passes, strategy_buf_id,
3756 strategy_delta);
3757 #endif
3758 next_to_clean = strategy_buf_id;
3759 next_passes = strategy_passes;
3760 bufs_to_lap = NBuffers;
3761 }
3762 }
3763 else
3764 {
3765 /*
3766 * Initializing at startup or after LRU scanning had been off. Always
3767 * start at the strategy point.
3768 */
3769 #ifdef BGW_DEBUG
3770 elog(DEBUG2, "bgwriter initializing: strategy %u-%u",
3771 strategy_passes, strategy_buf_id);
3772 #endif
3773 strategy_delta = 0;
3774 next_to_clean = strategy_buf_id;
3775 next_passes = strategy_passes;
3776 bufs_to_lap = NBuffers;
3777 }
3778
3779 /* Update saved info for next time */
3780 prev_strategy_buf_id = strategy_buf_id;
3781 prev_strategy_passes = strategy_passes;
3782 saved_info_valid = true;
3783
3784 /*
3785 * Compute how many buffers had to be scanned for each new allocation, ie,
3786 * 1/density of reusable buffers, and track a moving average of that.
3787 *
3788 * If the strategy point didn't move, we don't update the density estimate
3789 */
3790 if (strategy_delta > 0 && recent_alloc > 0)
3791 {
3792 scans_per_alloc = (float) strategy_delta / (float) recent_alloc;
3793 smoothed_density += (scans_per_alloc - smoothed_density) /
3794 smoothing_samples;
3795 }
3796
3797 /*
3798 * Estimate how many reusable buffers there are between the current
3799 * strategy point and where we've scanned ahead to, based on the smoothed
3800 * density estimate.
3801 */
3802 bufs_ahead = NBuffers - bufs_to_lap;
3803 reusable_buffers_est = (float) bufs_ahead / smoothed_density;
3804
3805 /*
3806 * Track a moving average of recent buffer allocations. Here, rather than
3807 * a true average we want a fast-attack, slow-decline behavior: we
3808 * immediately follow any increase.
3809 */
3810 if (smoothed_alloc <= (float) recent_alloc)
3811 smoothed_alloc = recent_alloc;
3812 else
3813 smoothed_alloc += ((float) recent_alloc - smoothed_alloc) /
3814 smoothing_samples;
3815
3816 /* Scale the estimate by a GUC to allow more aggressive tuning. */
3817 upcoming_alloc_est = (int) (smoothed_alloc * bgwriter_lru_multiplier);
3818
3819 /*
3820 * If recent_alloc remains at zero for many cycles, smoothed_alloc will
3821 * eventually underflow to zero, and the underflows produce annoying
3822 * kernel warnings on some platforms. Once upcoming_alloc_est has gone to
3823 * zero, there's no point in tracking smaller and smaller values of
3824 * smoothed_alloc, so just reset it to exactly zero to avoid this
3825 * syndrome. It will pop back up as soon as recent_alloc increases.
3826 */
3827 if (upcoming_alloc_est == 0)
3828 smoothed_alloc = 0;
3829
3830 /*
3831 * Even in cases where there's been little or no buffer allocation
3832 * activity, we want to make a small amount of progress through the buffer
3833 * cache so that as many reusable buffers as possible are clean after an
3834 * idle period.
3835 *
3836 * (scan_whole_pool_milliseconds / BgWriterDelay) computes how many times
3837 * the BGW will be called during the scan_whole_pool time; slice the
3838 * buffer pool into that many sections.
3839 */
3840 min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay));
3841
3842 if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est))
3843 {
3844 #ifdef BGW_DEBUG
3845 elog(DEBUG2, "bgwriter: alloc_est=%d too small, using min=%d + reusable_est=%d",
3846 upcoming_alloc_est, min_scan_buffers, reusable_buffers_est);
3847 #endif
3848 upcoming_alloc_est = min_scan_buffers + reusable_buffers_est;
3849 }
3850
3851 /*
3852 * Now write out dirty reusable buffers, working forward from the
3853 * next_to_clean point, until we have lapped the strategy scan, or cleaned
3854 * enough buffers to match our estimate of the next cycle's allocation
3855 * requirements, or hit the bgwriter_lru_maxpages limit.
3856 */
3857
3858 num_to_scan = bufs_to_lap;
3859 num_written = 0;
3860 reusable_buffers = reusable_buffers_est;
3861
3862 /* Execute the LRU scan */
3863 while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est)
3864 {
3865 int sync_state = SyncOneBuffer(next_to_clean, true,
3866 wb_context);
3867
3868 if (++next_to_clean >= NBuffers)
3869 {
3870 next_to_clean = 0;
3871 next_passes++;
3872 }
3873 num_to_scan--;
3874
3875 if (sync_state & BUF_WRITTEN)
3876 {
3877 reusable_buffers++;
3878 if (++num_written >= bgwriter_lru_maxpages)
3879 {
3880 PendingBgWriterStats.maxwritten_clean++;
3881 break;
3882 }
3883 }
3884 else if (sync_state & BUF_REUSABLE)
3885 reusable_buffers++;
3886 }
3887
3888 PendingBgWriterStats.buf_written_clean += num_written;
3889
3890 #ifdef BGW_DEBUG
3891 elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d",
3892 recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead,
3893 smoothed_density, reusable_buffers_est, upcoming_alloc_est,
3894 bufs_to_lap - num_to_scan,
3895 num_written,
3896 reusable_buffers - reusable_buffers_est);
3897 #endif
3898
3899 /*
3900 * Consider the above scan as being like a new allocation scan.
3901 * Characterize its density and update the smoothed one based on it. This
3902 * effectively halves the moving average period in cases where both the
3903 * strategy and the background writer are doing some useful scanning,
3904 * which is helpful because a long memory isn't as desirable on the
3905 * density estimates.
3906 */
3907 new_strategy_delta = bufs_to_lap - num_to_scan;
3908 new_recent_alloc = reusable_buffers - reusable_buffers_est;
3909 if (new_strategy_delta > 0 && new_recent_alloc > 0)
3910 {
3911 scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc;
3912 smoothed_density += (scans_per_alloc - smoothed_density) /
3913 smoothing_samples;
3914
3915 #ifdef BGW_DEBUG
3916 elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f",
3917 new_recent_alloc, new_strategy_delta,
3918 scans_per_alloc, smoothed_density);
3919 #endif
3920 }
3921
3922 /* Return true if OK to hibernate */
3923 return (bufs_to_lap == 0 && recent_alloc == 0);
3924 }
3925
3926 /*
3927 * SyncOneBuffer -- process a single buffer during syncing.
3928 *
3929 * If skip_recently_used is true, we don't write currently-pinned buffers, nor
3930 * buffers marked recently used, as these are not replacement candidates.
3931 *
3932 * Returns a bitmask containing the following flag bits:
3933 * BUF_WRITTEN: we wrote the buffer.
3934 * BUF_REUSABLE: buffer is available for replacement, ie, it has
3935 * pin count 0 and usage count 0.
3936 *
3937 * (BUF_WRITTEN could be set in error if FlushBuffer finds the buffer clean
3938 * after locking it, but we don't care all that much.)
3939 */
3940 static int
3941 SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context)
3942 {
3943 BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
3944 int result = 0;
3945 uint32 buf_state;
3946 BufferTag tag;
3947
3948 /* Make sure we can handle the pin */
3949 ReservePrivateRefCountEntry();
3950 ResourceOwnerEnlarge(CurrentResourceOwner);
3951
3952 /*
3953 * Check whether buffer needs writing.
3954 *
3955 * We can make this check without taking the buffer content lock so long
3956 * as we mark pages dirty in access methods *before* logging changes with
3957 * XLogInsert(): if someone marks the buffer dirty just after our check we
3958 * don't worry because our checkpoint.redo points before log record for
3959 * upcoming changes and so we are not required to write such dirty buffer.
3960 */
3961 buf_state = LockBufHdr(bufHdr);
3962
3963 if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 &&
3964 BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
3965 {
3966 result |= BUF_REUSABLE;
3967 }
3968 else if (skip_recently_used)
3969 {
3970 /* Caller told us not to write recently-used buffers */
3971 UnlockBufHdr(bufHdr, buf_state);
3972 return result;
3973 }
3974
3975 if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY))
3976 {
3977 /* It's clean, so nothing to do */
3978 UnlockBufHdr(bufHdr, buf_state);
3979 return result;
3980 }
3981
3982 /*
3983 * Pin it, share-lock it, write it. (FlushBuffer will do nothing if the
3984 * buffer is clean by the time we've locked it.)
3985 */
3986 PinBuffer_Locked(bufHdr);
3987 LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
3988
3989 FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
3990
3991 LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
3992
3993 tag = bufHdr->tag;
3994
3995 UnpinBuffer(bufHdr);
3996
3997 /*
3998 * SyncOneBuffer() is only called by checkpointer and bgwriter, so
3999 * IOContext will always be IOCONTEXT_NORMAL.
4000 */
4001 ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag);
4002
4003 return result | BUF_WRITTEN;
4004 }
4005
4006 /*
4007 * AtEOXact_Buffers - clean up at end of transaction.
4008 *
4009 * As of PostgreSQL 8.0, buffer pins should get released by the
4010 * ResourceOwner mechanism. This routine is just a debugging
4011 * cross-check that no pins remain.
4012 */
4013 void
4014 AtEOXact_Buffers(bool isCommit)
4015 {
4016 CheckForBufferLeaks();
4017
4018 AtEOXact_LocalBuffers(isCommit);
4019
4020 Assert(PrivateRefCountOverflowed == 0);
4021 }
4022
4023 /*
4024 * Initialize access to shared buffer pool
4025 *
4026 * This is called during backend startup (whether standalone or under the
4027 * postmaster). It sets up for this backend's access to the already-existing
4028 * buffer pool.
4029 */
4030 void
4031 InitBufferManagerAccess(void)
4032 {
4033 HASHCTL hash_ctl;
4034
4035 /*
4036 * An advisory limit on the number of pins each backend should hold, based
4037 * on shared_buffers and the maximum number of connections possible.
4038 * That's very pessimistic, but outside toy-sized shared_buffers it should
4039 * allow plenty of pins. LimitAdditionalPins() and
4040 * GetAdditionalPinLimit() can be used to check the remaining balance.
4041 */
4042 MaxProportionalPins = NBuffers / (MaxBackends + NUM_AUXILIARY_PROCS);
4043
4044 memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray));
4045
4046 hash_ctl.keysize = sizeof(int32);
4047 hash_ctl.entrysize = sizeof(PrivateRefCountEntry);
4048
4049 PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl,
4050 HASH_ELEM | HASH_BLOBS);
4051
4052 /*
4053 * AtProcExit_Buffers needs LWLock access, and thereby has to be called at
4054 * the corresponding phase of backend shutdown.
4055 */
4056 Assert(MyProc != NULL);
4057 on_shmem_exit(AtProcExit_Buffers, 0);
4058 }
4059
4060 /*
4061 * During backend exit, ensure that we released all shared-buffer locks and
4062 * assert that we have no remaining pins.
4063 */
4064 static void
4065 AtProcExit_Buffers(int code, Datum arg)
4066 {
4067 UnlockBuffers();
4068
4069 CheckForBufferLeaks();
4070
4071 /* localbuf.c needs a chance too */
4072 AtProcExit_LocalBuffers();
4073 }
4074
4075 /*
4076 * CheckForBufferLeaks - ensure this backend holds no buffer pins
4077 *
4078 * As of PostgreSQL 8.0, buffer pins should get released by the
4079 * ResourceOwner mechanism. This routine is just a debugging
4080 * cross-check that no pins remain.
4081 */
4082 static void
4083 CheckForBufferLeaks(void)
4084 {
4085 #ifdef USE_ASSERT_CHECKING
4086 int RefCountErrors = 0;
4087 PrivateRefCountEntry *res;
4088 int i;
4089 char *s;
4090
4091 /* check the array */
4092 for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
4093 {
4094 res = &PrivateRefCountArray[i];
4095
4096 if (res->buffer != InvalidBuffer)
4097 {
4098 s = DebugPrintBufferRefcount(res->buffer);
4099 elog(WARNING, "buffer refcount leak: %s", s);
4100 pfree(s);
4101
4102 RefCountErrors++;
4103 }
4104 }
4105
4106 /* if necessary search the hash */
4107 if (PrivateRefCountOverflowed)
4108 {
4109 HASH_SEQ_STATUS hstat;
4110
4111 hash_seq_init(&hstat, PrivateRefCountHash);
4112 while ((res = (PrivateRefCountEntry *) hash_seq_search(&hstat)) != NULL)
4113 {
4114 s = DebugPrintBufferRefcount(res->buffer);
4115 elog(WARNING, "buffer refcount leak: %s", s);
4116 pfree(s);
4117 RefCountErrors++;
4118 }
4119 }
4120
4121 Assert(RefCountErrors == 0);
4122 #endif
4123 }
4124
4125 #ifdef USE_ASSERT_CHECKING
4126 /*
4127 * Check for exclusive-locked catalog buffers. This is the core of
4128 * AssertCouldGetRelation().
4129 *
4130 * A backend would self-deadlock on LWLocks if the catalog scan read the
4131 * exclusive-locked buffer. The main threat is exclusive-locked buffers of
4132 * catalogs used in relcache, because a catcache search on any catalog may
4133 * build that catalog's relcache entry. We don't have an inventory of
4134 * catalogs relcache uses, so just check buffers of most catalogs.
4135 *
4136 * It's better to minimize waits while holding an exclusive buffer lock, so it
4137 * would be nice to broaden this check not to be catalog-specific. However,
4138 * bttextcmp() accesses pg_collation, and non-core opclasses might similarly
4139 * read tables. That is deadlock-free as long as there's no loop in the
4140 * dependency graph: modifying table A may cause an opclass to read table B,
4141 * but it must not cause a read of table A.
4142 */
4143 void
4144 AssertBufferLocksPermitCatalogRead(void)
4145 {
4146 ForEachLWLockHeldByMe(AssertNotCatalogBufferLock, NULL);
4147 }
4148
4149 static void
4150 AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode,
4151 void *unused_context)
4152 {
4153 BufferDesc *bufHdr;
4154 BufferTag tag;
4155 Oid relid;
4156
4157 if (mode != LW_EXCLUSIVE)
4158 return;
4159
4160 if (!((BufferDescPadded *) lock > BufferDescriptors &&
4161 (BufferDescPadded *) lock < BufferDescriptors + NBuffers))
4162 return; /* not a buffer lock */
4163
4164 bufHdr = (BufferDesc *)
4165 ((char *) lock - offsetof(BufferDesc, content_lock));
4166 tag = bufHdr->tag;
4167
4168 /*
4169 * This relNumber==relid assumption holds until a catalog experiences
4170 * VACUUM FULL or similar. After a command like that, relNumber will be
4171 * in the normal (non-catalog) range, and we lose the ability to detect
4172 * hazardous access to that catalog. Calling RelidByRelfilenumber() would
4173 * close that gap, but RelidByRelfilenumber() might then deadlock with a
4174 * held lock.
4175 */
4176 relid = tag.relNumber;
4177
4178 if (IsCatalogTextUniqueIndexOid(relid)) /* see comments at the callee */
4179 return;
4180
4181 Assert(!IsCatalogRelationOid(relid));
4182 }
4183 #endif
4184
4185
4186 /*
4187 * Helper routine to issue warnings when a buffer is unexpectedly pinned
4188 */
4189 char *
4190 DebugPrintBufferRefcount(Buffer buffer)
4191 {
4192 BufferDesc *buf;
4193 int32 loccount;
4194 char *result;
4195 ProcNumber backend;
4196 uint32 buf_state;
4197
4198 Assert(BufferIsValid(buffer));
4199 if (BufferIsLocal(buffer))
4200 {
4201 buf = GetLocalBufferDescriptor(-buffer - 1);
4202 loccount = LocalRefCount[-buffer - 1];
4203 backend = MyProcNumber;
4204 }
4205 else
4206 {
4207 buf = GetBufferDescriptor(buffer - 1);
4208 loccount = GetPrivateRefCount(buffer);
4209 backend = INVALID_PROC_NUMBER;
4210 }
4211
4212 /* theoretically we should lock the bufhdr here */
4213 buf_state = pg_atomic_read_u32(&buf->state);
4214
4215 result = psprintf("[%03d] (rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)",
4216 buffer,
4217 relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend,
4218 BufTagGetForkNum(&buf->tag)).str,
4219 buf->tag.blockNum, buf_state & BUF_FLAG_MASK,
4220 BUF_STATE_GET_REFCOUNT(buf_state), loccount);
4221 return result;
4222 }
4223
4224 /*
4225 * CheckPointBuffers
4226 *
4227 * Flush all dirty blocks in buffer pool to disk at checkpoint time.
4228 *
4229 * Note: temporary relations do not participate in checkpoints, so they don't
4230 * need to be flushed.
4231 */
4232 void
4233 CheckPointBuffers(int flags)
4234 {
4235 BufferSync(flags);
4236 }
4237
4238 /*
4239 * BufferGetBlockNumber
4240 * Returns the block number associated with a buffer.
4241 *
4242 * Note:
4243 * Assumes that the buffer is valid and pinned, else the
4244 * value may be obsolete immediately...
4245 */
4246 BlockNumber
4247 BufferGetBlockNumber(Buffer buffer)
4248 {
4249 BufferDesc *bufHdr;
4250
4251 Assert(BufferIsPinned(buffer));
4252
4253 if (BufferIsLocal(buffer))
4254 bufHdr = GetLocalBufferDescriptor(-buffer - 1);
4255 else
4256 bufHdr = GetBufferDescriptor(buffer - 1);
4257
4258 /* pinned, so OK to read tag without spinlock */
4259 return bufHdr->tag.blockNum;
4260 }
4261
4262 /*
4263 * BufferGetTag
4264 * Returns the relfilelocator, fork number and block number associated with
4265 * a buffer.
4266 */
4267 void
4268 BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum,
4269 BlockNumber *blknum)
4270 {
4271 BufferDesc *bufHdr;
4272
4273 /* Do the same checks as BufferGetBlockNumber. */
4274 Assert(BufferIsPinned(buffer));
4275
4276 if (BufferIsLocal(buffer))
4277 bufHdr = GetLocalBufferDescriptor(-buffer - 1);
4278 else
4279 bufHdr = GetBufferDescriptor(buffer - 1);
4280
4281 /* pinned, so OK to read tag without spinlock */
4282 *rlocator = BufTagGetRelFileLocator(&bufHdr->tag);
4283 *forknum = BufTagGetForkNum(&bufHdr->tag);
4284 *blknum = bufHdr->tag.blockNum;
4285 }
4286
4287 /*
4288 * FlushBuffer
4289 * Physically write out a shared buffer.
4290 *
4291 * NOTE: this actually just passes the buffer contents to the kernel; the
4292 * real write to disk won't happen until the kernel feels like it. This
4293 * is okay from our point of view since we can redo the changes from WAL.
4294 * However, we will need to force the changes to disk via fsync before
4295 * we can checkpoint WAL.
4296 *
4297 * The caller must hold a pin on the buffer and have share-locked the
4298 * buffer contents. (Note: a share-lock does not prevent updates of
4299 * hint bits in the buffer, so the page could change while the write
4300 * is in progress, but we assume that that will not invalidate the data
4301 * written.)
4302 *
4303 * If the caller has an smgr reference for the buffer's relation, pass it
4304 * as the second parameter. If not, pass NULL.
4305 */
4306 static void
4307 FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
4308 IOContext io_context)
4309 {
4310 XLogRecPtr recptr;
4311 ErrorContextCallback errcallback;
4312 instr_time io_start;
4313 Block bufBlock;
4314 char *bufToWrite;
4315 uint32 buf_state;
4316
4317 /*
4318 * Try to start an I/O operation. If StartBufferIO returns false, then
4319 * someone else flushed the buffer before we could, so we need not do
4320 * anything.
4321 */
4322 if (!StartBufferIO(buf, false, false))
4323 return;
4324
4325 /* Setup error traceback support for ereport() */
4326 errcallback.callback = shared_buffer_write_error_callback;
4327 errcallback.arg = buf;
4328 errcallback.previous = error_context_stack;
4329 error_context_stack = &errcallback;
4330
4331 /* Find smgr relation for buffer */
4332 if (reln == NULL)
4333 reln = smgropen(BufTagGetRelFileLocator(&buf->tag), INVALID_PROC_NUMBER);
4334
4335 TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag),
4336 buf->tag.blockNum,
4337 reln->smgr_rlocator.locator.spcOid,
4338 reln->smgr_rlocator.locator.dbOid,
4339 reln->smgr_rlocator.locator.relNumber);
4340
4341 buf_state = LockBufHdr(buf);
4342
4343 /*
4344 * Run PageGetLSN while holding header lock, since we don't have the
4345 * buffer locked exclusively in all cases.
4346 */
4347 recptr = BufferGetLSN(buf);
4348
4349 /* To check if block content changes while flushing. - vadim 01/17/97 */
4350 buf_state &= ~BM_JUST_DIRTIED;
4351 UnlockBufHdr(buf, buf_state);
4352
4353 /*
4354 * Force XLOG flush up to buffer's LSN. This implements the basic WAL
4355 * rule that log updates must hit disk before any of the data-file changes
4356 * they describe do.
4357 *
4358 * However, this rule does not apply to unlogged relations, which will be
4359 * lost after a crash anyway. Most unlogged relation pages do not bear
4360 * LSNs since we never emit WAL records for them, and therefore flushing
4361 * up through the buffer LSN would be useless, but harmless. However,
4362 * GiST indexes use LSNs internally to track page-splits, and therefore
4363 * unlogged GiST pages bear "fake" LSNs generated by
4364 * GetFakeLSNForUnloggedRel. It is unlikely but possible that the fake
4365 * LSN counter could advance past the WAL insertion point; and if it did
4366 * happen, attempting to flush WAL through that location would fail, with
4367 * disastrous system-wide consequences. To make sure that can't happen,
4368 * skip the flush if the buffer isn't permanent.
4369 */
4370 if (buf_state & BM_PERMANENT)
4371 XLogFlush(recptr);
4372
4373 /*
4374 * Now it's safe to write the buffer to disk. Note that no one else should
4375 * have been able to write it, while we were busy with log flushing,
4376 * because we got the exclusive right to perform I/O by setting the
4377 * BM_IO_IN_PROGRESS bit.
4378 */
4379 bufBlock = BufHdrGetBlock(buf);
4380
4381 /*
4382 * Update page checksum if desired. Since we have only shared lock on the
4383 * buffer, other processes might be updating hint bits in it, so we must
4384 * copy the page to private storage if we do checksumming.
4385 */
4386 bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
4387
4388 io_start = pgstat_prepare_io_time(track_io_timing);
4389
4390 /*
4391 * bufToWrite is either the shared buffer or a copy, as appropriate.
4392 */
4393 smgrwrite(reln,
4394 BufTagGetForkNum(&buf->tag),
4395 buf->tag.blockNum,
4396 bufToWrite,
4397 false);
4398
4399 /*
4400 * When a strategy is in use, only flushes of dirty buffers already in the
4401 * strategy ring are counted as strategy writes (IOCONTEXT
4402 * [BULKREAD|BULKWRITE|VACUUM] IOOP_WRITE) for the purpose of IO
4403 * statistics tracking.
4404 *
4405 * If a shared buffer initially added to the ring must be flushed before
4406 * being used, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE.
4407 *
4408 * If a shared buffer which was added to the ring later because the
4409 * current strategy buffer is pinned or in use or because all strategy
4410 * buffers were dirty and rejected (for BAS_BULKREAD operations only)
4411 * requires flushing, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE
4412 * (from_ring will be false).
4413 *
4414 * When a strategy is not in use, the write can only be a "regular" write
4415 * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE).
4416 */
4417 pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
4418 IOOP_WRITE, io_start, 1, BLCKSZ);
4419
4420 pgBufferUsage.shared_blks_written++;
4421
4422 /*
4423 * Mark the buffer as clean (unless BM_JUST_DIRTIED has become set) and
4424 * end the BM_IO_IN_PROGRESS state.
4425 */
4426 TerminateBufferIO(buf, true, 0, true, false);
4427
4428 TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag),
4429 buf->tag.blockNum,
4430 reln->smgr_rlocator.locator.spcOid,
4431 reln->smgr_rlocator.locator.dbOid,
4432 reln->smgr_rlocator.locator.relNumber);
4433
4434 /* Pop the error context stack */
4435 error_context_stack = errcallback.previous;
4436 }
4437
4438 /*
4439 * RelationGetNumberOfBlocksInFork
4440 * Determines the current number of pages in the specified relation fork.
4441 *
4442 * Note that the accuracy of the result will depend on the details of the
4443 * relation's storage. For builtin AMs it'll be accurate, but for external AMs
4444 * it might not be.
4445 */
4446 BlockNumber
4447 RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum)
4448 {
4449 if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind))
4450 {
4451 /*
4452 * Not every table AM uses BLCKSZ wide fixed size blocks. Therefore
4453 * tableam returns the size in bytes - but for the purpose of this
4454 * routine, we want the number of blocks. Therefore divide, rounding
4455 * up.
4456 */
4457 uint64 szbytes;
4458
4459 szbytes = table_relation_size(relation, forkNum);
4460
4461 return (szbytes + (BLCKSZ - 1)) / BLCKSZ;
4462 }
4463 else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind))
4464 {
4465 return smgrnblocks(RelationGetSmgr(relation), forkNum);
4466 }
4467 else
4468 Assert(false);
4469
4470 return 0; /* keep compiler quiet */
4471 }
4472
4473 /*
4474 * BufferIsPermanent
4475 * Determines whether a buffer will potentially still be around after
4476 * a crash. Caller must hold a buffer pin.
4477 */
4478 bool
4479 BufferIsPermanent(Buffer buffer)
4480 {
4481 BufferDesc *bufHdr;
4482
4483 /* Local buffers are used only for temp relations. */
4484 if (BufferIsLocal(buffer))
4485 return false;
4486
4487 /* Make sure we've got a real buffer, and that we hold a pin on it. */
4488 Assert(BufferIsValid(buffer));
4489 Assert(BufferIsPinned(buffer));
4490
4491 /*
4492 * BM_PERMANENT can't be changed while we hold a pin on the buffer, so we
4493 * need not bother with the buffer header spinlock. Even if someone else
4494 * changes the buffer header state while we're doing this, the state is
4495 * changed atomically, so we'll read the old value or the new value, but
4496 * not random garbage.
4497 */
4498 bufHdr = GetBufferDescriptor(buffer - 1);
4499 return (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT) != 0;
4500 }
4501
4502 /*
4503 * BufferGetLSNAtomic
4504 * Retrieves the LSN of the buffer atomically using a buffer header lock.
4505 * This is necessary for some callers who may not have an exclusive lock
4506 * on the buffer.
4507 */
4508 XLogRecPtr
4509 BufferGetLSNAtomic(Buffer buffer)
4510 {
4511 char *page = BufferGetPage(buffer);
4512 BufferDesc *bufHdr;
4513 XLogRecPtr lsn;
4514 uint32 buf_state;
4515
4516 /*
4517 * If we don't need locking for correctness, fastpath out.
4518 */
4519 if (!XLogHintBitIsNeeded() || BufferIsLocal(buffer))
4520 return PageGetLSN(page);
4521
4522 /* Make sure we've got a real buffer, and that we hold a pin on it. */
4523 Assert(BufferIsValid(buffer));
4524 Assert(BufferIsPinned(buffer));
4525
4526 bufHdr = GetBufferDescriptor(buffer - 1);
4527 buf_state = LockBufHdr(bufHdr);
4528 lsn = PageGetLSN(page);
4529 UnlockBufHdr(bufHdr, buf_state);
4530
4531 return lsn;
4532 }
4533
4534 /* ---------------------------------------------------------------------
4535 * DropRelationBuffers
4536 *
4537 * This function removes from the buffer pool all the pages of the
4538 * specified relation forks that have block numbers >= firstDelBlock.
4539 * (In particular, with firstDelBlock = 0, all pages are removed.)
4540 * Dirty pages are simply dropped, without bothering to write them
4541 * out first. Therefore, this is NOT rollback-able, and so should be
4542 * used only with extreme caution!
4543 *
4544 * Currently, this is called only from smgr.c when the underlying file
4545 * is about to be deleted or truncated (firstDelBlock is needed for
4546 * the truncation case). The data in the affected pages would therefore
4547 * be deleted momentarily anyway, and there is no point in writing it.
4548 * It is the responsibility of higher-level code to ensure that the
4549 * deletion or truncation does not lose any data that could be needed
4550 * later. It is also the responsibility of higher-level code to ensure
4551 * that no other process could be trying to load more pages of the
4552 * relation into buffers.
4553 * --------------------------------------------------------------------
4554 */
4555 void
4556 DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum,
4557 int nforks, BlockNumber *firstDelBlock)
4558 {
4559 int i;
4560 int j;
4561 RelFileLocatorBackend rlocator;
4562 BlockNumber nForkBlock[MAX_FORKNUM];
4563 uint64 nBlocksToInvalidate = 0;
4564
4565 rlocator = smgr_reln->smgr_rlocator;
4566
4567 /* If it's a local relation, it's localbuf.c's problem. */
4568 if (RelFileLocatorBackendIsTemp(rlocator))
4569 {
4570 if (rlocator.backend == MyProcNumber)
4571 {
4572 for (j = 0; j < nforks; j++)
4573 DropRelationLocalBuffers(rlocator.locator, forkNum[j],
4574 firstDelBlock[j]);
4575 }
4576 return;
4577 }
4578
4579 /*
4580 * To remove all the pages of the specified relation forks from the buffer
4581 * pool, we need to scan the entire buffer pool but we can optimize it by
4582 * finding the buffers from BufMapping table provided we know the exact
4583 * size of each fork of the relation. The exact size is required to ensure
4584 * that we don't leave any buffer for the relation being dropped as
4585 * otherwise the background writer or checkpointer can lead to a PANIC
4586 * error while flushing buffers corresponding to files that don't exist.
4587 *
4588 * To know the exact size, we rely on the size cached for each fork by us
4589 * during recovery which limits the optimization to recovery and on
4590 * standbys but we can easily extend it once we have shared cache for
4591 * relation size.
4592 *
4593 * In recovery, we cache the value returned by the first lseek(SEEK_END)
4594 * and the future writes keeps the cached value up-to-date. See
4595 * smgrextend. It is possible that the value of the first lseek is smaller
4596 * than the actual number of existing blocks in the file due to buggy
4597 * Linux kernels that might not have accounted for the recent write. But
4598 * that should be fine because there must not be any buffers after that
4599 * file size.
4600 */
4601 for (i = 0; i < nforks; i++)
4602 {
4603 /* Get the number of blocks for a relation's fork */
4604 nForkBlock[i] = smgrnblocks_cached(smgr_reln, forkNum[i]);
4605
4606 if (nForkBlock[i] == InvalidBlockNumber)
4607 {
4608 nBlocksToInvalidate = InvalidBlockNumber;
4609 break;
4610 }
4611
4612 /* calculate the number of blocks to be invalidated */
4613 nBlocksToInvalidate += (nForkBlock[i] - firstDelBlock[i]);
4614 }
4615
4616 /*
4617 * We apply the optimization iff the total number of blocks to invalidate
4618 * is below the BUF_DROP_FULL_SCAN_THRESHOLD.
4619 */
4620 if (BlockNumberIsValid(nBlocksToInvalidate) &&
4621 nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD)
4622 {
4623 for (j = 0; j < nforks; j++)
4624 FindAndDropRelationBuffers(rlocator.locator, forkNum[j],
4625 nForkBlock[j], firstDelBlock[j]);
4626 return;
4627 }
4628
4629 for (i = 0; i < NBuffers; i++)
4630 {
4631 BufferDesc *bufHdr = GetBufferDescriptor(i);
4632 uint32 buf_state;
4633
4634 /*
4635 * We can make this a tad faster by prechecking the buffer tag before
4636 * we attempt to lock the buffer; this saves a lot of lock
4637 * acquisitions in typical cases. It should be safe because the
4638 * caller must have AccessExclusiveLock on the relation, or some other
4639 * reason to be certain that no one is loading new pages of the rel
4640 * into the buffer pool. (Otherwise we might well miss such pages
4641 * entirely.) Therefore, while the tag might be changing while we
4642 * look at it, it can't be changing *to* a value we care about, only
4643 * *away* from such a value. So false negatives are impossible, and
4644 * false positives are safe because we'll recheck after getting the
4645 * buffer lock.
4646 *
4647 * We could check forkNum and blockNum as well as the rlocator, but
4648 * the incremental win from doing so seems small.
4649 */
4650 if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator))
4651 continue;
4652
4653 buf_state = LockBufHdr(bufHdr);
4654
4655 for (j = 0; j < nforks; j++)
4656 {
4657 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) &&
4658 BufTagGetForkNum(&bufHdr->tag) == forkNum[j] &&
4659 bufHdr->tag.blockNum >= firstDelBlock[j])
4660 {
4661 InvalidateBuffer(bufHdr); /* releases spinlock */
4662 break;
4663 }
4664 }
4665 if (j >= nforks)
4666 UnlockBufHdr(bufHdr, buf_state);
4667 }
4668 }
4669
4670 /* ---------------------------------------------------------------------
4671 * DropRelationsAllBuffers
4672 *
4673 * This function removes from the buffer pool all the pages of all
4674 * forks of the specified relations. It's equivalent to calling
4675 * DropRelationBuffers once per fork per relation with firstDelBlock = 0.
4676 * --------------------------------------------------------------------
4677 */
4678 void
4679 DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
4680 {
4681 int i;
4682 int n = 0;
4683 SMgrRelation *rels;
4684 BlockNumber (*block)[MAX_FORKNUM + 1];
4685 uint64 nBlocksToInvalidate = 0;
4686 RelFileLocator *locators;
4687 bool cached = true;
4688 bool use_bsearch;
4689
4690 if (nlocators == 0)
4691 return;
4692
4693 rels = palloc(sizeof(SMgrRelation) * nlocators); /* non-local relations */
4694
4695 /* If it's a local relation, it's localbuf.c's problem. */
4696 for (i = 0; i < nlocators; i++)
4697 {
4698 if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator))
4699 {
4700 if (smgr_reln[i]->smgr_rlocator.backend == MyProcNumber)
4701 DropRelationAllLocalBuffers(smgr_reln[i]->smgr_rlocator.locator);
4702 }
4703 else
4704 rels[n++] = smgr_reln[i];
4705 }
4706
4707 /*
4708 * If there are no non-local relations, then we're done. Release the
4709 * memory and return.
4710 */
4711 if (n == 0)
4712 {
4713 pfree(rels);
4714 return;
4715 }
4716
4717 /*
4718 * This is used to remember the number of blocks for all the relations
4719 * forks.
4720 */
4721 block = (BlockNumber (*)[MAX_FORKNUM + 1])
4722 palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1));
4723
4724 /*
4725 * We can avoid scanning the entire buffer pool if we know the exact size
4726 * of each of the given relation forks. See DropRelationBuffers.
4727 */
4728 for (i = 0; i < n && cached; i++)
4729 {
4730 for (int j = 0; j <= MAX_FORKNUM; j++)
4731 {
4732 /* Get the number of blocks for a relation's fork. */
4733 block[i][j] = smgrnblocks_cached(rels[i], j);
4734
4735 /* We need to only consider the relation forks that exists. */
4736 if (block[i][j] == InvalidBlockNumber)
4737 {
4738 if (!smgrexists(rels[i], j))
4739 continue;
4740 cached = false;
4741 break;
4742 }
4743
4744 /* calculate the total number of blocks to be invalidated */
4745 nBlocksToInvalidate += block[i][j];
4746 }
4747 }
4748
4749 /*
4750 * We apply the optimization iff the total number of blocks to invalidate
4751 * is below the BUF_DROP_FULL_SCAN_THRESHOLD.
4752 */
4753 if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD)
4754 {
4755 for (i = 0; i < n; i++)
4756 {
4757 for (int j = 0; j <= MAX_FORKNUM; j++)
4758 {
4759 /* ignore relation forks that doesn't exist */
4760 if (!BlockNumberIsValid(block[i][j]))
4761 continue;
4762
4763 /* drop all the buffers for a particular relation fork */
4764 FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator,
4765 j, block[i][j], 0);
4766 }
4767 }
4768
4769 pfree(block);
4770 pfree(rels);
4771 return;
4772 }
4773
4774 pfree(block);
4775 locators = palloc(sizeof(RelFileLocator) * n); /* non-local relations */
4776 for (i = 0; i < n; i++)
4777 locators[i] = rels[i]->smgr_rlocator.locator;
4778
4779 /*
4780 * For low number of relations to drop just use a simple walk through, to
4781 * save the bsearch overhead. The threshold to use is rather a guess than
4782 * an exactly determined value, as it depends on many factors (CPU and RAM
4783 * speeds, amount of shared buffers etc.).
4784 */
4785 use_bsearch = n > RELS_BSEARCH_THRESHOLD;
4786
4787 /* sort the list of rlocators if necessary */
4788 if (use_bsearch)
4789 qsort(locators, n, sizeof(RelFileLocator), rlocator_comparator);
4790
4791 for (i = 0; i < NBuffers; i++)
4792 {
4793 RelFileLocator *rlocator = NULL;
4794 BufferDesc *bufHdr = GetBufferDescriptor(i);
4795 uint32 buf_state;
4796
4797 /*
4798 * As in DropRelationBuffers, an unlocked precheck should be safe and
4799 * saves some cycles.
4800 */
4801
4802 if (!use_bsearch)
4803 {
4804 int j;
4805
4806 for (j = 0; j < n; j++)
4807 {
4808 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j]))
4809 {
4810 rlocator = &locators[j];
4811 break;
4812 }
4813 }
4814 }
4815 else
4816 {
4817 RelFileLocator locator;
4818
4819 locator = BufTagGetRelFileLocator(&bufHdr->tag);
4820 rlocator = bsearch(&locator,
4821 locators, n, sizeof(RelFileLocator),
4822 rlocator_comparator);
4823 }
4824
4825 /* buffer doesn't belong to any of the given relfilelocators; skip it */
4826 if (rlocator == NULL)
4827 continue;
4828
4829 buf_state = LockBufHdr(bufHdr);
4830 if (BufTagMatchesRelFileLocator(&bufHdr->tag, rlocator))
4831 InvalidateBuffer(bufHdr); /* releases spinlock */
4832 else
4833 UnlockBufHdr(bufHdr, buf_state);
4834 }
4835
4836 pfree(locators);
4837 pfree(rels);
4838 }
4839
4840 /* ---------------------------------------------------------------------
4841 * FindAndDropRelationBuffers
4842 *
4843 * This function performs look up in BufMapping table and removes from the
4844 * buffer pool all the pages of the specified relation fork that has block
4845 * number >= firstDelBlock. (In particular, with firstDelBlock = 0, all
4846 * pages are removed.)
4847 * --------------------------------------------------------------------
4848 */
4849 static void
4850 FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum,
4851 BlockNumber nForkBlock,
4852 BlockNumber firstDelBlock)
4853 {
4854 BlockNumber curBlock;
4855
4856 for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++)
4857 {
4858 uint32 bufHash; /* hash value for tag */
4859 BufferTag bufTag; /* identity of requested block */
4860 LWLock *bufPartitionLock; /* buffer partition lock for it */
4861 int buf_id;
4862 BufferDesc *bufHdr;
4863 uint32 buf_state;
4864
4865 /* create a tag so we can lookup the buffer */
4866 InitBufferTag(&bufTag, &rlocator, forkNum, curBlock);
4867
4868 /* determine its hash code and partition lock ID */
4869 bufHash = BufTableHashCode(&bufTag);
4870 bufPartitionLock = BufMappingPartitionLock(bufHash);
4871
4872 /* Check that it is in the buffer pool. If not, do nothing. */
4873 LWLockAcquire(bufPartitionLock, LW_SHARED);
4874 buf_id = BufTableLookup(&bufTag, bufHash);
4875 LWLockRelease(bufPartitionLock);
4876
4877 if (buf_id < 0)
4878 continue;
4879
4880 bufHdr = GetBufferDescriptor(buf_id);
4881
4882 /*
4883 * We need to lock the buffer header and recheck if the buffer is
4884 * still associated with the same block because the buffer could be
4885 * evicted by some other backend loading blocks for a different
4886 * relation after we release lock on the BufMapping table.
4887 */
4888 buf_state = LockBufHdr(bufHdr);
4889
4890 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) &&
4891 BufTagGetForkNum(&bufHdr->tag) == forkNum &&
4892 bufHdr->tag.blockNum >= firstDelBlock)
4893 InvalidateBuffer(bufHdr); /* releases spinlock */
4894 else
4895 UnlockBufHdr(bufHdr, buf_state);
4896 }
4897 }
4898
4899 /* ---------------------------------------------------------------------
4900 * DropDatabaseBuffers
4901 *
4902 * This function removes all the buffers in the buffer cache for a
4903 * particular database. Dirty pages are simply dropped, without
4904 * bothering to write them out first. This is used when we destroy a
4905 * database, to avoid trying to flush data to disk when the directory
4906 * tree no longer exists. Implementation is pretty similar to
4907 * DropRelationBuffers() which is for destroying just one relation.
4908 * --------------------------------------------------------------------
4909 */
4910 void
4911 DropDatabaseBuffers(Oid dbid)
4912 {
4913 int i;
4914
4915 /*
4916 * We needn't consider local buffers, since by assumption the target
4917 * database isn't our own.
4918 */
4919
4920 for (i = 0; i < NBuffers; i++)
4921 {
4922 BufferDesc *bufHdr = GetBufferDescriptor(i);
4923 uint32 buf_state;
4924
4925 /*
4926 * As in DropRelationBuffers, an unlocked precheck should be safe and
4927 * saves some cycles.
4928 */
4929 if (bufHdr->tag.dbOid != dbid)
4930 continue;
4931
4932 buf_state = LockBufHdr(bufHdr);
4933 if (bufHdr->tag.dbOid == dbid)
4934 InvalidateBuffer(bufHdr); /* releases spinlock */
4935 else
4936 UnlockBufHdr(bufHdr, buf_state);
4937 }
4938 }
4939
4940 /* ---------------------------------------------------------------------
4941 * FlushRelationBuffers
4942 *
4943 * This function writes all dirty pages of a relation out to disk
4944 * (or more accurately, out to kernel disk buffers), ensuring that the
4945 * kernel has an up-to-date view of the relation.
4946 *
4947 * Generally, the caller should be holding AccessExclusiveLock on the
4948 * target relation to ensure that no other backend is busy dirtying
4949 * more blocks of the relation; the effects can't be expected to last
4950 * after the lock is released.
4951 *
4952 * XXX currently it sequentially searches the buffer pool, should be
4953 * changed to more clever ways of searching. This routine is not
4954 * used in any performance-critical code paths, so it's not worth
4955 * adding additional overhead to normal paths to make it go faster.
4956 * --------------------------------------------------------------------
4957 */
4958 void
4959 FlushRelationBuffers(Relation rel)
4960 {
4961 int i;
4962 BufferDesc *bufHdr;
4963 SMgrRelation srel = RelationGetSmgr(rel);
4964
4965 if (RelationUsesLocalBuffers(rel))
4966 {
4967 for (i = 0; i < NLocBuffer; i++)
4968 {
4969 uint32 buf_state;
4970
4971 bufHdr = GetLocalBufferDescriptor(i);
4972 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) &&
4973 ((buf_state = pg_atomic_read_u32(&bufHdr->state)) &
4974 (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
4975 {
4976 ErrorContextCallback errcallback;
4977
4978 /* Setup error traceback support for ereport() */
4979 errcallback.callback = local_buffer_write_error_callback;
4980 errcallback.arg = bufHdr;
4981 errcallback.previous = error_context_stack;
4982 error_context_stack = &errcallback;
4983
4984 /* Make sure we can handle the pin */
4985 ReservePrivateRefCountEntry();
4986 ResourceOwnerEnlarge(CurrentResourceOwner);
4987
4988 /*
4989 * Pin/unpin mostly to make valgrind work, but it also seems
4990 * like the right thing to do.
4991 */
4992 PinLocalBuffer(bufHdr, false);
4993
4994
4995 FlushLocalBuffer(bufHdr, srel);
4996
4997 UnpinLocalBuffer(BufferDescriptorGetBuffer(bufHdr));
4998
4999 /* Pop the error context stack */
5000 error_context_stack = errcallback.previous;
5001 }
5002 }
5003
5004 return;
5005 }
5006
5007 for (i = 0; i < NBuffers; i++)
5008 {
5009 uint32 buf_state;
5010
5011 bufHdr = GetBufferDescriptor(i);
5012
5013 /*
5014 * As in DropRelationBuffers, an unlocked precheck should be safe and
5015 * saves some cycles.
5016 */
5017 if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator))
5018 continue;
5019
5020 /* Make sure we can handle the pin */
5021 ReservePrivateRefCountEntry();
5022 ResourceOwnerEnlarge(CurrentResourceOwner);
5023
5024 buf_state = LockBufHdr(bufHdr);
5025 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) &&
5026 (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
5027 {
5028 PinBuffer_Locked(bufHdr);
5029 LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
5030 FlushBuffer(bufHdr, srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
5031 LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
5032 UnpinBuffer(bufHdr);
5033 }
5034 else
5035 UnlockBufHdr(bufHdr, buf_state);
5036 }
5037 }
5038
5039 /* ---------------------------------------------------------------------
5040 * FlushRelationsAllBuffers
5041 *
5042 * This function flushes out of the buffer pool all the pages of all
5043 * forks of the specified smgr relations. It's equivalent to calling
5044 * FlushRelationBuffers once per relation. The relations are assumed not
5045 * to use local buffers.
5046 * --------------------------------------------------------------------
5047 */
5048 void
5049 FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels)
5050 {
5051 int i;
5052 SMgrSortArray *srels;
5053 bool use_bsearch;
5054
5055 if (nrels == 0)
5056 return;
5057
5058 /* fill-in array for qsort */
5059 srels = palloc(sizeof(SMgrSortArray) * nrels);
5060
5061 for (i = 0; i < nrels; i++)
5062 {
5063 Assert(!RelFileLocatorBackendIsTemp(smgrs[i]->smgr_rlocator));
5064
5065 srels[i].rlocator = smgrs[i]->smgr_rlocator.locator;
5066 srels[i].srel = smgrs[i];
5067 }
5068
5069 /*
5070 * Save the bsearch overhead for low number of relations to sync. See
5071 * DropRelationsAllBuffers for details.
5072 */
5073 use_bsearch = nrels > RELS_BSEARCH_THRESHOLD;
5074
5075 /* sort the list of SMgrRelations if necessary */
5076 if (use_bsearch)
5077 qsort(srels, nrels, sizeof(SMgrSortArray), rlocator_comparator);
5078
5079 for (i = 0; i < NBuffers; i++)
5080 {
5081 SMgrSortArray *srelent = NULL;
5082 BufferDesc *bufHdr = GetBufferDescriptor(i);
5083 uint32 buf_state;
5084
5085 /*
5086 * As in DropRelationBuffers, an unlocked precheck should be safe and
5087 * saves some cycles.
5088 */
5089
5090 if (!use_bsearch)
5091 {
5092 int j;
5093
5094 for (j = 0; j < nrels; j++)
5095 {
5096 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator))
5097 {
5098 srelent = &srels[j];
5099 break;
5100 }
5101 }
5102 }
5103 else
5104 {
5105 RelFileLocator rlocator;
5106
5107 rlocator = BufTagGetRelFileLocator(&bufHdr->tag);
5108 srelent = bsearch(&rlocator,
5109 srels, nrels, sizeof(SMgrSortArray),
5110 rlocator_comparator);
5111 }
5112
5113 /* buffer doesn't belong to any of the given relfilelocators; skip it */
5114 if (srelent == NULL)
5115 continue;
5116
5117 /* Make sure we can handle the pin */
5118 ReservePrivateRefCountEntry();
5119 ResourceOwnerEnlarge(CurrentResourceOwner);
5120
5121 buf_state = LockBufHdr(bufHdr);
5122 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) &&
5123 (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
5124 {
5125 PinBuffer_Locked(bufHdr);
5126 LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
5127 FlushBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
5128 LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
5129 UnpinBuffer(bufHdr);
5130 }
5131 else
5132 UnlockBufHdr(bufHdr, buf_state);
5133 }
5134
5135 pfree(srels);
5136 }
5137
5138 /* ---------------------------------------------------------------------
5139 * RelationCopyStorageUsingBuffer
5140 *
5141 * Copy fork's data using bufmgr. Same as RelationCopyStorage but instead
5142 * of using smgrread and smgrextend this will copy using bufmgr APIs.
5143 *
5144 * Refer comments atop CreateAndCopyRelationData() for details about
5145 * 'permanent' parameter.
5146 * --------------------------------------------------------------------
5147 */
5148 static void
5149 RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
5150 RelFileLocator dstlocator,
5151 ForkNumber forkNum, bool permanent)
5152 {
5153 Buffer srcBuf;
5154 Buffer dstBuf;
5155 Page srcPage;
5156 Page dstPage;
5157 bool use_wal;
5158 BlockNumber nblocks;
5159 BlockNumber blkno;
5160 PGIOAlignedBlock buf;
5161 BufferAccessStrategy bstrategy_src;
5162 BufferAccessStrategy bstrategy_dst;
5163 BlockRangeReadStreamPrivate p;
5164 ReadStream *src_stream;
5165 SMgrRelation src_smgr;
5166
5167 /*
5168 * In general, we want to write WAL whenever wal_level > 'minimal', but we
5169 * can skip it when copying any fork of an unlogged relation other than
5170 * the init fork.
5171 */
5172 use_wal = XLogIsNeeded() && (permanent || forkNum == INIT_FORKNUM);
5173
5174 /* Get number of blocks in the source relation. */
5175 nblocks = smgrnblocks(smgropen(srclocator, INVALID_PROC_NUMBER),
5176 forkNum);
5177
5178 /* Nothing to copy; just return. */
5179 if (nblocks == 0)
5180 return;
5181
5182 /*
5183 * Bulk extend the destination relation of the same size as the source
5184 * relation before starting to copy block by block.
5185 */
5186 memset(buf.data, 0, BLCKSZ);
5187 smgrextend(smgropen(dstlocator, INVALID_PROC_NUMBER), forkNum, nblocks - 1,
5188 buf.data, true);
5189
5190 /* This is a bulk operation, so use buffer access strategies. */
5191 bstrategy_src = GetAccessStrategy(BAS_BULKREAD);
5192 bstrategy_dst = GetAccessStrategy(BAS_BULKWRITE);
5193
5194 /* Initialize streaming read */
5195 p.current_blocknum = 0;
5196 p.last_exclusive = nblocks;
5197 src_smgr = smgropen(srclocator, INVALID_PROC_NUMBER);
5198
5199 /*
5200 * It is safe to use batchmode as block_range_read_stream_cb takes no
5201 * locks.
5202 */
5203 src_stream = read_stream_begin_smgr_relation(READ_STREAM_FULL |
5204 READ_STREAM_USE_BATCHING,
5205 bstrategy_src,
5206 src_smgr,
5207 permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED,
5208 forkNum,
5209 block_range_read_stream_cb,
5210 &p,
5211 0);
5212
5213 /* Iterate over each block of the source relation file. */
5214 for (blkno = 0; blkno < nblocks; blkno++)
5215 {
5216 CHECK_FOR_INTERRUPTS();
5217
5218 /* Read block from source relation. */
5219 srcBuf = read_stream_next_buffer(src_stream, NULL);
5220 LockBuffer(srcBuf, BUFFER_LOCK_SHARE);
5221 srcPage = BufferGetPage(srcBuf);
5222
5223 dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum,
5224 BufferGetBlockNumber(srcBuf),
5225 RBM_ZERO_AND_LOCK, bstrategy_dst,
5226 permanent);
5227 dstPage = BufferGetPage(dstBuf);
5228
5229 START_CRIT_SECTION();
5230
5231 /* Copy page data from the source to the destination. */
5232 memcpy(dstPage, srcPage, BLCKSZ);
5233 MarkBufferDirty(dstBuf);
5234
5235 /* WAL-log the copied page. */
5236 if (use_wal)
5237 log_newpage_buffer(dstBuf, true);
5238
5239 END_CRIT_SECTION();
5240
5241 UnlockReleaseBuffer(dstBuf);
5242 UnlockReleaseBuffer(srcBuf);
5243 }
5244 Assert(read_stream_next_buffer(src_stream, NULL) == InvalidBuffer);
5245 read_stream_end(src_stream);
5246
5247 FreeAccessStrategy(bstrategy_src);
5248 FreeAccessStrategy(bstrategy_dst);
5249 }
5250
5251 /* ---------------------------------------------------------------------
5252 * CreateAndCopyRelationData
5253 *
5254 * Create destination relation storage and copy all forks from the
5255 * source relation to the destination.
5256 *
5257 * Pass permanent as true for permanent relations and false for
5258 * unlogged relations. Currently this API is not supported for
5259 * temporary relations.
5260 * --------------------------------------------------------------------
5261 */
5262 void
5263 CreateAndCopyRelationData(RelFileLocator src_rlocator,
5264 RelFileLocator dst_rlocator, bool permanent)
5265 {
5266 char relpersistence;
5267 SMgrRelation src_rel;
5268 SMgrRelation dst_rel;
5269
5270 /* Set the relpersistence. */
5271 relpersistence = permanent ?
5272 RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED;
5273
5274 src_rel = smgropen(src_rlocator, INVALID_PROC_NUMBER);
5275 dst_rel = smgropen(dst_rlocator, INVALID_PROC_NUMBER);
5276
5277 /*
5278 * Create and copy all forks of the relation. During create database we
5279 * have a separate cleanup mechanism which deletes complete database
5280 * directory. Therefore, each individual relation doesn't need to be
5281 * registered for cleanup.
5282 */
5283 RelationCreateStorage(dst_rlocator, relpersistence, false);
5284
5285 /* copy main fork. */
5286 RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM,
5287 permanent);
5288
5289 /* copy those extra forks that exist */
5290 for (ForkNumber forkNum = MAIN_FORKNUM + 1;
5291 forkNum <= MAX_FORKNUM; forkNum++)
5292 {
5293 if (smgrexists(src_rel, forkNum))
5294 {
5295 smgrcreate(dst_rel, forkNum, false);
5296
5297 /*
5298 * WAL log creation if the relation is persistent, or this is the
5299 * init fork of an unlogged relation.
5300 */
5301 if (permanent || forkNum == INIT_FORKNUM)
5302 log_smgrcreate(&dst_rlocator, forkNum);
5303
5304 /* Copy a fork's data, block by block. */
5305 RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum,
5306 permanent);
5307 }
5308 }
5309 }
5310
5311 /* ---------------------------------------------------------------------
5312 * FlushDatabaseBuffers
5313 *
5314 * This function writes all dirty pages of a database out to disk
5315 * (or more accurately, out to kernel disk buffers), ensuring that the
5316 * kernel has an up-to-date view of the database.
5317 *
5318 * Generally, the caller should be holding an appropriate lock to ensure
5319 * no other backend is active in the target database; otherwise more
5320 * pages could get dirtied.
5321 *
5322 * Note we don't worry about flushing any pages of temporary relations.
5323 * It's assumed these wouldn't be interesting.
5324 * --------------------------------------------------------------------
5325 */
5326 void
5327 FlushDatabaseBuffers(Oid dbid)
5328 {
5329 int i;
5330 BufferDesc *bufHdr;
5331
5332 for (i = 0; i < NBuffers; i++)
5333 {
5334 uint32 buf_state;
5335
5336 bufHdr = GetBufferDescriptor(i);
5337
5338 /*
5339 * As in DropRelationBuffers, an unlocked precheck should be safe and
5340 * saves some cycles.
5341 */
5342 if (bufHdr->tag.dbOid != dbid)
5343 continue;
5344
5345 /* Make sure we can handle the pin */
5346 ReservePrivateRefCountEntry();
5347 ResourceOwnerEnlarge(CurrentResourceOwner);
5348
5349 buf_state = LockBufHdr(bufHdr);
5350 if (bufHdr->tag.dbOid == dbid &&
5351 (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
5352 {
5353 PinBuffer_Locked(bufHdr);
5354 LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
5355 FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
5356 LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
5357 UnpinBuffer(bufHdr);
5358 }
5359 else
5360 UnlockBufHdr(bufHdr, buf_state);
5361 }
5362 }
5363
5364 /*
5365 * Flush a previously, shared or exclusively, locked and pinned buffer to the
5366 * OS.
5367 */
5368 void
5369 FlushOneBuffer(Buffer buffer)
5370 {
5371 BufferDesc *bufHdr;
5372
5373 /* currently not needed, but no fundamental reason not to support */
5374 Assert(!BufferIsLocal(buffer));
5375
5376 Assert(BufferIsPinned(buffer));
5377
5378 bufHdr = GetBufferDescriptor(buffer - 1);
5379
5380 Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr)));
5381
5382 FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
5383 }
5384
5385 /*
5386 * ReleaseBuffer -- release the pin on a buffer
5387 */
5388 void
5389 ReleaseBuffer(Buffer buffer)
5390 {
5391 if (!BufferIsValid(buffer))
5392 elog(ERROR, "bad buffer ID: %d", buffer);
5393
5394 if (BufferIsLocal(buffer))
5395 UnpinLocalBuffer(buffer);
5396 else
5397 UnpinBuffer(GetBufferDescriptor(buffer - 1));
5398 }
5399
5400 /*
5401 * UnlockReleaseBuffer -- release the content lock and pin on a buffer
5402 *
5403 * This is just a shorthand for a common combination.
5404 */
5405 void
5406 UnlockReleaseBuffer(Buffer buffer)
5407 {
5408 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
5409 ReleaseBuffer(buffer);
5410 }
5411
5412 /*
5413 * IncrBufferRefCount
5414 * Increment the pin count on a buffer that we have *already* pinned
5415 * at least once.
5416 *
5417 * This function cannot be used on a buffer we do not have pinned,
5418 * because it doesn't change the shared buffer state.
5419 */
5420 void
5421 IncrBufferRefCount(Buffer buffer)
5422 {
5423 Assert(BufferIsPinned(buffer));
5424 ResourceOwnerEnlarge(CurrentResourceOwner);
5425 if (BufferIsLocal(buffer))
5426 LocalRefCount[-buffer - 1]++;
5427 else
5428 {
5429 PrivateRefCountEntry *ref;
5430
5431 ref = GetPrivateRefCountEntry(buffer, true);
5432 Assert(ref != NULL);
5433 ref->refcount++;
5434 }
5435 ResourceOwnerRememberBuffer(CurrentResourceOwner, buffer);
5436 }
5437
5438 /*
5439 * MarkBufferDirtyHint
5440 *
5441 * Mark a buffer dirty for non-critical changes.
5442 *
5443 * This is essentially the same as MarkBufferDirty, except:
5444 *
5445 * 1. The caller does not write WAL; so if checksums are enabled, we may need
5446 * to write an XLOG_FPI_FOR_HINT WAL record to protect against torn pages.
5447 * 2. The caller might have only share-lock instead of exclusive-lock on the
5448 * buffer's content lock.
5449 * 3. This function does not guarantee that the buffer is always marked dirty
5450 * (due to a race condition), so it cannot be used for important changes.
5451 */
5452 void
5453 MarkBufferDirtyHint(Buffer buffer, bool buffer_std)
5454 {
5455 BufferDesc *bufHdr;
5456 Page page = BufferGetPage(buffer);
5457
5458 if (!BufferIsValid(buffer))
5459 elog(ERROR, "bad buffer ID: %d", buffer);
5460
5461 if (BufferIsLocal(buffer))
5462 {
5463 MarkLocalBufferDirty(buffer);
5464 return;
5465 }
5466
5467 bufHdr = GetBufferDescriptor(buffer - 1);
5468
5469 Assert(GetPrivateRefCount(buffer) > 0);
5470 /* here, either share or exclusive lock is OK */
5471 Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr)));
5472
5473 /*
5474 * This routine might get called many times on the same page, if we are
5475 * making the first scan after commit of an xact that added/deleted many
5476 * tuples. So, be as quick as we can if the buffer is already dirty. We
5477 * do this by not acquiring spinlock if it looks like the status bits are
5478 * already set. Since we make this test unlocked, there's a chance we
5479 * might fail to notice that the flags have just been cleared, and failed
5480 * to reset them, due to memory-ordering issues. But since this function
5481 * is only intended to be used in cases where failing to write out the
5482 * data would be harmless anyway, it doesn't really matter.
5483 */
5484 if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) !=
5485 (BM_DIRTY | BM_JUST_DIRTIED))
5486 {
5487 XLogRecPtr lsn = InvalidXLogRecPtr;
5488 bool dirtied = false;
5489 bool delayChkptFlags = false;
5490 uint32 buf_state;
5491
5492 /*
5493 * If we need to protect hint bit updates from torn writes, WAL-log a
5494 * full page image of the page. This full page image is only necessary
5495 * if the hint bit update is the first change to the page since the
5496 * last checkpoint.
5497 *
5498 * We don't check full_page_writes here because that logic is included
5499 * when we call XLogInsert() since the value changes dynamically.
5500 */
5501 if (XLogHintBitIsNeeded() &&
5502 (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT))
5503 {
5504 /*
5505 * If we must not write WAL, due to a relfilelocator-specific
5506 * condition or being in recovery, don't dirty the page. We can
5507 * set the hint, just not dirty the page as a result so the hint
5508 * is lost when we evict the page or shutdown.
5509 *
5510 * See src/backend/storage/page/README for longer discussion.
5511 */
5512 if (RecoveryInProgress() ||
5513 RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag)))
5514 return;
5515
5516 /*
5517 * If the block is already dirty because we either made a change
5518 * or set a hint already, then we don't need to write a full page
5519 * image. Note that aggressive cleaning of blocks dirtied by hint
5520 * bit setting would increase the call rate. Bulk setting of hint
5521 * bits would reduce the call rate...
5522 *
5523 * We must issue the WAL record before we mark the buffer dirty.
5524 * Otherwise we might write the page before we write the WAL. That
5525 * causes a race condition, since a checkpoint might occur between
5526 * writing the WAL record and marking the buffer dirty. We solve
5527 * that with a kluge, but one that is already in use during
5528 * transaction commit to prevent race conditions. Basically, we
5529 * simply prevent the checkpoint WAL record from being written
5530 * until we have marked the buffer dirty. We don't start the
5531 * checkpoint flush until we have marked dirty, so our checkpoint
5532 * must flush the change to disk successfully or the checkpoint
5533 * never gets written, so crash recovery will fix.
5534 *
5535 * It's possible we may enter here without an xid, so it is
5536 * essential that CreateCheckPoint waits for virtual transactions
5537 * rather than full transactionids.
5538 */
5539 Assert((MyProc->delayChkptFlags & DELAY_CHKPT_START) == 0);
5540 MyProc->delayChkptFlags |= DELAY_CHKPT_START;
5541 delayChkptFlags = true;
5542 lsn = XLogSaveBufferForHint(buffer, buffer_std);
5543 }
5544
5545 buf_state = LockBufHdr(bufHdr);
5546
5547 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
5548
5549 if (!(buf_state & BM_DIRTY))
5550 {
5551 dirtied = true; /* Means "will be dirtied by this action" */
5552
5553 /*
5554 * Set the page LSN if we wrote a backup block. We aren't supposed
5555 * to set this when only holding a share lock but as long as we
5556 * serialise it somehow we're OK. We choose to set LSN while
5557 * holding the buffer header lock, which causes any reader of an
5558 * LSN who holds only a share lock to also obtain a buffer header
5559 * lock before using PageGetLSN(), which is enforced in
5560 * BufferGetLSNAtomic().
5561 *
5562 * If checksums are enabled, you might think we should reset the
5563 * checksum here. That will happen when the page is written
5564 * sometime later in this checkpoint cycle.
5565 */
5566 if (!XLogRecPtrIsInvalid(lsn))
5567 PageSetLSN(page, lsn);
5568 }
5569
5570 buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
5571 UnlockBufHdr(bufHdr, buf_state);
5572
5573 if (delayChkptFlags)
5574 MyProc->delayChkptFlags &= ~DELAY_CHKPT_START;
5575
5576 if (dirtied)
5577 {
5578 pgBufferUsage.shared_blks_dirtied++;
5579 if (VacuumCostActive)
5580 VacuumCostBalance += VacuumCostPageDirty;
5581 }
5582 }
5583 }
5584
5585 /*
5586 * Release buffer content locks for shared buffers.
5587 *
5588 * Used to clean up after errors.
5589 *
5590 * Currently, we can expect that lwlock.c's LWLockReleaseAll() took care
5591 * of releasing buffer content locks per se; the only thing we need to deal
5592 * with here is clearing any PIN_COUNT request that was in progress.
5593 */
5594 void
5595 UnlockBuffers(void)
5596 {
5597 BufferDesc *buf = PinCountWaitBuf;
5598
5599 if (buf)
5600 {
5601 uint32 buf_state;
5602
5603 buf_state = LockBufHdr(buf);
5604
5605 /*
5606 * Don't complain if flag bit not set; it could have been reset but we
5607 * got a cancel/die interrupt before getting the signal.
5608 */
5609 if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
5610 buf->wait_backend_pgprocno == MyProcNumber)
5611 buf_state &= ~BM_PIN_COUNT_WAITER;
5612
5613 UnlockBufHdr(buf, buf_state);
5614
5615 PinCountWaitBuf = NULL;
5616 }
5617 }
5618
5619 /*
5620 * Acquire or release the content_lock for the buffer.
5621 */
5622 void
5623 LockBuffer(Buffer buffer, int mode)
5624 {
5625 BufferDesc *buf;
5626
5627 Assert(BufferIsPinned(buffer));
5628 if (BufferIsLocal(buffer))
5629 return; /* local buffers need no lock */
5630
5631 buf = GetBufferDescriptor(buffer - 1);
5632
5633 if (mode == BUFFER_LOCK_UNLOCK)
5634 LWLockRelease(BufferDescriptorGetContentLock(buf));
5635 else if (mode == BUFFER_LOCK_SHARE)
5636 LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED);
5637 else if (mode == BUFFER_LOCK_EXCLUSIVE)
5638 LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE);
5639 else
5640 elog(ERROR, "unrecognized buffer lock mode: %d", mode);
5641 }
5642
5643 /*
5644 * Acquire the content_lock for the buffer, but only if we don't have to wait.
5645 *
5646 * This assumes the caller wants BUFFER_LOCK_EXCLUSIVE mode.
5647 */
5648 bool
5649 ConditionalLockBuffer(Buffer buffer)
5650 {
5651 BufferDesc *buf;
5652
5653 Assert(BufferIsPinned(buffer));
5654 if (BufferIsLocal(buffer))
5655 return true; /* act as though we got it */
5656
5657 buf = GetBufferDescriptor(buffer - 1);
5658
5659 return LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf),
5660 LW_EXCLUSIVE);
5661 }
5662
5663 /*
5664 * Verify that this backend is pinning the buffer exactly once.
5665 *
5666 * NOTE: Like in BufferIsPinned(), what we check here is that *this* backend
5667 * holds a pin on the buffer. We do not care whether some other backend does.
5668 */
5669 void
5670 CheckBufferIsPinnedOnce(Buffer buffer)
5671 {
5672 if (BufferIsLocal(buffer))
5673 {
5674 if (LocalRefCount[-buffer - 1] != 1)
5675 elog(ERROR, "incorrect local pin count: %d",
5676 LocalRefCount[-buffer - 1]);
5677 }
5678 else
5679 {
5680 if (GetPrivateRefCount(buffer) != 1)
5681 elog(ERROR, "incorrect local pin count: %d",
5682 GetPrivateRefCount(buffer));
5683 }
5684 }
5685
5686 /*
5687 * LockBufferForCleanup - lock a buffer in preparation for deleting items
5688 *
5689 * Items may be deleted from a disk page only when the caller (a) holds an
5690 * exclusive lock on the buffer and (b) has observed that no other backend
5691 * holds a pin on the buffer. If there is a pin, then the other backend
5692 * might have a pointer into the buffer (for example, a heapscan reference
5693 * to an item --- see README for more details). It's OK if a pin is added
5694 * after the cleanup starts, however; the newly-arrived backend will be
5695 * unable to look at the page until we release the exclusive lock.
5696 *
5697 * To implement this protocol, a would-be deleter must pin the buffer and
5698 * then call LockBufferForCleanup(). LockBufferForCleanup() is similar to
5699 * LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE), except that it loops until
5700 * it has successfully observed pin count = 1.
5701 */
5702 void
5703 LockBufferForCleanup(Buffer buffer)
5704 {
5705 BufferDesc *bufHdr;
5706 TimestampTz waitStart = 0;
5707 bool waiting = false;
5708 bool logged_recovery_conflict = false;
5709
5710 Assert(BufferIsPinned(buffer));
5711 Assert(PinCountWaitBuf == NULL);
5712
5713 CheckBufferIsPinnedOnce(buffer);
5714
5715 /*
5716 * We do not yet need to be worried about in-progress AIOs holding a pin,
5717 * as we, so far, only support doing reads via AIO and this function can
5718 * only be called once the buffer is valid (i.e. no read can be in
5719 * flight).
5720 */
5721
5722 /* Nobody else to wait for */
5723 if (BufferIsLocal(buffer))
5724 return;
5725
5726 bufHdr = GetBufferDescriptor(buffer - 1);
5727
5728 for (;;)
5729 {
5730 uint32 buf_state;
5731
5732 /* Try to acquire lock */
5733 LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
5734 buf_state = LockBufHdr(bufHdr);
5735
5736 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
5737 if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
5738 {
5739 /* Successfully acquired exclusive lock with pincount 1 */
5740 UnlockBufHdr(bufHdr, buf_state);
5741
5742 /*
5743 * Emit the log message if recovery conflict on buffer pin was
5744 * resolved but the startup process waited longer than
5745 * deadlock_timeout for it.
5746 */
5747 if (logged_recovery_conflict)
5748 LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN,
5749 waitStart, GetCurrentTimestamp(),
5750 NULL, false);
5751
5752 if (waiting)
5753 {
5754 /* reset ps display to remove the suffix if we added one */
5755 set_ps_display_remove_suffix();
5756 waiting = false;
5757 }
5758 return;
5759 }
5760 /* Failed, so mark myself as waiting for pincount 1 */
5761 if (buf_state & BM_PIN_COUNT_WAITER)
5762 {
5763 UnlockBufHdr(bufHdr, buf_state);
5764 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
5765 elog(ERROR, "multiple backends attempting to wait for pincount 1");
5766 }
5767 bufHdr->wait_backend_pgprocno = MyProcNumber;
5768 PinCountWaitBuf = bufHdr;
5769 buf_state |= BM_PIN_COUNT_WAITER;
5770 UnlockBufHdr(bufHdr, buf_state);
5771 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
5772
5773 /* Wait to be signaled by UnpinBuffer() */
5774 if (InHotStandby)
5775 {
5776 if (!waiting)
5777 {
5778 /* adjust the process title to indicate that it's waiting */
5779 set_ps_display_suffix("waiting");
5780 waiting = true;
5781 }
5782
5783 /*
5784 * Emit the log message if the startup process is waiting longer
5785 * than deadlock_timeout for recovery conflict on buffer pin.
5786 *
5787 * Skip this if first time through because the startup process has
5788 * not started waiting yet in this case. So, the wait start
5789 * timestamp is set after this logic.
5790 */
5791 if (waitStart != 0 && !logged_recovery_conflict)
5792 {
5793 TimestampTz now = GetCurrentTimestamp();
5794
5795 if (TimestampDifferenceExceeds(waitStart, now,
5796 DeadlockTimeout))
5797 {
5798 LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN,
5799 waitStart, now, NULL, true);
5800 logged_recovery_conflict = true;
5801 }
5802 }
5803
5804 /*
5805 * Set the wait start timestamp if logging is enabled and first
5806 * time through.
5807 */
5808 if (log_recovery_conflict_waits && waitStart == 0)
5809 waitStart = GetCurrentTimestamp();
5810
5811 /* Publish the bufid that Startup process waits on */
5812 SetStartupBufferPinWaitBufId(buffer - 1);
5813 /* Set alarm and then wait to be signaled by UnpinBuffer() */
5814 ResolveRecoveryConflictWithBufferPin();
5815 /* Reset the published bufid */
5816 SetStartupBufferPinWaitBufId(-1);
5817 }
5818 else
5819 ProcWaitForSignal(WAIT_EVENT_BUFFER_PIN);
5820
5821 /*
5822 * Remove flag marking us as waiter. Normally this will not be set
5823 * anymore, but ProcWaitForSignal() can return for other signals as
5824 * well. We take care to only reset the flag if we're the waiter, as
5825 * theoretically another backend could have started waiting. That's
5826 * impossible with the current usages due to table level locking, but
5827 * better be safe.
5828 */
5829 buf_state = LockBufHdr(bufHdr);
5830 if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
5831 bufHdr->wait_backend_pgprocno == MyProcNumber)
5832 buf_state &= ~BM_PIN_COUNT_WAITER;
5833 UnlockBufHdr(bufHdr, buf_state);
5834
5835 PinCountWaitBuf = NULL;
5836 /* Loop back and try again */
5837 }
5838 }
5839
5840 /*
5841 * Check called from ProcessRecoveryConflictInterrupts() when Startup process
5842 * requests cancellation of all pin holders that are blocking it.
5843 */
5844 bool
5845 HoldingBufferPinThatDelaysRecovery(void)
5846 {
5847 int bufid = GetStartupBufferPinWaitBufId();
5848
5849 /*
5850 * If we get woken slowly then it's possible that the Startup process was
5851 * already woken by other backends before we got here. Also possible that
5852 * we get here by multiple interrupts or interrupts at inappropriate
5853 * times, so make sure we do nothing if the bufid is not set.
5854 */
5855 if (bufid < 0)
5856 return false;
5857
5858 if (GetPrivateRefCount(bufid + 1) > 0)
5859 return true;
5860
5861 return false;
5862 }
5863
5864 /*
5865 * ConditionalLockBufferForCleanup - as above, but don't wait to get the lock
5866 *
5867 * We won't loop, but just check once to see if the pin count is OK. If
5868 * not, return false with no lock held.
5869 */
5870 bool
5871 ConditionalLockBufferForCleanup(Buffer buffer)
5872 {
5873 BufferDesc *bufHdr;
5874 uint32 buf_state,
5875 refcount;
5876
5877 Assert(BufferIsValid(buffer));
5878
5879 /* see AIO related comment in LockBufferForCleanup() */
5880
5881 if (BufferIsLocal(buffer))
5882 {
5883 refcount = LocalRefCount[-buffer - 1];
5884 /* There should be exactly one pin */
5885 Assert(refcount > 0);
5886 if (refcount != 1)
5887 return false;
5888 /* Nobody else to wait for */
5889 return true;
5890 }
5891
5892 /* There should be exactly one local pin */
5893 refcount = GetPrivateRefCount(buffer);
5894 Assert(refcount);
5895 if (refcount != 1)
5896 return false;
5897
5898 /* Try to acquire lock */
5899 if (!ConditionalLockBuffer(buffer))
5900 return false;
5901
5902 bufHdr = GetBufferDescriptor(buffer - 1);
5903 buf_state = LockBufHdr(bufHdr);
5904 refcount = BUF_STATE_GET_REFCOUNT(buf_state);
5905
5906 Assert(refcount > 0);
5907 if (refcount == 1)
5908 {
5909 /* Successfully acquired exclusive lock with pincount 1 */
5910 UnlockBufHdr(bufHdr, buf_state);
5911 return true;
5912 }
5913
5914 /* Failed, so release the lock */
5915 UnlockBufHdr(bufHdr, buf_state);
5916 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
5917 return false;
5918 }
5919
5920 /*
5921 * IsBufferCleanupOK - as above, but we already have the lock
5922 *
5923 * Check whether it's OK to perform cleanup on a buffer we've already
5924 * locked. If we observe that the pin count is 1, our exclusive lock
5925 * happens to be a cleanup lock, and we can proceed with anything that
5926 * would have been allowable had we sought a cleanup lock originally.
5927 */
5928 bool
5929 IsBufferCleanupOK(Buffer buffer)
5930 {
5931 BufferDesc *bufHdr;
5932 uint32 buf_state;
5933
5934 Assert(BufferIsValid(buffer));
5935
5936 /* see AIO related comment in LockBufferForCleanup() */
5937
5938 if (BufferIsLocal(buffer))
5939 {
5940 /* There should be exactly one pin */
5941 if (LocalRefCount[-buffer - 1] != 1)
5942 return false;
5943 /* Nobody else to wait for */
5944 return true;
5945 }
5946
5947 /* There should be exactly one local pin */
5948 if (GetPrivateRefCount(buffer) != 1)
5949 return false;
5950
5951 bufHdr = GetBufferDescriptor(buffer - 1);
5952
5953 /* caller must hold exclusive lock on buffer */
5954 Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
5955 LW_EXCLUSIVE));
5956
5957 buf_state = LockBufHdr(bufHdr);
5958
5959 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
5960 if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
5961 {
5962 /* pincount is OK. */
5963 UnlockBufHdr(bufHdr, buf_state);
5964 return true;
5965 }
5966
5967 UnlockBufHdr(bufHdr, buf_state);
5968 return false;
5969 }
5970
5971
5972 /*
5973 * Functions for buffer I/O handling
5974 *
5975 * Also note that these are used only for shared buffers, not local ones.
5976 */
5977
5978 /*
5979 * WaitIO -- Block until the IO_IN_PROGRESS flag on 'buf' is cleared.
5980 */
5981 static void
5982 WaitIO(BufferDesc *buf)
5983 {
5984 ConditionVariable *cv = BufferDescriptorGetIOCV(buf);
5985
5986 ConditionVariablePrepareToSleep(cv);
5987 for (;;)
5988 {
5989 uint32 buf_state;
5990 PgAioWaitRef iow;
5991
5992 /*
5993 * It may not be necessary to acquire the spinlock to check the flag
5994 * here, but since this test is essential for correctness, we'd better
5995 * play it safe.
5996 */
5997 buf_state = LockBufHdr(buf);
5998
5999 /*
6000 * Copy the wait reference while holding the spinlock. This protects
6001 * against a concurrent TerminateBufferIO() in another backend from
6002 * clearing the wref while it's being read.
6003 */
6004 iow = buf->io_wref;
6005 UnlockBufHdr(buf, buf_state);
6006
6007 /* no IO in progress, we don't need to wait */
6008 if (!(buf_state & BM_IO_IN_PROGRESS))
6009 break;
6010
6011 /*
6012 * The buffer has asynchronous IO in progress, wait for it to
6013 * complete.
6014 */
6015 if (pgaio_wref_valid(&iow))
6016 {
6017 pgaio_wref_wait(&iow);
6018
6019 /*
6020 * The AIO subsystem internally uses condition variables and thus
6021 * might remove this backend from the BufferDesc's CV. While that
6022 * wouldn't cause a correctness issue (the first CV sleep just
6023 * immediately returns if not already registered), it seems worth
6024 * avoiding unnecessary loop iterations, given that we take care
6025 * to do so at the start of the function.
6026 */
6027 ConditionVariablePrepareToSleep(cv);
6028 continue;
6029 }
6030
6031 /* wait on BufferDesc->cv, e.g. for concurrent synchronous IO */
6032 ConditionVariableSleep(cv, WAIT_EVENT_BUFFER_IO);
6033 }
6034 ConditionVariableCancelSleep();
6035 }
6036
6037 /*
6038 * StartBufferIO: begin I/O on this buffer
6039 * (Assumptions)
6040 * My process is executing no IO on this buffer
6041 * The buffer is Pinned
6042 *
6043 * In some scenarios multiple backends could attempt the same I/O operation
6044 * concurrently. If someone else has already started I/O on this buffer then
6045 * we will wait for completion of the IO using WaitIO().
6046 *
6047 * Input operations are only attempted on buffers that are not BM_VALID,
6048 * and output operations only on buffers that are BM_VALID and BM_DIRTY,
6049 * so we can always tell if the work is already done.
6050 *
6051 * Returns true if we successfully marked the buffer as I/O busy,
6052 * false if someone else already did the work.
6053 *
6054 * If nowait is true, then we don't wait for an I/O to be finished by another
6055 * backend. In that case, false indicates either that the I/O was already
6056 * finished, or is still in progress. This is useful for callers that want to
6057 * find out if they can perform the I/O as part of a larger operation, without
6058 * waiting for the answer or distinguishing the reasons why not.
6059 */
6060 bool
6061 StartBufferIO(BufferDesc *buf, bool forInput, bool nowait)
6062 {
6063 uint32 buf_state;
6064
6065 ResourceOwnerEnlarge(CurrentResourceOwner);
6066
6067 for (;;)
6068 {
6069 buf_state = LockBufHdr(buf);
6070
6071 if (!(buf_state & BM_IO_IN_PROGRESS))
6072 break;
6073 UnlockBufHdr(buf, buf_state);
6074 if (nowait)
6075 return false;
6076 WaitIO(buf);
6077 }
6078
6079 /* Once we get here, there is definitely no I/O active on this buffer */
6080
6081 /* Check if someone else already did the I/O */
6082 if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
6083 {
6084 UnlockBufHdr(buf, buf_state);
6085 return false;
6086 }
6087
6088 buf_state |= BM_IO_IN_PROGRESS;
6089 UnlockBufHdr(buf, buf_state);
6090
6091 ResourceOwnerRememberBufferIO(CurrentResourceOwner,
6092 BufferDescriptorGetBuffer(buf));
6093
6094 return true;
6095 }
6096
6097 /*
6098 * TerminateBufferIO: release a buffer we were doing I/O on
6099 * (Assumptions)
6100 * My process is executing IO for the buffer
6101 * BM_IO_IN_PROGRESS bit is set for the buffer
6102 * The buffer is Pinned
6103 *
6104 * If clear_dirty is true and BM_JUST_DIRTIED is not set, we clear the
6105 * buffer's BM_DIRTY flag. This is appropriate when terminating a
6106 * successful write. The check on BM_JUST_DIRTIED is necessary to avoid
6107 * marking the buffer clean if it was re-dirtied while we were writing.
6108 *
6109 * set_flag_bits gets ORed into the buffer's flags. It must include
6110 * BM_IO_ERROR in a failure case. For successful completion it could
6111 * be 0, or BM_VALID if we just finished reading in the page.
6112 *
6113 * If forget_owner is true, we release the buffer I/O from the current
6114 * resource owner. (forget_owner=false is used when the resource owner itself
6115 * is being released)
6116 */
6117 void
6118 TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits,
6119 bool forget_owner, bool release_aio)
6120 {
6121 uint32 buf_state;
6122
6123 buf_state = LockBufHdr(buf);
6124
6125 Assert(buf_state & BM_IO_IN_PROGRESS);
6126 buf_state &= ~BM_IO_IN_PROGRESS;
6127
6128 /* Clear earlier errors, if this IO failed, it'll be marked again */
6129 buf_state &= ~BM_IO_ERROR;
6130
6131 if (clear_dirty && !(buf_state & BM_JUST_DIRTIED))
6132 buf_state &= ~(BM_DIRTY | BM_CHECKPOINT_NEEDED);
6133
6134 if (release_aio)
6135 {
6136 /* release ownership by the AIO subsystem */
6137 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
6138 buf_state -= BUF_REFCOUNT_ONE;
6139 pgaio_wref_clear(&buf->io_wref);
6140 }
6141
6142 buf_state |= set_flag_bits;
6143 UnlockBufHdr(buf, buf_state);
6144
6145 if (forget_owner)
6146 ResourceOwnerForgetBufferIO(CurrentResourceOwner,
6147 BufferDescriptorGetBuffer(buf));
6148
6149 ConditionVariableBroadcast(BufferDescriptorGetIOCV(buf));
6150
6151 /*
6152 * Support LockBufferForCleanup()
6153 *
6154 * We may have just released the last pin other than the waiter's. In most
6155 * cases, this backend holds another pin on the buffer. But, if, for
6156 * example, this backend is completing an IO issued by another backend, it
6157 * may be time to wake the waiter.
6158 */
6159 if (release_aio && (buf_state & BM_PIN_COUNT_WAITER))
6160 WakePinCountWaiter(buf);
6161 }
6162
6163 /*
6164 * AbortBufferIO: Clean up active buffer I/O after an error.
6165 *
6166 * All LWLocks we might have held have been released,
6167 * but we haven't yet released buffer pins, so the buffer is still pinned.
6168 *
6169 * If I/O was in progress, we always set BM_IO_ERROR, even though it's
6170 * possible the error condition wasn't related to the I/O.
6171 *
6172 * Note: this does not remove the buffer I/O from the resource owner.
6173 * That's correct when we're releasing the whole resource owner, but
6174 * beware if you use this in other contexts.
6175 */
6176 static void
6177 AbortBufferIO(Buffer buffer)
6178 {
6179 BufferDesc *buf_hdr = GetBufferDescriptor(buffer - 1);
6180 uint32 buf_state;
6181
6182 buf_state = LockBufHdr(buf_hdr);
6183 Assert(buf_state & (BM_IO_IN_PROGRESS | BM_TAG_VALID));
6184
6185 if (!(buf_state & BM_VALID))
6186 {
6187 Assert(!(buf_state & BM_DIRTY));
6188 UnlockBufHdr(buf_hdr, buf_state);
6189 }
6190 else
6191 {
6192 Assert(buf_state & BM_DIRTY);
6193 UnlockBufHdr(buf_hdr, buf_state);
6194
6195 /* Issue notice if this is not the first failure... */
6196 if (buf_state & BM_IO_ERROR)
6197 {
6198 /* Buffer is pinned, so we can read tag without spinlock */
6199 ereport(WARNING,
6200 (errcode(ERRCODE_IO_ERROR),
6201 errmsg("could not write block %u of %s",
6202 buf_hdr->tag.blockNum,
6203 relpathperm(BufTagGetRelFileLocator(&buf_hdr->tag),
6204 BufTagGetForkNum(&buf_hdr->tag)).str),
6205 errdetail("Multiple failures --- write error might be permanent.")));
6206 }
6207 }
6208
6209 TerminateBufferIO(buf_hdr, false, BM_IO_ERROR, false, false);
6210 }
6211
6212 /*
6213 * Error context callback for errors occurring during shared buffer writes.
6214 */
6215 static void
6216 shared_buffer_write_error_callback(void *arg)
6217 {
6218 BufferDesc *bufHdr = (BufferDesc *) arg;
6219
6220 /* Buffer is pinned, so we can read the tag without locking the spinlock */
6221 if (bufHdr != NULL)
6222 errcontext("writing block %u of relation \"%s\"",
6223 bufHdr->tag.blockNum,
6224 relpathperm(BufTagGetRelFileLocator(&bufHdr->tag),
6225 BufTagGetForkNum(&bufHdr->tag)).str);
6226 }
6227
6228 /*
6229 * Error context callback for errors occurring during local buffer writes.
6230 */
6231 static void
6232 local_buffer_write_error_callback(void *arg)
6233 {
6234 BufferDesc *bufHdr = (BufferDesc *) arg;
6235
6236 if (bufHdr != NULL)
6237 errcontext("writing block %u of relation \"%s\"",
6238 bufHdr->tag.blockNum,
6239 relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag),
6240 MyProcNumber,
6241 BufTagGetForkNum(&bufHdr->tag)).str);
6242 }
6243
6244 /*
6245 * RelFileLocator qsort/bsearch comparator; see RelFileLocatorEquals.
6246 */
6247 static int
6248 rlocator_comparator(const void *p1, const void *p2)
6249 {
6250 RelFileLocator n1 = *(const RelFileLocator *) p1;
6251 RelFileLocator n2 = *(const RelFileLocator *) p2;
6252
6253 if (n1.relNumber < n2.relNumber)
6254 return -1;
6255 else if (n1.relNumber > n2.relNumber)
6256 return 1;
6257
6258 if (n1.dbOid < n2.dbOid)
6259 return -1;
6260 else if (n1.dbOid > n2.dbOid)
6261 return 1;
6262
6263 if (n1.spcOid < n2.spcOid)
6264 return -1;
6265 else if (n1.spcOid > n2.spcOid)
6266 return 1;
6267 else
6268 return 0;
6269 }
6270
6271 /*
6272 * Lock buffer header - set BM_LOCKED in buffer state.
6273 */
6274 uint32
6275 LockBufHdr(BufferDesc *desc)
6276 {
6277 SpinDelayStatus delayStatus;
6278 uint32 old_buf_state;
6279
6280 Assert(!BufferIsLocal(BufferDescriptorGetBuffer(desc)));
6281
6282 init_local_spin_delay(&delayStatus);
6283
6284 while (true)
6285 {
6286 /* set BM_LOCKED flag */
6287 old_buf_state = pg_atomic_fetch_or_u32(&desc->state, BM_LOCKED);
6288 /* if it wasn't set before we're OK */
6289 if (!(old_buf_state & BM_LOCKED))
6290 break;
6291 perform_spin_delay(&delayStatus);
6292 }
6293 finish_spin_delay(&delayStatus);
6294 return old_buf_state | BM_LOCKED;
6295 }
6296
6297 /*
6298 * Wait until the BM_LOCKED flag isn't set anymore and return the buffer's
6299 * state at that point.
6300 *
6301 * Obviously the buffer could be locked by the time the value is returned, so
6302 * this is primarily useful in CAS style loops.
6303 */
6304 static uint32
6305 WaitBufHdrUnlocked(BufferDesc *buf)
6306 {
6307 SpinDelayStatus delayStatus;
6308 uint32 buf_state;
6309
6310 init_local_spin_delay(&delayStatus);
6311
6312 buf_state = pg_atomic_read_u32(&buf->state);
6313
6314 while (buf_state & BM_LOCKED)
6315 {
6316 perform_spin_delay(&delayStatus);
6317 buf_state = pg_atomic_read_u32(&buf->state);
6318 }
6319
6320 finish_spin_delay(&delayStatus);
6321
6322 return buf_state;
6323 }
6324
6325 /*
6326 * BufferTag comparator.
6327 */
6328 static inline int
6329 buffertag_comparator(const BufferTag *ba, const BufferTag *bb)
6330 {
6331 int ret;
6332 RelFileLocator rlocatora;
6333 RelFileLocator rlocatorb;
6334
6335 rlocatora = BufTagGetRelFileLocator(ba);
6336 rlocatorb = BufTagGetRelFileLocator(bb);
6337
6338 ret = rlocator_comparator(&rlocatora, &rlocatorb);
6339
6340 if (ret != 0)
6341 return ret;
6342
6343 if (BufTagGetForkNum(ba) < BufTagGetForkNum(bb))
6344 return -1;
6345 if (BufTagGetForkNum(ba) > BufTagGetForkNum(bb))
6346 return 1;
6347
6348 if (ba->blockNum < bb->blockNum)
6349 return -1;
6350 if (ba->blockNum > bb->blockNum)
6351 return 1;
6352
6353 return 0;
6354 }
6355
6356 /*
6357 * Comparator determining the writeout order in a checkpoint.
6358 *
6359 * It is important that tablespaces are compared first, the logic balancing
6360 * writes between tablespaces relies on it.
6361 */
6362 static inline int
6363 ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b)
6364 {
6365 /* compare tablespace */
6366 if (a->tsId < b->tsId)
6367 return -1;
6368 else if (a->tsId > b->tsId)
6369 return 1;
6370 /* compare relation */
6371 if (a->relNumber < b->relNumber)
6372 return -1;
6373 else if (a->relNumber > b->relNumber)
6374 return 1;
6375 /* compare fork */
6376 else if (a->forkNum < b->forkNum)
6377 return -1;
6378 else if (a->forkNum > b->forkNum)
6379 return 1;
6380 /* compare block number */
6381 else if (a->blockNum < b->blockNum)
6382 return -1;
6383 else if (a->blockNum > b->blockNum)
6384 return 1;
6385 /* equal page IDs are unlikely, but not impossible */
6386 return 0;
6387 }
6388
6389 /*
6390 * Comparator for a Min-Heap over the per-tablespace checkpoint completion
6391 * progress.
6392 */
6393 static int
6394 ts_ckpt_progress_comparator(Datum a, Datum b, void *arg)
6395 {
6396 CkptTsStatus *sa = (CkptTsStatus *) a;
6397 CkptTsStatus *sb = (CkptTsStatus *) b;
6398
6399 /* we want a min-heap, so return 1 for the a < b */
6400 if (sa->progress < sb->progress)
6401 return 1;
6402 else if (sa->progress == sb->progress)
6403 return 0;
6404 else
6405 return -1;
6406 }
6407
6408 /*
6409 * Initialize a writeback context, discarding potential previous state.
6410 *
6411 * *max_pending is a pointer instead of an immediate value, so the coalesce
6412 * limits can easily changed by the GUC mechanism, and so calling code does
6413 * not have to check the current configuration. A value of 0 means that no
6414 * writeback control will be performed.
6415 */
6416 void
6417 WritebackContextInit(WritebackContext *context, int *max_pending)
6418 {
6419 Assert(*max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
6420
6421 context->max_pending = max_pending;
6422 context->nr_pending = 0;
6423 }
6424
6425 /*
6426 * Add buffer to list of pending writeback requests.
6427 */
6428 void
6429 ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context,
6430 BufferTag *tag)
6431 {
6432 PendingWriteback *pending;
6433
6434 /*
6435 * As pg_flush_data() doesn't do anything with fsync disabled, there's no
6436 * point in tracking in that case.
6437 */
6438 if (io_direct_flags & IO_DIRECT_DATA ||
6439 !enableFsync)
6440 return;
6441
6442 /*
6443 * Add buffer to the pending writeback array, unless writeback control is
6444 * disabled.
6445 */
6446 if (*wb_context->max_pending > 0)
6447 {
6448 Assert(*wb_context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
6449
6450 pending = &wb_context->pending_writebacks[wb_context->nr_pending++];
6451
6452 pending->tag = *tag;
6453 }
6454
6455 /*
6456 * Perform pending flushes if the writeback limit is exceeded. This
6457 * includes the case where previously an item has been added, but control
6458 * is now disabled.
6459 */
6460 if (wb_context->nr_pending >= *wb_context->max_pending)
6461 IssuePendingWritebacks(wb_context, io_context);
6462 }
6463
6464 #define ST_SORT sort_pending_writebacks
6465 #define ST_ELEMENT_TYPE PendingWriteback
6466 #define ST_COMPARE(a, b) buffertag_comparator(&a->tag, &b->tag)
6467 #define ST_SCOPE static
6468 #define ST_DEFINE
6469 #include "lib/sort_template.h"
6470
6471 /*
6472 * Issue all pending writeback requests, previously scheduled with
6473 * ScheduleBufferTagForWriteback, to the OS.
6474 *
6475 * Because this is only used to improve the OSs IO scheduling we try to never
6476 * error out - it's just a hint.
6477 */
6478 void
6479 IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context)
6480 {
6481 instr_time io_start;
6482 int i;
6483
6484 if (wb_context->nr_pending == 0)
6485 return;
6486
6487 /*
6488 * Executing the writes in-order can make them a lot faster, and allows to
6489 * merge writeback requests to consecutive blocks into larger writebacks.
6490 */
6491 sort_pending_writebacks(wb_context->pending_writebacks,
6492 wb_context->nr_pending);
6493
6494 io_start = pgstat_prepare_io_time(track_io_timing);
6495
6496 /*
6497 * Coalesce neighbouring writes, but nothing else. For that we iterate
6498 * through the, now sorted, array of pending flushes, and look forward to
6499 * find all neighbouring (or identical) writes.
6500 */
6501 for (i = 0; i < wb_context->nr_pending; i++)
6502 {
6503 PendingWriteback *cur;
6504 PendingWriteback *next;
6505 SMgrRelation reln;
6506 int ahead;
6507 BufferTag tag;
6508 RelFileLocator currlocator;
6509 Size nblocks = 1;
6510
6511 cur = &wb_context->pending_writebacks[i];
6512 tag = cur->tag;
6513 currlocator = BufTagGetRelFileLocator(&tag);
6514
6515 /*
6516 * Peek ahead, into following writeback requests, to see if they can
6517 * be combined with the current one.
6518 */
6519 for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++)
6520 {
6521
6522 next = &wb_context->pending_writebacks[i + ahead + 1];
6523
6524 /* different file, stop */
6525 if (!RelFileLocatorEquals(currlocator,
6526 BufTagGetRelFileLocator(&next->tag)) ||
6527 BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag))
6528 break;
6529
6530 /* ok, block queued twice, skip */
6531 if (cur->tag.blockNum == next->tag.blockNum)
6532 continue;
6533
6534 /* only merge consecutive writes */
6535 if (cur->tag.blockNum + 1 != next->tag.blockNum)
6536 break;
6537
6538 nblocks++;
6539 cur = next;
6540 }
6541
6542 i += ahead;
6543
6544 /* and finally tell the kernel to write the data to storage */
6545 reln = smgropen(currlocator, INVALID_PROC_NUMBER);
6546 smgrwriteback(reln, BufTagGetForkNum(&tag), tag.blockNum, nblocks);
6547 }
6548
6549 /*
6550 * Assume that writeback requests are only issued for buffers containing
6551 * blocks of permanent relations.
6552 */
6553 pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
6554 IOOP_WRITEBACK, io_start, wb_context->nr_pending, 0);
6555
6556 wb_context->nr_pending = 0;
6557 }
6558
6559 /* ResourceOwner callbacks */
6560
6561 static void
6562 ResOwnerReleaseBufferIO(Datum res)
6563 {
6564 Buffer buffer = DatumGetInt32(res);
6565
6566 AbortBufferIO(buffer);
6567 }
6568
6569 static char *
6570 ResOwnerPrintBufferIO(Datum res)
6571 {
6572 Buffer buffer = DatumGetInt32(res);
6573
6574 return psprintf("lost track of buffer IO on buffer %d", buffer);
6575 }
6576
6577 static void
6578 ResOwnerReleaseBufferPin(Datum res)
6579 {
6580 Buffer buffer = DatumGetInt32(res);
6581
6582 /* Like ReleaseBuffer, but don't call ResourceOwnerForgetBuffer */
6583 if (!BufferIsValid(buffer))
6584 elog(ERROR, "bad buffer ID: %d", buffer);
6585
6586 if (BufferIsLocal(buffer))
6587 UnpinLocalBufferNoOwner(buffer);
6588 else
6589 UnpinBufferNoOwner(GetBufferDescriptor(buffer - 1));
6590 }
6591
6592 static char *
6593 ResOwnerPrintBufferPin(Datum res)
6594 {
6595 return DebugPrintBufferRefcount(DatumGetInt32(res));
6596 }
6597
6598 /*
6599 * Helper function to evict unpinned buffer whose buffer header lock is
6600 * already acquired.
6601 */
6602 static bool
6603 EvictUnpinnedBufferInternal(BufferDesc *desc, bool *buffer_flushed)
6604 {
6605 uint32 buf_state;
6606 bool result;
6607
6608 *buffer_flushed = false;
6609
6610 buf_state = pg_atomic_read_u32(&(desc->state));
6611 Assert(buf_state & BM_LOCKED);
6612
6613 if ((buf_state & BM_VALID) == 0)
6614 {
6615 UnlockBufHdr(desc, buf_state);
6616 return false;
6617 }
6618
6619 /* Check that it's not pinned already. */
6620 if (BUF_STATE_GET_REFCOUNT(buf_state) > 0)
6621 {
6622 UnlockBufHdr(desc, buf_state);
6623 return false;
6624 }
6625
6626 PinBuffer_Locked(desc); /* releases spinlock */
6627
6628 /* If it was dirty, try to clean it once. */
6629 if (buf_state & BM_DIRTY)
6630 {
6631 LWLockAcquire(BufferDescriptorGetContentLock(desc), LW_SHARED);
6632 FlushBuffer(desc, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
6633 *buffer_flushed = true;
6634 LWLockRelease(BufferDescriptorGetContentLock(desc));
6635 }
6636
6637 /* This will return false if it becomes dirty or someone else pins it. */
6638 result = InvalidateVictimBuffer(desc);
6639
6640 UnpinBuffer(desc);
6641
6642 return result;
6643 }
6644
6645 /*
6646 * Try to evict the current block in a shared buffer.
6647 *
6648 * This function is intended for testing/development use only!
6649 *
6650 * To succeed, the buffer must not be pinned on entry, so if the caller had a
6651 * particular block in mind, it might already have been replaced by some other
6652 * block by the time this function runs. It's also unpinned on return, so the
6653 * buffer might be occupied again by the time control is returned, potentially
6654 * even by the same block. This inherent raciness without other interlocking
6655 * makes the function unsuitable for non-testing usage.
6656 *
6657 * *buffer_flushed is set to true if the buffer was dirty and has been
6658 * flushed, false otherwise. However, *buffer_flushed=true does not
6659 * necessarily mean that we flushed the buffer, it could have been flushed by
6660 * someone else.
6661 *
6662 * Returns true if the buffer was valid and it has now been made invalid.
6663 * Returns false if it wasn't valid, if it couldn't be evicted due to a pin,
6664 * or if the buffer becomes dirty again while we're trying to write it out.
6665 */
6666 bool
6667 EvictUnpinnedBuffer(Buffer buf, bool *buffer_flushed)
6668 {
6669 BufferDesc *desc;
6670
6671 Assert(BufferIsValid(buf) && !BufferIsLocal(buf));
6672
6673 /* Make sure we can pin the buffer. */
6674 ResourceOwnerEnlarge(CurrentResourceOwner);
6675 ReservePrivateRefCountEntry();
6676
6677 desc = GetBufferDescriptor(buf - 1);
6678 LockBufHdr(desc);
6679
6680 return EvictUnpinnedBufferInternal(desc, buffer_flushed);
6681 }
6682
6683 /*
6684 * Try to evict all the shared buffers.
6685 *
6686 * This function is intended for testing/development use only! See
6687 * EvictUnpinnedBuffer().
6688 *
6689 * The buffers_* parameters are mandatory and indicate the total count of
6690 * buffers that:
6691 * - buffers_evicted - were evicted
6692 * - buffers_flushed - were flushed
6693 * - buffers_skipped - could not be evicted
6694 */
6695 void
6696 EvictAllUnpinnedBuffers(int32 *buffers_evicted, int32 *buffers_flushed,
6697 int32 *buffers_skipped)
6698 {
6699 *buffers_evicted = 0;
6700 *buffers_skipped = 0;
6701 *buffers_flushed = 0;
6702
6703 for (int buf = 1; buf <= NBuffers; buf++)
6704 {
6705 BufferDesc *desc = GetBufferDescriptor(buf - 1);
6706 uint32 buf_state;
6707 bool buffer_flushed;
6708
6709 CHECK_FOR_INTERRUPTS();
6710
6711 buf_state = pg_atomic_read_u32(&desc->state);
6712 if (!(buf_state & BM_VALID))
6713 continue;
6714
6715 ResourceOwnerEnlarge(CurrentResourceOwner);
6716 ReservePrivateRefCountEntry();
6717
6718 LockBufHdr(desc);
6719
6720 if (EvictUnpinnedBufferInternal(desc, &buffer_flushed))
6721 (*buffers_evicted)++;
6722 else
6723 (*buffers_skipped)++;
6724
6725 if (buffer_flushed)
6726 (*buffers_flushed)++;
6727 }
6728 }
6729
6730 /*
6731 * Try to evict all the shared buffers containing provided relation's pages.
6732 *
6733 * This function is intended for testing/development use only! See
6734 * EvictUnpinnedBuffer().
6735 *
6736 * The caller must hold at least AccessShareLock on the relation to prevent
6737 * the relation from being dropped.
6738 *
6739 * The buffers_* parameters are mandatory and indicate the total count of
6740 * buffers that:
6741 * - buffers_evicted - were evicted
6742 * - buffers_flushed - were flushed
6743 * - buffers_skipped - could not be evicted
6744 */
6745 void
6746 EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted,
6747 int32 *buffers_flushed, int32 *buffers_skipped)
6748 {
6749 Assert(!RelationUsesLocalBuffers(rel));
6750
6751 *buffers_skipped = 0;
6752 *buffers_evicted = 0;
6753 *buffers_flushed = 0;
6754
6755 for (int buf = 1; buf <= NBuffers; buf++)
6756 {
6757 BufferDesc *desc = GetBufferDescriptor(buf - 1);
6758 uint32 buf_state = pg_atomic_read_u32(&(desc->state));
6759 bool buffer_flushed;
6760
6761 CHECK_FOR_INTERRUPTS();
6762
6763 /* An unlocked precheck should be safe and saves some cycles. */
6764 if ((buf_state & BM_VALID) == 0 ||
6765 !BufTagMatchesRelFileLocator(&desc->tag, &rel->rd_locator))
6766 continue;
6767
6768 /* Make sure we can pin the buffer. */
6769 ResourceOwnerEnlarge(CurrentResourceOwner);
6770 ReservePrivateRefCountEntry();
6771
6772 buf_state = LockBufHdr(desc);
6773
6774 /* recheck, could have changed without the lock */
6775 if ((buf_state & BM_VALID) == 0 ||
6776 !BufTagMatchesRelFileLocator(&desc->tag, &rel->rd_locator))
6777 {
6778 UnlockBufHdr(desc, buf_state);
6779 continue;
6780 }
6781
6782 if (EvictUnpinnedBufferInternal(desc, &buffer_flushed))
6783 (*buffers_evicted)++;
6784 else
6785 (*buffers_skipped)++;
6786
6787 if (buffer_flushed)
6788 (*buffers_flushed)++;
6789 }
6790 }
6791
6792 /*
6793 * Generic implementation of the AIO handle staging callback for readv/writev
6794 * on local/shared buffers.
6795 *
6796 * Each readv/writev can target multiple buffers. The buffers have already
6797 * been registered with the IO handle.
6798 *
6799 * To make the IO ready for execution ("staging"), we need to ensure that the
6800 * targeted buffers are in an appropriate state while the IO is ongoing. For
6801 * that the AIO subsystem needs to have its own buffer pin, otherwise an error
6802 * in this backend could lead to this backend's buffer pin being released as
6803 * part of error handling, which in turn could lead to the buffer being
6804 * replaced while IO is ongoing.
6805 */
6806 static pg_always_inline void
6807 buffer_stage_common(PgAioHandle *ioh, bool is_write, bool is_temp)
6808 {
6809 uint64 *io_data;
6810 uint8 handle_data_len;
6811 PgAioWaitRef io_ref;
6812 BufferTag first PG_USED_FOR_ASSERTS_ONLY = {0};
6813
6814 io_data = pgaio_io_get_handle_data(ioh, &handle_data_len);
6815
6816 pgaio_io_get_wref(ioh, &io_ref);
6817
6818 /* iterate over all buffers affected by the vectored readv/writev */
6819 for (int i = 0; i < handle_data_len; i++)
6820 {
6821 Buffer buffer = (Buffer) io_data[i];
6822 BufferDesc *buf_hdr = is_temp ?
6823 GetLocalBufferDescriptor(-buffer - 1)
6824 : GetBufferDescriptor(buffer - 1);
6825 uint32 buf_state;
6826
6827 /*
6828 * Check that all the buffers are actually ones that could conceivably
6829 * be done in one IO, i.e. are sequential. This is the last
6830 * buffer-aware code before IO is actually executed and confusion
6831 * about which buffers are targeted by IO can be hard to debug, making
6832 * it worth doing extra-paranoid checks.
6833 */
6834 if (i == 0)
6835 first = buf_hdr->tag;
6836 else
6837 {
6838 Assert(buf_hdr->tag.relNumber == first.relNumber);
6839 Assert(buf_hdr->tag.blockNum == first.blockNum + i);
6840 }
6841
6842 if (is_temp)
6843 buf_state = pg_atomic_read_u32(&buf_hdr->state);
6844 else
6845 buf_state = LockBufHdr(buf_hdr);
6846
6847 /* verify the buffer is in the expected state */
6848 Assert(buf_state & BM_TAG_VALID);
6849 if (is_write)
6850 {
6851 Assert(buf_state & BM_VALID);
6852 Assert(buf_state & BM_DIRTY);
6853 }
6854 else
6855 {
6856 Assert(!(buf_state & BM_VALID));
6857 Assert(!(buf_state & BM_DIRTY));
6858 }
6859
6860 /* temp buffers don't use BM_IO_IN_PROGRESS */
6861 if (!is_temp)
6862 Assert(buf_state & BM_IO_IN_PROGRESS);
6863
6864 Assert(BUF_STATE_GET_REFCOUNT(buf_state) >= 1);
6865
6866 /*
6867 * Reflect that the buffer is now owned by the AIO subsystem.
6868 *
6869 * For local buffers: This can't be done just via LocalRefCount, as
6870 * one might initially think, as this backend could error out while
6871 * AIO is still in progress, releasing all the pins by the backend
6872 * itself.
6873 *
6874 * This pin is released again in TerminateBufferIO().
6875 */
6876 buf_state += BUF_REFCOUNT_ONE;
6877 buf_hdr->io_wref = io_ref;
6878
6879 if (is_temp)
6880 pg_atomic_unlocked_write_u32(&buf_hdr->state, buf_state);
6881 else
6882 UnlockBufHdr(buf_hdr, buf_state);
6883
6884 /*
6885 * Ensure the content lock that prevents buffer modifications while
6886 * the buffer is being written out is not released early due to an
6887 * error.
6888 */
6889 if (is_write && !is_temp)
6890 {
6891 LWLock *content_lock;
6892
6893 content_lock = BufferDescriptorGetContentLock(buf_hdr);
6894
6895 Assert(LWLockHeldByMe(content_lock));
6896
6897 /*
6898 * Lock is now owned by AIO subsystem.
6899 */
6900 LWLockDisown(content_lock);
6901 }
6902
6903 /*
6904 * Stop tracking this buffer via the resowner - the AIO system now
6905 * keeps track.
6906 */
6907 if (!is_temp)
6908 ResourceOwnerForgetBufferIO(CurrentResourceOwner, buffer);
6909 }
6910 }
6911
6912 /*
6913 * Decode readv errors as encoded by buffer_readv_encode_error().
6914 */
6915 static inline void
6916 buffer_readv_decode_error(PgAioResult result,
6917 bool *zeroed_any,
6918 bool *ignored_any,
6919 uint8 *zeroed_or_error_count,
6920 uint8 *checkfail_count,
6921 uint8 *first_off)
6922 {
6923 uint32 rem_error = result.error_data;
6924
6925 /* see static asserts in buffer_readv_encode_error */
6926 #define READV_COUNT_BITS 7
6927 #define READV_COUNT_MASK ((1 << READV_COUNT_BITS) - 1)
6928
6929 *zeroed_any = rem_error & 1;
6930 rem_error >>= 1;
6931
6932 *ignored_any = rem_error & 1;
6933 rem_error >>= 1;
6934
6935 *zeroed_or_error_count = rem_error & READV_COUNT_MASK;
6936 rem_error >>= READV_COUNT_BITS;
6937
6938 *checkfail_count = rem_error & READV_COUNT_MASK;
6939 rem_error >>= READV_COUNT_BITS;
6940
6941 *first_off = rem_error & READV_COUNT_MASK;
6942 rem_error >>= READV_COUNT_BITS;
6943 }
6944
6945 /*
6946 * Helper to encode errors for buffer_readv_complete()
6947 *
6948 * Errors are encoded as follows:
6949 * - bit 0 indicates whether any page was zeroed (1) or not (0)
6950 * - bit 1 indicates whether any checksum failure was ignored (1) or not (0)
6951 * - next READV_COUNT_BITS bits indicate the number of errored or zeroed pages
6952 * - next READV_COUNT_BITS bits indicate the number of checksum failures
6953 * - next READV_COUNT_BITS bits indicate the first offset of the first page
6954 * that was errored or zeroed or, if no errors/zeroes, the first ignored
6955 * checksum
6956 */
6957 static inline void
6958 buffer_readv_encode_error(PgAioResult *result,
6959 bool is_temp,
6960 bool zeroed_any,
6961 bool ignored_any,
6962 uint8 error_count,
6963 uint8 zeroed_count,
6964 uint8 checkfail_count,
6965 uint8 first_error_off,
6966 uint8 first_zeroed_off,
6967 uint8 first_ignored_off)
6968 {
6969
6970 uint8 shift = 0;
6971 uint8 zeroed_or_error_count =
6972 error_count > 0 ? error_count : zeroed_count;
6973 uint8 first_off;
6974
6975 StaticAssertStmt(PG_IOV_MAX <= 1 << READV_COUNT_BITS,
6976 "PG_IOV_MAX is bigger than reserved space for error data");
6977 StaticAssertStmt((1 + 1 + 3 * READV_COUNT_BITS) <= PGAIO_RESULT_ERROR_BITS,
6978 "PGAIO_RESULT_ERROR_BITS is insufficient for buffer_readv");
6979
6980 /*
6981 * We only have space to encode one offset - but luckily that's good
6982 * enough. If there is an error, the error is the interesting offset, same
6983 * with a zeroed buffer vs an ignored buffer.
6984 */
6985 if (error_count > 0)
6986 first_off = first_error_off;
6987 else if (zeroed_count > 0)
6988 first_off = first_zeroed_off;
6989 else
6990 first_off = first_ignored_off;
6991
6992 Assert(!zeroed_any || error_count == 0);
6993
6994 result->error_data = 0;
6995
6996 result->error_data |= zeroed_any << shift;
6997 shift += 1;
6998
6999 result->error_data |= ignored_any << shift;
7000 shift += 1;
7001
7002 result->error_data |= ((uint32) zeroed_or_error_count) << shift;
7003 shift += READV_COUNT_BITS;
7004
7005 result->error_data |= ((uint32) checkfail_count) << shift;
7006 shift += READV_COUNT_BITS;
7007
7008 result->error_data |= ((uint32) first_off) << shift;
7009 shift += READV_COUNT_BITS;
7010
7011 result->id = is_temp ? PGAIO_HCB_LOCAL_BUFFER_READV :
7012 PGAIO_HCB_SHARED_BUFFER_READV;
7013
7014 if (error_count > 0)
7015 result->status = PGAIO_RS_ERROR;
7016 else
7017 result->status = PGAIO_RS_WARNING;
7018
7019 /*
7020 * The encoding is complicated enough to warrant cross-checking it against
7021 * the decode function.
7022 */
7023 #ifdef USE_ASSERT_CHECKING
7024 {
7025 bool zeroed_any_2,
7026 ignored_any_2;
7027 uint8 zeroed_or_error_count_2,
7028 checkfail_count_2,
7029 first_off_2;
7030
7031 buffer_readv_decode_error(*result,
7032 &zeroed_any_2, &ignored_any_2,
7033 &zeroed_or_error_count_2,
7034 &checkfail_count_2,
7035 &first_off_2);
7036 Assert(zeroed_any == zeroed_any_2);
7037 Assert(ignored_any == ignored_any_2);
7038 Assert(zeroed_or_error_count == zeroed_or_error_count_2);
7039 Assert(checkfail_count == checkfail_count_2);
7040 Assert(first_off == first_off_2);
7041 }
7042 #endif
7043
7044 #undef READV_COUNT_BITS
7045 #undef READV_COUNT_MASK
7046 }
7047
7048 /*
7049 * Helper for AIO readv completion callbacks, supporting both shared and temp
7050 * buffers. Gets called once for each buffer in a multi-page read.
7051 */
7052 static pg_always_inline void
7053 buffer_readv_complete_one(PgAioTargetData *td, uint8 buf_off, Buffer buffer,
7054 uint8 flags, bool failed, bool is_temp,
7055 bool *buffer_invalid,
7056 bool *failed_checksum,
7057 bool *ignored_checksum,
7058 bool *zeroed_buffer)
7059 {
7060 BufferDesc *buf_hdr = is_temp ?
7061 GetLocalBufferDescriptor(-buffer - 1)
7062 : GetBufferDescriptor(buffer - 1);
7063 BufferTag tag = buf_hdr->tag;
7064 char *bufdata = BufferGetBlock(buffer);
7065 uint32 set_flag_bits;
7066 int piv_flags;
7067
7068 /* check that the buffer is in the expected state for a read */
7069 #ifdef USE_ASSERT_CHECKING
7070 {
7071 uint32 buf_state = pg_atomic_read_u32(&buf_hdr->state);
7072
7073 Assert(buf_state & BM_TAG_VALID);
7074 Assert(!(buf_state & BM_VALID));
7075 /* temp buffers don't use BM_IO_IN_PROGRESS */
7076 if (!is_temp)
7077 Assert(buf_state & BM_IO_IN_PROGRESS);
7078 Assert(!(buf_state & BM_DIRTY));
7079 }
7080 #endif
7081
7082 *buffer_invalid = false;
7083 *failed_checksum = false;
7084 *ignored_checksum = false;
7085 *zeroed_buffer = false;
7086
7087 /*
7088 * We ask PageIsVerified() to only log the message about checksum errors,
7089 * as the completion might be run in any backend (or IO workers). We will
7090 * report checksum errors in buffer_readv_report().
7091 */
7092 piv_flags = PIV_LOG_LOG;
7093
7094 /* the local zero_damaged_pages may differ from the definer's */
7095 if (flags & READ_BUFFERS_IGNORE_CHECKSUM_FAILURES)
7096 piv_flags |= PIV_IGNORE_CHECKSUM_FAILURE;
7097
7098 /* Check for garbage data. */
7099 if (!failed)
7100 {
7101 /*
7102 * If the buffer is not currently pinned by this backend, e.g. because
7103 * we're completing this IO after an error, the buffer data will have
7104 * been marked as inaccessible when the buffer was unpinned. The AIO
7105 * subsystem holds a pin, but that doesn't prevent the buffer from
7106 * having been marked as inaccessible. The completion might also be
7107 * executed in a different process.
7108 */
7109 #ifdef USE_VALGRIND
7110 if (!BufferIsPinned(buffer))
7111 VALGRIND_MAKE_MEM_DEFINED(bufdata, BLCKSZ);
7112 #endif
7113
7114 if (!PageIsVerified((Page) bufdata, tag.blockNum, piv_flags,
7115 failed_checksum))
7116 {
7117 if (flags & READ_BUFFERS_ZERO_ON_ERROR)
7118 {
7119 memset(bufdata, 0, BLCKSZ);
7120 *zeroed_buffer = true;
7121 }
7122 else
7123 {
7124 *buffer_invalid = true;
7125 /* mark buffer as having failed */
7126 failed = true;
7127 }
7128 }
7129 else if (*failed_checksum)
7130 *ignored_checksum = true;
7131
7132 /* undo what we did above */
7133 #ifdef USE_VALGRIND
7134 if (!BufferIsPinned(buffer))
7135 VALGRIND_MAKE_MEM_NOACCESS(bufdata, BLCKSZ);
7136 #endif
7137
7138 /*
7139 * Immediately log a message about the invalid page, but only to the
7140 * server log. The reason to do so immediately is that this may be
7141 * executed in a different backend than the one that originated the
7142 * request. The reason to do so immediately is that the originator
7143 * might not process the query result immediately (because it is busy
7144 * doing another part of query processing) or at all (e.g. if it was
7145 * cancelled or errored out due to another IO also failing). The
7146 * definer of the IO will emit an ERROR or WARNING when processing the
7147 * IO's results
7148 *
7149 * To avoid duplicating the code to emit these log messages, we reuse
7150 * buffer_readv_report().
7151 */
7152 if (*buffer_invalid || *failed_checksum || *zeroed_buffer)
7153 {
7154 PgAioResult result_one = {0};
7155
7156 buffer_readv_encode_error(&result_one, is_temp,
7157 *zeroed_buffer,
7158 *ignored_checksum,
7159 *buffer_invalid,
7160 *zeroed_buffer ? 1 : 0,
7161 *failed_checksum ? 1 : 0,
7162 buf_off, buf_off, buf_off);
7163 pgaio_result_report(result_one, td, LOG_SERVER_ONLY);
7164 }
7165 }
7166
7167 /* Terminate I/O and set BM_VALID. */
7168 set_flag_bits = failed ? BM_IO_ERROR : BM_VALID;
7169 if (is_temp)
7170 TerminateLocalBufferIO(buf_hdr, false, set_flag_bits, true);
7171 else
7172 TerminateBufferIO(buf_hdr, false, set_flag_bits, false, true);
7173
7174 /*
7175 * Call the BUFFER_READ_DONE tracepoint in the callback, even though the
7176 * callback may not be executed in the same backend that called
7177 * BUFFER_READ_START. The alternative would be to defer calling the
7178 * tracepoint to a later point (e.g. the local completion callback for
7179 * shared buffer reads), which seems even less helpful.
7180 */
7181 TRACE_POSTGRESQL_BUFFER_READ_DONE(tag.forkNum,
7182 tag.blockNum,
7183 tag.spcOid,
7184 tag.dbOid,
7185 tag.relNumber,
7186 is_temp ? MyProcNumber : INVALID_PROC_NUMBER,
7187 false);
7188 }
7189
7190 /*
7191 * Perform completion handling of a single AIO read. This read may cover
7192 * multiple blocks / buffers.
7193 *
7194 * Shared between shared and local buffers, to reduce code duplication.
7195 */
7196 static pg_always_inline PgAioResult
7197 buffer_readv_complete(PgAioHandle *ioh, PgAioResult prior_result,
7198 uint8 cb_data, bool is_temp)
7199 {
7200 PgAioResult result = prior_result;
7201 PgAioTargetData *td = pgaio_io_get_target_data(ioh);
7202 uint8 first_error_off = 0;
7203 uint8 first_zeroed_off = 0;
7204 uint8 first_ignored_off = 0;
7205 uint8 error_count = 0;
7206 uint8 zeroed_count = 0;
7207 uint8 ignored_count = 0;
7208 uint8 checkfail_count = 0;
7209 uint64 *io_data;
7210 uint8 handle_data_len;
7211
7212 if (is_temp)
7213 {
7214 Assert(td->smgr.is_temp);
7215 Assert(pgaio_io_get_owner(ioh) == MyProcNumber);
7216 }
7217 else
7218 Assert(!td->smgr.is_temp);
7219
7220 /*
7221 * Iterate over all the buffers affected by this IO and call the
7222 * per-buffer completion function for each buffer.
7223 */
7224 io_data = pgaio_io_get_handle_data(ioh, &handle_data_len);
7225 for (uint8 buf_off = 0; buf_off < handle_data_len; buf_off++)
7226 {
7227 Buffer buf = io_data[buf_off];
7228 bool failed;
7229 bool failed_verification = false;
7230 bool failed_checksum = false;
7231 bool zeroed_buffer = false;
7232 bool ignored_checksum = false;
7233
7234 Assert(BufferIsValid(buf));
7235
7236 /*
7237 * If the entire I/O failed on a lower-level, each buffer needs to be
7238 * marked as failed. In case of a partial read, the first few buffers
7239 * may be ok.
7240 */
7241 failed =
7242 prior_result.status == PGAIO_RS_ERROR
7243 || prior_result.result <= buf_off;
7244
7245 buffer_readv_complete_one(td, buf_off, buf, cb_data, failed, is_temp,
7246 &failed_verification,
7247 &failed_checksum,
7248 &ignored_checksum,
7249 &zeroed_buffer);
7250
7251 /*
7252 * Track information about the number of different kinds of error
7253 * conditions across all pages, as there can be multiple pages failing
7254 * verification as part of one IO.
7255 */
7256 if (failed_verification && !zeroed_buffer && error_count++ == 0)
7257 first_error_off = buf_off;
7258 if (zeroed_buffer && zeroed_count++ == 0)
7259 first_zeroed_off = buf_off;
7260 if (ignored_checksum && ignored_count++ == 0)
7261 first_ignored_off = buf_off;
7262 if (failed_checksum)
7263 checkfail_count++;
7264 }
7265
7266 /*
7267 * If the smgr read succeeded [partially] and page verification failed for
7268 * some of the pages, adjust the IO's result state appropriately.
7269 */
7270 if (prior_result.status != PGAIO_RS_ERROR &&
7271 (error_count > 0 || ignored_count > 0 || zeroed_count > 0))
7272 {
7273 buffer_readv_encode_error(&result, is_temp,
7274 zeroed_count > 0, ignored_count > 0,
7275 error_count, zeroed_count, checkfail_count,
7276 first_error_off, first_zeroed_off,
7277 first_ignored_off);
7278 pgaio_result_report(result, td, DEBUG1);
7279 }
7280
7281 /*
7282 * For shared relations this reporting is done in
7283 * shared_buffer_readv_complete_local().
7284 */
7285 if (is_temp && checkfail_count > 0)
7286 pgstat_report_checksum_failures_in_db(td->smgr.rlocator.dbOid,
7287 checkfail_count);
7288
7289 return result;
7290 }
7291
7292 /*
7293 * AIO error reporting callback for aio_shared_buffer_readv_cb and
7294 * aio_local_buffer_readv_cb.
7295 *
7296 * The error is encoded / decoded in buffer_readv_encode_error() /
7297 * buffer_readv_decode_error().
7298 */
7299 static void
7300 buffer_readv_report(PgAioResult result, const PgAioTargetData *td,
7301 int elevel)
7302 {
7303 int nblocks = td->smgr.nblocks;
7304 BlockNumber first = td->smgr.blockNum;
7305 BlockNumber last = first + nblocks - 1;
7306 ProcNumber errProc =
7307 td->smgr.is_temp ? MyProcNumber : INVALID_PROC_NUMBER;
7308 RelPathStr rpath =
7309 relpathbackend(td->smgr.rlocator, errProc, td->smgr.forkNum);
7310 bool zeroed_any,
7311 ignored_any;
7312 uint8 zeroed_or_error_count,
7313 checkfail_count,
7314 first_off;
7315 uint8 affected_count;
7316 const char *msg_one,
7317 *msg_mult,
7318 *det_mult,
7319 *hint_mult;
7320
7321 buffer_readv_decode_error(result, &zeroed_any, &ignored_any,
7322 &zeroed_or_error_count,
7323 &checkfail_count,
7324 &first_off);
7325
7326 /*
7327 * Treat a read that had both zeroed buffers *and* ignored checksums as a
7328 * special case, it's too irregular to be emitted the same way as the
7329 * other cases.
7330 */
7331 if (zeroed_any && ignored_any)
7332 {
7333 Assert(zeroed_any && ignored_any);
7334 Assert(nblocks > 1); /* same block can't be both zeroed and ignored */
7335 Assert(result.status != PGAIO_RS_ERROR);
7336 affected_count = zeroed_or_error_count;
7337
7338 ereport(elevel,
7339 errcode(ERRCODE_DATA_CORRUPTED),
7340 errmsg("zeroing %u page(s) and ignoring %u checksum failure(s) among blocks %u..%u of relation \"%s\"",
7341 affected_count, checkfail_count, first, last, rpath.str),
7342 affected_count > 1 ?
7343 errdetail("Block %u held the first zeroed page.",
7344 first + first_off) : 0,
7345 errhint_plural("See server log for details about the other %d invalid block.",
7346 "See server log for details about the other %d invalid blocks.",
7347 affected_count + checkfail_count - 1,
7348 affected_count + checkfail_count - 1));
7349 return;
7350 }
7351
7352 /*
7353 * The other messages are highly repetitive. To avoid duplicating a long
7354 * and complicated ereport(), gather the translated format strings
7355 * separately and then do one common ereport.
7356 */
7357 if (result.status == PGAIO_RS_ERROR)
7358 {
7359 Assert(!zeroed_any); /* can't have invalid pages when zeroing them */
7360 affected_count = zeroed_or_error_count;
7361 msg_one = _("invalid page in block %u of relation \"%s\"");
7362 msg_mult = _("%u invalid pages among blocks %u..%u of relation \"%s\"");
7363 det_mult = _("Block %u held the first invalid page.");
7364 hint_mult = _("See server log for the other %u invalid block(s).");
7365 }
7366 else if (zeroed_any && !ignored_any)
7367 {
7368 affected_count = zeroed_or_error_count;
7369 msg_one = _("invalid page in block %u of relation \"%s\"; zeroing out page");
7370 msg_mult = _("zeroing out %u invalid pages among blocks %u..%u of relation \"%s\"");
7371 det_mult = _("Block %u held the first zeroed page.");
7372 hint_mult = _("See server log for the other %u zeroed block(s).");
7373 }
7374 else if (!zeroed_any && ignored_any)
7375 {
7376 affected_count = checkfail_count;
7377 msg_one = _("ignoring checksum failure in block %u of relation \"%s\"");
7378 msg_mult = _("ignoring %u checksum failures among blocks %u..%u of relation \"%s\"");
7379 det_mult = _("Block %u held the first ignored page.");
7380 hint_mult = _("See server log for the other %u ignored block(s).");
7381 }
7382 else
7383 pg_unreachable();
7384
7385 ereport(elevel,
7386 errcode(ERRCODE_DATA_CORRUPTED),
7387 affected_count == 1 ?
7388 errmsg_internal(msg_one, first + first_off, rpath.str) :
7389 errmsg_internal(msg_mult, affected_count, first, last, rpath.str),
7390 affected_count > 1 ? errdetail_internal(det_mult, first + first_off) : 0,
7391 affected_count > 1 ? errhint_internal(hint_mult, affected_count - 1) : 0);
7392 }
7393
7394 static void
7395 shared_buffer_readv_stage(PgAioHandle *ioh, uint8 cb_data)
7396 {
7397 buffer_stage_common(ioh, false, false);
7398 }
7399
7400 static PgAioResult
7401 shared_buffer_readv_complete(PgAioHandle *ioh, PgAioResult prior_result,
7402 uint8 cb_data)
7403 {
7404 return buffer_readv_complete(ioh, prior_result, cb_data, false);
7405 }
7406
7407 /*
7408 * We need a backend-local completion callback for shared buffers, to be able
7409 * to report checksum errors correctly. Unfortunately that can only safely
7410 * happen if the reporting backend has previously called
7411 * pgstat_prepare_report_checksum_failure(), which we can only guarantee in
7412 * the backend that started the IO. Hence this callback.
7413 */
7414 static PgAioResult
7415 shared_buffer_readv_complete_local(PgAioHandle *ioh, PgAioResult prior_result,
7416 uint8 cb_data)
7417 {
7418 bool zeroed_any,
7419 ignored_any;
7420 uint8 zeroed_or_error_count,
7421 checkfail_count,
7422 first_off;
7423
7424 if (prior_result.status == PGAIO_RS_OK)
7425 return prior_result;
7426
7427 buffer_readv_decode_error(prior_result,
7428 &zeroed_any,
7429 &ignored_any,
7430 &zeroed_or_error_count,
7431 &checkfail_count,
7432 &first_off);
7433
7434 if (checkfail_count)
7435 {
7436 PgAioTargetData *td = pgaio_io_get_target_data(ioh);
7437
7438 pgstat_report_checksum_failures_in_db(td->smgr.rlocator.dbOid,
7439 checkfail_count);
7440 }
7441
7442 return prior_result;
7443 }
7444
7445 static void
7446 local_buffer_readv_stage(PgAioHandle *ioh, uint8 cb_data)
7447 {
7448 buffer_stage_common(ioh, false, true);
7449 }
7450
7451 static PgAioResult
7452 local_buffer_readv_complete(PgAioHandle *ioh, PgAioResult prior_result,
7453 uint8 cb_data)
7454 {
7455 return buffer_readv_complete(ioh, prior_result, cb_data, true);
7456 }
7457
7458 /* readv callback is passed READ_BUFFERS_* flags as callback data */
7459 const PgAioHandleCallbacks aio_shared_buffer_readv_cb = {
7460 .stage = shared_buffer_readv_stage,
7461 .complete_shared = shared_buffer_readv_complete,
7462 /* need a local callback to report checksum failures */
7463 .complete_local = shared_buffer_readv_complete_local,
7464 .report = buffer_readv_report,
7465 };
7466
7467 /* readv callback is passed READ_BUFFERS_* flags as callback data */
7468 const PgAioHandleCallbacks aio_local_buffer_readv_cb = {
7469 .stage = local_buffer_readv_stage,
7470
7471 /*
7472 * Note that this, in contrast to the shared_buffers case, uses
7473 * complete_local, as only the issuing backend has access to the required
7474 * datastructures. This is important in case the IO completion may be
7475 * consumed incidentally by another backend.
7476 */
7477 .complete_local = local_buffer_readv_complete,
7478 .report = buffer_readv_report,
7479 };