Debian Codegraph @

src/backend/storage/buffer/localbuf.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * localbuf.c
4 * local buffer manager. Fast buffer manager for temporary tables,
5 * which never need to be WAL-logged or checkpointed, etc.
6 *
7 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
8 * Portions Copyright (c) 1994-5, Regents of the University of California
9 *
10 *
11 * IDENTIFICATION
12 * src/backend/storage/buffer/localbuf.c
13 *
14 *-------------------------------------------------------------------------
15 */
16 #include "postgres.h"
17
18 #include "access/parallel.h"
19 #include "executor/instrument.h"
20 #include "pgstat.h"
21 #include "storage/aio.h"
22 #include "storage/buf_internals.h"
23 #include "storage/bufmgr.h"
24 #include "storage/fd.h"
25 #include "utils/guc_hooks.h"
26 #include "utils/memdebug.h"
27 #include "utils/memutils.h"
28 #include "utils/resowner.h"
29
30
31 /*#define LBDEBUG*/
32
33 /* entry for buffer lookup hashtable */
34 typedef struct
35 {
36 BufferTag key; /* Tag of a disk page */
37 int id; /* Associated local buffer's index */
38 } LocalBufferLookupEnt;
39
40 /* Note: this macro only works on local buffers, not shared ones! */
41 #define LocalBufHdrGetBlock(bufHdr) \
42 LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)]
43
44 int NLocBuffer = 0; /* until buffers are initialized */
45
46 BufferDesc *LocalBufferDescriptors = NULL;
47 Block *LocalBufferBlockPointers = NULL;
48 int32 *LocalRefCount = NULL;
49
50 static int nextFreeLocalBufId = 0;
51
52 static HTAB *LocalBufHash = NULL;
53
54 /* number of local buffers pinned at least once */
55 static int NLocalPinnedBuffers = 0;
56
57
58 static void InitLocalBuffers(void);
59 static Block GetLocalBufferStorage(void);
60 static Buffer GetLocalVictimBuffer(void);
61
62
63 /*
64 * PrefetchLocalBuffer -
65 * initiate asynchronous read of a block of a relation
66 *
67 * Do PrefetchBuffer's work for temporary relations.
68 * No-op if prefetching isn't compiled in.
69 */
70 PrefetchBufferResult
71 PrefetchLocalBuffer(SMgrRelation smgr, ForkNumber forkNum,
72 BlockNumber blockNum)
73 {
74 PrefetchBufferResult result = {InvalidBuffer, false};
75 BufferTag newTag; /* identity of requested block */
76 LocalBufferLookupEnt *hresult;
77
78 InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
79
80 /* Initialize local buffers if first request in this session */
81 if (LocalBufHash == NULL)
82 InitLocalBuffers();
83
84 /* See if the desired buffer already exists */
85 hresult = (LocalBufferLookupEnt *)
86 hash_search(LocalBufHash, &newTag, HASH_FIND, NULL);
87
88 if (hresult)
89 {
90 /* Yes, so nothing to do */
91 result.recent_buffer = -hresult->id - 1;
92 }
93 else
94 {
95 #ifdef USE_PREFETCH
96 /* Not in buffers, so initiate prefetch */
97 if ((io_direct_flags & IO_DIRECT_DATA) == 0 &&
98 smgrprefetch(smgr, forkNum, blockNum, 1))
99 {
100 result.initiated_io = true;
101 }
102 #endif /* USE_PREFETCH */
103 }
104
105 return result;
106 }
107
108
109 /*
110 * LocalBufferAlloc -
111 * Find or create a local buffer for the given page of the given relation.
112 *
113 * API is similar to bufmgr.c's BufferAlloc, except that we do not need to do
114 * any locking since this is all local. We support only default access
115 * strategy (hence, usage_count is always advanced).
116 */
117 BufferDesc *
118 LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum,
119 bool *foundPtr)
120 {
121 BufferTag newTag; /* identity of requested block */
122 LocalBufferLookupEnt *hresult;
123 BufferDesc *bufHdr;
124 Buffer victim_buffer;
125 int bufid;
126 bool found;
127
128 InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
129
130 /* Initialize local buffers if first request in this session */
131 if (LocalBufHash == NULL)
132 InitLocalBuffers();
133
134 ResourceOwnerEnlarge(CurrentResourceOwner);
135
136 /* See if the desired buffer already exists */
137 hresult = (LocalBufferLookupEnt *)
138 hash_search(LocalBufHash, &newTag, HASH_FIND, NULL);
139
140 if (hresult)
141 {
142 bufid = hresult->id;
143 bufHdr = GetLocalBufferDescriptor(bufid);
144 Assert(BufferTagsEqual(&bufHdr->tag, &newTag));
145
146 *foundPtr = PinLocalBuffer(bufHdr, true);
147 }
148 else
149 {
150 uint32 buf_state;
151
152 victim_buffer = GetLocalVictimBuffer();
153 bufid = -victim_buffer - 1;
154 bufHdr = GetLocalBufferDescriptor(bufid);
155
156 hresult = (LocalBufferLookupEnt *)
157 hash_search(LocalBufHash, &newTag, HASH_ENTER, &found);
158 if (found) /* shouldn't happen */
159 elog(ERROR, "local buffer hash table corrupted");
160 hresult->id = bufid;
161
162 /*
163 * it's all ours now.
164 */
165 bufHdr->tag = newTag;
166
167 buf_state = pg_atomic_read_u32(&bufHdr->state);
168 buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
169 buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
170 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
171
172 *foundPtr = false;
173 }
174
175 return bufHdr;
176 }
177
178 /*
179 * Like FlushBuffer(), just for local buffers.
180 */
181 void
182 FlushLocalBuffer(BufferDesc *bufHdr, SMgrRelation reln)
183 {
184 instr_time io_start;
185 Page localpage = (char *) LocalBufHdrGetBlock(bufHdr);
186
187 Assert(LocalRefCount[-BufferDescriptorGetBuffer(bufHdr) - 1] > 0);
188
189 /*
190 * Try to start an I/O operation. There currently are no reasons for
191 * StartLocalBufferIO to return false, so we raise an error in that case.
192 */
193 if (!StartLocalBufferIO(bufHdr, false, false))
194 elog(ERROR, "failed to start write IO on local buffer");
195
196 /* Find smgr relation for buffer */
197 if (reln == NULL)
198 reln = smgropen(BufTagGetRelFileLocator(&bufHdr->tag),
199 MyProcNumber);
200
201 PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
202
203 io_start = pgstat_prepare_io_time(track_io_timing);
204
205 /* And write... */
206 smgrwrite(reln,
207 BufTagGetForkNum(&bufHdr->tag),
208 bufHdr->tag.blockNum,
209 localpage,
210 false);
211
212 /* Temporary table I/O does not use Buffer Access Strategies */
213 pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL,
214 IOOP_WRITE, io_start, 1, BLCKSZ);
215
216 /* Mark not-dirty */
217 TerminateLocalBufferIO(bufHdr, true, 0, false);
218
219 pgBufferUsage.local_blks_written++;
220 }
221
222 static Buffer
223 GetLocalVictimBuffer(void)
224 {
225 int victim_bufid;
226 int trycounter;
227 BufferDesc *bufHdr;
228
229 ResourceOwnerEnlarge(CurrentResourceOwner);
230
231 /*
232 * Need to get a new buffer. We use a clock sweep algorithm (essentially
233 * the same as what freelist.c does now...)
234 */
235 trycounter = NLocBuffer;
236 for (;;)
237 {
238 victim_bufid = nextFreeLocalBufId;
239
240 if (++nextFreeLocalBufId >= NLocBuffer)
241 nextFreeLocalBufId = 0;
242
243 bufHdr = GetLocalBufferDescriptor(victim_bufid);
244
245 if (LocalRefCount[victim_bufid] == 0)
246 {
247 uint32 buf_state = pg_atomic_read_u32(&bufHdr->state);
248
249 if (BUF_STATE_GET_USAGECOUNT(buf_state) > 0)
250 {
251 buf_state -= BUF_USAGECOUNT_ONE;
252 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
253 trycounter = NLocBuffer;
254 }
255 else if (BUF_STATE_GET_REFCOUNT(buf_state) > 0)
256 {
257 /*
258 * This can be reached if the backend initiated AIO for this
259 * buffer and then errored out.
260 */
261 }
262 else
263 {
264 /* Found a usable buffer */
265 PinLocalBuffer(bufHdr, false);
266 break;
267 }
268 }
269 else if (--trycounter == 0)
270 ereport(ERROR,
271 (errcode(ERRCODE_INSUFFICIENT_RESOURCES),
272 errmsg("no empty local buffer available")));
273 }
274
275 /*
276 * lazy memory allocation: allocate space on first use of a buffer.
277 */
278 if (LocalBufHdrGetBlock(bufHdr) == NULL)
279 {
280 /* Set pointer for use by BufferGetBlock() macro */
281 LocalBufHdrGetBlock(bufHdr) = GetLocalBufferStorage();
282 }
283
284 /*
285 * this buffer is not referenced but it might still be dirty. if that's
286 * the case, write it out before reusing it!
287 */
288 if (pg_atomic_read_u32(&bufHdr->state) & BM_DIRTY)
289 FlushLocalBuffer(bufHdr, NULL);
290
291 /*
292 * Remove the victim buffer from the hashtable and mark as invalid.
293 */
294 if (pg_atomic_read_u32(&bufHdr->state) & BM_TAG_VALID)
295 {
296 InvalidateLocalBuffer(bufHdr, false);
297
298 pgstat_count_io_op(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL, IOOP_EVICT, 1, 0);
299 }
300
301 return BufferDescriptorGetBuffer(bufHdr);
302 }
303
304 /* see GetPinLimit() */
305 uint32
306 GetLocalPinLimit(void)
307 {
308 /*
309 * Every backend has its own temporary buffers, but we leave headroom for
310 * concurrent pin-holders -- like multiple scans in the same query.
311 */
312 return num_temp_buffers / 4;
313 }
314
315 /* see GetAdditionalPinLimit() */
316 uint32
317 GetAdditionalLocalPinLimit(void)
318 {
319 uint32 total = GetLocalPinLimit();
320
321 Assert(NLocalPinnedBuffers <= num_temp_buffers);
322
323 if (NLocalPinnedBuffers >= total)
324 return 0;
325 return total - NLocalPinnedBuffers;
326 }
327
328 /* see LimitAdditionalPins() */
329 void
330 LimitAdditionalLocalPins(uint32 *additional_pins)
331 {
332 uint32 max_pins;
333
334 if (*additional_pins <= 1)
335 return;
336
337 /*
338 * In contrast to LimitAdditionalPins() other backends don't play a role
339 * here. We can allow up to NLocBuffer pins in total, but it might not be
340 * initialized yet so read num_temp_buffers.
341 */
342 max_pins = (num_temp_buffers - NLocalPinnedBuffers);
343
344 if (*additional_pins >= max_pins)
345 *additional_pins = max_pins;
346 }
347
348 /*
349 * Implementation of ExtendBufferedRelBy() and ExtendBufferedRelTo() for
350 * temporary buffers.
351 */
352 BlockNumber
353 ExtendBufferedRelLocal(BufferManagerRelation bmr,
354 ForkNumber fork,
355 uint32 flags,
356 uint32 extend_by,
357 BlockNumber extend_upto,
358 Buffer *buffers,
359 uint32 *extended_by)
360 {
361 BlockNumber first_block;
362 instr_time io_start;
363
364 /* Initialize local buffers if first request in this session */
365 if (LocalBufHash == NULL)
366 InitLocalBuffers();
367
368 LimitAdditionalLocalPins(&extend_by);
369
370 for (uint32 i = 0; i < extend_by; i++)
371 {
372 BufferDesc *buf_hdr;
373 Block buf_block;
374
375 buffers[i] = GetLocalVictimBuffer();
376 buf_hdr = GetLocalBufferDescriptor(-buffers[i] - 1);
377 buf_block = LocalBufHdrGetBlock(buf_hdr);
378
379 /* new buffers are zero-filled */
380 MemSet(buf_block, 0, BLCKSZ);
381 }
382
383 first_block = smgrnblocks(bmr.smgr, fork);
384
385 if (extend_upto != InvalidBlockNumber)
386 {
387 /*
388 * In contrast to shared relations, nothing could change the relation
389 * size concurrently. Thus we shouldn't end up finding that we don't
390 * need to do anything.
391 */
392 Assert(first_block <= extend_upto);
393
394 Assert((uint64) first_block + extend_by <= extend_upto);
395 }
396
397 /* Fail if relation is already at maximum possible length */
398 if ((uint64) first_block + extend_by >= MaxBlockNumber)
399 ereport(ERROR,
400 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
401 errmsg("cannot extend relation %s beyond %u blocks",
402 relpath(bmr.smgr->smgr_rlocator, fork).str,
403 MaxBlockNumber)));
404
405 for (uint32 i = 0; i < extend_by; i++)
406 {
407 int victim_buf_id;
408 BufferDesc *victim_buf_hdr;
409 BufferTag tag;
410 LocalBufferLookupEnt *hresult;
411 bool found;
412
413 victim_buf_id = -buffers[i] - 1;
414 victim_buf_hdr = GetLocalBufferDescriptor(victim_buf_id);
415
416 /* in case we need to pin an existing buffer below */
417 ResourceOwnerEnlarge(CurrentResourceOwner);
418
419 InitBufferTag(&tag, &bmr.smgr->smgr_rlocator.locator, fork, first_block + i);
420
421 hresult = (LocalBufferLookupEnt *)
422 hash_search(LocalBufHash, &tag, HASH_ENTER, &found);
423 if (found)
424 {
425 BufferDesc *existing_hdr;
426 uint32 buf_state;
427
428 UnpinLocalBuffer(BufferDescriptorGetBuffer(victim_buf_hdr));
429
430 existing_hdr = GetLocalBufferDescriptor(hresult->id);
431 PinLocalBuffer(existing_hdr, false);
432 buffers[i] = BufferDescriptorGetBuffer(existing_hdr);
433
434 /*
435 * Clear the BM_VALID bit, do StartLocalBufferIO() and proceed.
436 */
437 buf_state = pg_atomic_read_u32(&existing_hdr->state);
438 Assert(buf_state & BM_TAG_VALID);
439 Assert(!(buf_state & BM_DIRTY));
440 buf_state &= ~BM_VALID;
441 pg_atomic_unlocked_write_u32(&existing_hdr->state, buf_state);
442
443 /* no need to loop for local buffers */
444 StartLocalBufferIO(existing_hdr, true, false);
445 }
446 else
447 {
448 uint32 buf_state = pg_atomic_read_u32(&victim_buf_hdr->state);
449
450 Assert(!(buf_state & (BM_VALID | BM_TAG_VALID | BM_DIRTY | BM_JUST_DIRTIED)));
451
452 victim_buf_hdr->tag = tag;
453
454 buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
455
456 pg_atomic_unlocked_write_u32(&victim_buf_hdr->state, buf_state);
457
458 hresult->id = victim_buf_id;
459
460 StartLocalBufferIO(victim_buf_hdr, true, false);
461 }
462 }
463
464 io_start = pgstat_prepare_io_time(track_io_timing);
465
466 /* actually extend relation */
467 smgrzeroextend(bmr.smgr, fork, first_block, extend_by, false);
468
469 pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL, IOOP_EXTEND,
470 io_start, 1, extend_by * BLCKSZ);
471
472 for (uint32 i = 0; i < extend_by; i++)
473 {
474 Buffer buf = buffers[i];
475 BufferDesc *buf_hdr;
476 uint32 buf_state;
477
478 buf_hdr = GetLocalBufferDescriptor(-buf - 1);
479
480 buf_state = pg_atomic_read_u32(&buf_hdr->state);
481 buf_state |= BM_VALID;
482 pg_atomic_unlocked_write_u32(&buf_hdr->state, buf_state);
483 }
484
485 *extended_by = extend_by;
486
487 pgBufferUsage.local_blks_written += extend_by;
488
489 return first_block;
490 }
491
492 /*
493 * MarkLocalBufferDirty -
494 * mark a local buffer dirty
495 */
496 void
497 MarkLocalBufferDirty(Buffer buffer)
498 {
499 int bufid;
500 BufferDesc *bufHdr;
501 uint32 buf_state;
502
503 Assert(BufferIsLocal(buffer));
504
505 #ifdef LBDEBUG
506 fprintf(stderr, "LB DIRTY %d\n", buffer);
507 #endif
508
509 bufid = -buffer - 1;
510
511 Assert(LocalRefCount[bufid] > 0);
512
513 bufHdr = GetLocalBufferDescriptor(bufid);
514
515 buf_state = pg_atomic_read_u32(&bufHdr->state);
516
517 if (!(buf_state & BM_DIRTY))
518 pgBufferUsage.local_blks_dirtied++;
519
520 buf_state |= BM_DIRTY;
521
522 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
523 }
524
525 /*
526 * Like StartBufferIO, but for local buffers
527 */
528 bool
529 StartLocalBufferIO(BufferDesc *bufHdr, bool forInput, bool nowait)
530 {
531 uint32 buf_state;
532
533 /*
534 * With AIO the buffer could have IO in progress, e.g. when there are two
535 * scans of the same relation. Either wait for the other IO or return
536 * false.
537 */
538 if (pgaio_wref_valid(&bufHdr->io_wref))
539 {
540 PgAioWaitRef iow = bufHdr->io_wref;
541
542 if (nowait)
543 return false;
544
545 pgaio_wref_wait(&iow);
546 }
547
548 /* Once we get here, there is definitely no I/O active on this buffer */
549
550 /* Check if someone else already did the I/O */
551 buf_state = pg_atomic_read_u32(&bufHdr->state);
552 if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
553 {
554 return false;
555 }
556
557 /* BM_IO_IN_PROGRESS isn't currently used for local buffers */
558
559 /* local buffers don't track IO using resowners */
560
561 return true;
562 }
563
564 /*
565 * Like TerminateBufferIO, but for local buffers
566 */
567 void
568 TerminateLocalBufferIO(BufferDesc *bufHdr, bool clear_dirty, uint32 set_flag_bits,
569 bool release_aio)
570 {
571 /* Only need to adjust flags */
572 uint32 buf_state = pg_atomic_read_u32(&bufHdr->state);
573
574 /* BM_IO_IN_PROGRESS isn't currently used for local buffers */
575
576 /* Clear earlier errors, if this IO failed, it'll be marked again */
577 buf_state &= ~BM_IO_ERROR;
578
579 if (clear_dirty)
580 buf_state &= ~BM_DIRTY;
581
582 if (release_aio)
583 {
584 /* release pin held by IO subsystem, see also buffer_stage_common() */
585 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
586 buf_state -= BUF_REFCOUNT_ONE;
587 pgaio_wref_clear(&bufHdr->io_wref);
588 }
589
590 buf_state |= set_flag_bits;
591 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
592
593 /* local buffers don't track IO using resowners */
594
595 /* local buffers don't use the IO CV, as no other process can see buffer */
596
597 /* local buffers don't use BM_PIN_COUNT_WAITER, so no need to wake */
598 }
599
600 /*
601 * InvalidateLocalBuffer -- mark a local buffer invalid.
602 *
603 * If check_unreferenced is true, error out if the buffer is still
604 * pinned. Passing false is appropriate when calling InvalidateLocalBuffer()
605 * as part of changing the identity of a buffer, instead of just dropping the
606 * buffer.
607 *
608 * See also InvalidateBuffer().
609 */
610 void
611 InvalidateLocalBuffer(BufferDesc *bufHdr, bool check_unreferenced)
612 {
613 Buffer buffer = BufferDescriptorGetBuffer(bufHdr);
614 int bufid = -buffer - 1;
615 uint32 buf_state;
616 LocalBufferLookupEnt *hresult;
617
618 /*
619 * It's possible that we started IO on this buffer before e.g. aborting
620 * the transaction that created a table. We need to wait for that IO to
621 * complete before removing / reusing the buffer.
622 */
623 if (pgaio_wref_valid(&bufHdr->io_wref))
624 {
625 PgAioWaitRef iow = bufHdr->io_wref;
626
627 pgaio_wref_wait(&iow);
628 Assert(!pgaio_wref_valid(&bufHdr->io_wref));
629 }
630
631 buf_state = pg_atomic_read_u32(&bufHdr->state);
632
633 /*
634 * We need to test not just LocalRefCount[bufid] but also the BufferDesc
635 * itself, as the latter is used to represent a pin by the AIO subsystem.
636 * This can happen if AIO is initiated and then the query errors out.
637 */
638 if (check_unreferenced &&
639 (LocalRefCount[bufid] != 0 || BUF_STATE_GET_REFCOUNT(buf_state) != 0))
640 elog(ERROR, "block %u of %s is still referenced (local %d)",
641 bufHdr->tag.blockNum,
642 relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag),
643 MyProcNumber,
644 BufTagGetForkNum(&bufHdr->tag)).str,
645 LocalRefCount[bufid]);
646
647 /* Remove entry from hashtable */
648 hresult = (LocalBufferLookupEnt *)
649 hash_search(LocalBufHash, &bufHdr->tag, HASH_REMOVE, NULL);
650 if (!hresult) /* shouldn't happen */
651 elog(ERROR, "local buffer hash table corrupted");
652 /* Mark buffer invalid */
653 ClearBufferTag(&bufHdr->tag);
654 buf_state &= ~BUF_FLAG_MASK;
655 buf_state &= ~BUF_USAGECOUNT_MASK;
656 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
657 }
658
659 /*
660 * DropRelationLocalBuffers
661 * This function removes from the buffer pool all the pages of the
662 * specified relation that have block numbers >= firstDelBlock.
663 * (In particular, with firstDelBlock = 0, all pages are removed.)
664 * Dirty pages are simply dropped, without bothering to write them
665 * out first. Therefore, this is NOT rollback-able, and so should be
666 * used only with extreme caution!
667 *
668 * See DropRelationBuffers in bufmgr.c for more notes.
669 */
670 void
671 DropRelationLocalBuffers(RelFileLocator rlocator, ForkNumber forkNum,
672 BlockNumber firstDelBlock)
673 {
674 int i;
675
676 for (i = 0; i < NLocBuffer; i++)
677 {
678 BufferDesc *bufHdr = GetLocalBufferDescriptor(i);
679 uint32 buf_state;
680
681 buf_state = pg_atomic_read_u32(&bufHdr->state);
682
683 if ((buf_state & BM_TAG_VALID) &&
684 BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) &&
685 BufTagGetForkNum(&bufHdr->tag) == forkNum &&
686 bufHdr->tag.blockNum >= firstDelBlock)
687 {
688 InvalidateLocalBuffer(bufHdr, true);
689 }
690 }
691 }
692
693 /*
694 * DropRelationAllLocalBuffers
695 * This function removes from the buffer pool all pages of all forks
696 * of the specified relation.
697 *
698 * See DropRelationsAllBuffers in bufmgr.c for more notes.
699 */
700 void
701 DropRelationAllLocalBuffers(RelFileLocator rlocator)
702 {
703 int i;
704
705 for (i = 0; i < NLocBuffer; i++)
706 {
707 BufferDesc *bufHdr = GetLocalBufferDescriptor(i);
708 uint32 buf_state;
709
710 buf_state = pg_atomic_read_u32(&bufHdr->state);
711
712 if ((buf_state & BM_TAG_VALID) &&
713 BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator))
714 {
715 InvalidateLocalBuffer(bufHdr, true);
716 }
717 }
718 }
719
720 /*
721 * InitLocalBuffers -
722 * init the local buffer cache. Since most queries (esp. multi-user ones)
723 * don't involve local buffers, we delay allocating actual memory for the
724 * buffers until we need them; just make the buffer headers here.
725 */
726 static void
727 InitLocalBuffers(void)
728 {
729 int nbufs = num_temp_buffers;
730 HASHCTL info;
731 int i;
732
733 /*
734 * Parallel workers can't access data in temporary tables, because they
735 * have no visibility into the local buffers of their leader. This is a
736 * convenient, low-cost place to provide a backstop check for that. Note
737 * that we don't wish to prevent a parallel worker from accessing catalog
738 * metadata about a temp table, so checks at higher levels would be
739 * inappropriate.
740 */
741 if (IsParallelWorker())
742 ereport(ERROR,
743 (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
744 errmsg("cannot access temporary tables during a parallel operation")));
745
746 /* Allocate and zero buffer headers and auxiliary arrays */
747 LocalBufferDescriptors = (BufferDesc *) calloc(nbufs, sizeof(BufferDesc));
748 LocalBufferBlockPointers = (Block *) calloc(nbufs, sizeof(Block));
749 LocalRefCount = (int32 *) calloc(nbufs, sizeof(int32));
750 if (!LocalBufferDescriptors || !LocalBufferBlockPointers || !LocalRefCount)
751 ereport(FATAL,
752 (errcode(ERRCODE_OUT_OF_MEMORY),
753 errmsg("out of memory")));
754
755 nextFreeLocalBufId = 0;
756
757 /* initialize fields that need to start off nonzero */
758 for (i = 0; i < nbufs; i++)
759 {
760 BufferDesc *buf = GetLocalBufferDescriptor(i);
761
762 /*
763 * negative to indicate local buffer. This is tricky: shared buffers
764 * start with 0. We have to start with -2. (Note that the routine
765 * BufferDescriptorGetBuffer adds 1 to buf_id so our first buffer id
766 * is -1.)
767 */
768 buf->buf_id = -i - 2;
769
770 pgaio_wref_clear(&buf->io_wref);
771
772 /*
773 * Intentionally do not initialize the buffer's atomic variable
774 * (besides zeroing the underlying memory above). That way we get
775 * errors on platforms without atomics, if somebody (re-)introduces
776 * atomic operations for local buffers.
777 */
778 }
779
780 /* Create the lookup hash table */
781 info.keysize = sizeof(BufferTag);
782 info.entrysize = sizeof(LocalBufferLookupEnt);
783
784 LocalBufHash = hash_create("Local Buffer Lookup Table",
785 nbufs,
786 &info,
787 HASH_ELEM | HASH_BLOBS);
788
789 if (!LocalBufHash)
790 elog(ERROR, "could not initialize local buffer hash table");
791
792 /* Initialization done, mark buffers allocated */
793 NLocBuffer = nbufs;
794 }
795
796 /*
797 * XXX: We could have a slightly more efficient version of PinLocalBuffer()
798 * that does not support adjusting the usagecount - but so far it does not
799 * seem worth the trouble.
800 *
801 * Note that ResourceOwnerEnlarge() must have been done already.
802 */
803 bool
804 PinLocalBuffer(BufferDesc *buf_hdr, bool adjust_usagecount)
805 {
806 uint32 buf_state;
807 Buffer buffer = BufferDescriptorGetBuffer(buf_hdr);
808 int bufid = -buffer - 1;
809
810 buf_state = pg_atomic_read_u32(&buf_hdr->state);
811
812 if (LocalRefCount[bufid] == 0)
813 {
814 NLocalPinnedBuffers++;
815 buf_state += BUF_REFCOUNT_ONE;
816 if (adjust_usagecount &&
817 BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
818 {
819 buf_state += BUF_USAGECOUNT_ONE;
820 }
821 pg_atomic_unlocked_write_u32(&buf_hdr->state, buf_state);
822
823 /*
824 * See comment in PinBuffer().
825 *
826 * If the buffer isn't allocated yet, it'll be marked as defined in
827 * GetLocalBufferStorage().
828 */
829 if (LocalBufHdrGetBlock(buf_hdr) != NULL)
830 VALGRIND_MAKE_MEM_DEFINED(LocalBufHdrGetBlock(buf_hdr), BLCKSZ);
831 }
832 LocalRefCount[bufid]++;
833 ResourceOwnerRememberBuffer(CurrentResourceOwner,
834 BufferDescriptorGetBuffer(buf_hdr));
835
836 return buf_state & BM_VALID;
837 }
838
839 void
840 UnpinLocalBuffer(Buffer buffer)
841 {
842 UnpinLocalBufferNoOwner(buffer);
843 ResourceOwnerForgetBuffer(CurrentResourceOwner, buffer);
844 }
845
846 void
847 UnpinLocalBufferNoOwner(Buffer buffer)
848 {
849 int buffid = -buffer - 1;
850
851 Assert(BufferIsLocal(buffer));
852 Assert(LocalRefCount[buffid] > 0);
853 Assert(NLocalPinnedBuffers > 0);
854
855 if (--LocalRefCount[buffid] == 0)
856 {
857 BufferDesc *buf_hdr = GetLocalBufferDescriptor(buffid);
858 uint32 buf_state;
859
860 NLocalPinnedBuffers--;
861
862 buf_state = pg_atomic_read_u32(&buf_hdr->state);
863 Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
864 buf_state -= BUF_REFCOUNT_ONE;
865 pg_atomic_unlocked_write_u32(&buf_hdr->state, buf_state);
866
867 /* see comment in UnpinBufferNoOwner */
868 VALGRIND_MAKE_MEM_NOACCESS(LocalBufHdrGetBlock(buf_hdr), BLCKSZ);
869 }
870 }
871
872 /*
873 * GUC check_hook for temp_buffers
874 */
875 bool
876 check_temp_buffers(int *newval, void **extra, GucSource source)
877 {
878 /*
879 * Once local buffers have been initialized, it's too late to change this.
880 * However, if this is only a test call, allow it.
881 */
882 if (source != PGC_S_TEST && NLocBuffer && NLocBuffer != *newval)
883 {
884 GUC_check_errdetail("\"temp_buffers\" cannot be changed after any temporary tables have been accessed in the session.");
885 return false;
886 }
887 return true;
888 }
889
890 /*
891 * GetLocalBufferStorage - allocate memory for a local buffer
892 *
893 * The idea of this function is to aggregate our requests for storage
894 * so that the memory manager doesn't see a whole lot of relatively small
895 * requests. Since we'll never give back a local buffer once it's created
896 * within a particular process, no point in burdening memmgr with separately
897 * managed chunks.
898 */
899 static Block
900 GetLocalBufferStorage(void)
901 {
902 static char *cur_block = NULL;
903 static int next_buf_in_block = 0;
904 static int num_bufs_in_block = 0;
905 static int total_bufs_allocated = 0;
906 static MemoryContext LocalBufferContext = NULL;
907
908 char *this_buf;
909
910 Assert(total_bufs_allocated < NLocBuffer);
911
912 if (next_buf_in_block >= num_bufs_in_block)
913 {
914 /* Need to make a new request to memmgr */
915 int num_bufs;
916
917 /*
918 * We allocate local buffers in a context of their own, so that the
919 * space eaten for them is easily recognizable in MemoryContextStats
920 * output. Create the context on first use.
921 */
922 if (LocalBufferContext == NULL)
923 LocalBufferContext =
924 AllocSetContextCreate(TopMemoryContext,
925 "LocalBufferContext",
926 ALLOCSET_DEFAULT_SIZES);
927
928 /* Start with a 16-buffer request; subsequent ones double each time */
929 num_bufs = Max(num_bufs_in_block * 2, 16);
930 /* But not more than what we need for all remaining local bufs */
931 num_bufs = Min(num_bufs, NLocBuffer - total_bufs_allocated);
932 /* And don't overflow MaxAllocSize, either */
933 num_bufs = Min(num_bufs, MaxAllocSize / BLCKSZ);
934
935 /* Buffers should be I/O aligned. */
936 cur_block = (char *)
937 TYPEALIGN(PG_IO_ALIGN_SIZE,
938 MemoryContextAlloc(LocalBufferContext,
939 num_bufs * BLCKSZ + PG_IO_ALIGN_SIZE));
940 next_buf_in_block = 0;
941 num_bufs_in_block = num_bufs;
942 }
943
944 /* Allocate next buffer in current memory block */
945 this_buf = cur_block + next_buf_in_block * BLCKSZ;
946 next_buf_in_block++;
947 total_bufs_allocated++;
948
949 /*
950 * Caller's PinLocalBuffer() was too early for Valgrind updates, so do it
951 * here. The block is actually undefined, but we want consistency with
952 * the regular case of not needing to allocate memory. This is
953 * specifically needed when method_io_uring.c fills the block, because
954 * Valgrind doesn't recognize io_uring reads causing undefined memory to
955 * become defined.
956 */
957 VALGRIND_MAKE_MEM_DEFINED(this_buf, BLCKSZ);
958
959 return (Block) this_buf;
960 }
961
962 /*
963 * CheckForLocalBufferLeaks - ensure this backend holds no local buffer pins
964 *
965 * This is just like CheckForBufferLeaks(), but for local buffers.
966 */
967 static void
968 CheckForLocalBufferLeaks(void)
969 {
970 #ifdef USE_ASSERT_CHECKING
971 if (LocalRefCount)
972 {
973 int RefCountErrors = 0;
974 int i;
975
976 for (i = 0; i < NLocBuffer; i++)
977 {
978 if (LocalRefCount[i] != 0)
979 {
980 Buffer b = -i - 1;
981 char *s;
982
983 s = DebugPrintBufferRefcount(b);
984 elog(WARNING, "local buffer refcount leak: %s", s);
985 pfree(s);
986
987 RefCountErrors++;
988 }
989 }
990 Assert(RefCountErrors == 0);
991 }
992 #endif
993 }
994
995 /*
996 * AtEOXact_LocalBuffers - clean up at end of transaction.
997 *
998 * This is just like AtEOXact_Buffers, but for local buffers.
999 */
1000 void
1001 AtEOXact_LocalBuffers(bool isCommit)
1002 {
1003 CheckForLocalBufferLeaks();
1004 }
1005
1006 /*
1007 * AtProcExit_LocalBuffers - ensure we have dropped pins during backend exit.
1008 *
1009 * This is just like AtProcExit_Buffers, but for local buffers.
1010 */
1011 void
1012 AtProcExit_LocalBuffers(void)
1013 {
1014 /*
1015 * We shouldn't be holding any remaining pins; if we are, and assertions
1016 * aren't enabled, we'll fail later in DropRelationBuffers while trying to
1017 * drop the temp rels.
1018 */
1019 CheckForLocalBufferLeaks();
1020 }