Debian Codegraph @

src/backend/storage/aio/read_stream.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * read_stream.c
4 * Mechanism for accessing buffered relation data with look-ahead
5 *
6 * Code that needs to access relation data typically pins blocks one at a
7 * time, often in a predictable order that might be sequential or data-driven.
8 * Calling the simple ReadBuffer() function for each block is inefficient,
9 * because blocks that are not yet in the buffer pool require I/O operations
10 * that are small and might stall waiting for storage. This mechanism looks
11 * into the future and calls StartReadBuffers() and WaitReadBuffers() to read
12 * neighboring blocks together and ahead of time, with an adaptive look-ahead
13 * distance.
14 *
15 * A user-provided callback generates a stream of block numbers that is used
16 * to form reads of up to io_combine_limit, by attempting to merge them with a
17 * pending read. When that isn't possible, the existing pending read is sent
18 * to StartReadBuffers() so that a new one can begin to form.
19 *
20 * The algorithm for controlling the look-ahead distance is based on recent
21 * cache hit and miss history. When no I/O is necessary, there is no benefit
22 * in looking ahead more than one block. This is the default initial
23 * assumption, but when blocks needing I/O are streamed, the distance is
24 * increased rapidly to try to benefit from I/O combining and concurrency. It
25 * is reduced gradually when cached blocks are streamed.
26 *
27 * The main data structure is a circular queue of buffers of size
28 * max_pinned_buffers plus some extra space for technical reasons, ready to be
29 * returned by read_stream_next_buffer(). Each buffer also has an optional
30 * variable sized object that is passed from the callback to the consumer of
31 * buffers.
32 *
33 * Parallel to the queue of buffers, there is a circular queue of in-progress
34 * I/Os that have been started with StartReadBuffers(), and for which
35 * WaitReadBuffers() must be called before returning the buffer.
36 *
37 * For example, if the callback returns block numbers 10, 42, 43, 44, 60 in
38 * successive calls, then these data structures might appear as follows:
39 *
40 * buffers buf/data ios
41 *
42 * +----+ +-----+ +--------+
43 * | | | | +----+ 42..44 | <- oldest_io_index
44 * +----+ +-----+ | +--------+
45 * oldest_buffer_index -> | 10 | | ? | | +--+ 60..60 |
46 * +----+ +-----+ | | +--------+
47 * | 42 | | ? |<-+ | | | <- next_io_index
48 * +----+ +-----+ | +--------+
49 * | 43 | | ? | | | |
50 * +----+ +-----+ | +--------+
51 * | 44 | | ? | | | |
52 * +----+ +-----+ | +--------+
53 * | 60 | | ? |<---+
54 * +----+ +-----+
55 * next_buffer_index -> | | | |
56 * +----+ +-----+
57 *
58 * In the example, 5 buffers are pinned, and the next buffer to be streamed to
59 * the client is block 10. Block 10 was a hit and has no associated I/O, but
60 * the range 42..44 requires an I/O wait before its buffers are returned, as
61 * does block 60.
62 *
63 *
64 * Portions Copyright (c) 2024-2025, PostgreSQL Global Development Group
65 * Portions Copyright (c) 1994, Regents of the University of California
66 *
67 * IDENTIFICATION
68 * src/backend/storage/aio/read_stream.c
69 *
70 *-------------------------------------------------------------------------
71 */
72 #include "postgres.h"
73
74 #include "miscadmin.h"
75 #include "storage/aio.h"
76 #include "storage/fd.h"
77 #include "storage/smgr.h"
78 #include "storage/read_stream.h"
79 #include "utils/memdebug.h"
80 #include "utils/rel.h"
81 #include "utils/spccache.h"
82
83 typedef struct InProgressIO
84 {
85 int16 buffer_index;
86 ReadBuffersOperation op;
87 } InProgressIO;
88
89 /*
90 * State for managing a stream of reads.
91 */
92 struct ReadStream
93 {
94 int16 max_ios;
95 int16 io_combine_limit;
96 int16 ios_in_progress;
97 int16 queue_size;
98 int16 max_pinned_buffers;
99 int16 forwarded_buffers;
100 int16 pinned_buffers;
101 int16 distance;
102 int16 initialized_buffers;
103 int read_buffers_flags;
104 bool sync_mode; /* using io_method=sync */
105 bool batch_mode; /* READ_STREAM_USE_BATCHING */
106 bool advice_enabled;
107 bool temporary;
108
109 /*
110 * One-block buffer to support 'ungetting' a block number, to resolve flow
111 * control problems when I/Os are split.
112 */
113 BlockNumber buffered_blocknum;
114
115 /*
116 * The callback that will tell us which block numbers to read, and an
117 * opaque pointer that will be pass to it for its own purposes.
118 */
119 ReadStreamBlockNumberCB callback;
120 void *callback_private_data;
121
122 /* Next expected block, for detecting sequential access. */
123 BlockNumber seq_blocknum;
124 BlockNumber seq_until_processed;
125
126 /* The read operation we are currently preparing. */
127 BlockNumber pending_read_blocknum;
128 int16 pending_read_nblocks;
129
130 /* Space for buffers and optional per-buffer private data. */
131 size_t per_buffer_data_size;
132 void *per_buffer_data;
133
134 /* Read operations that have been started but not waited for yet. */
135 InProgressIO *ios;
136 int16 oldest_io_index;
137 int16 next_io_index;
138
139 bool fast_path;
140
141 /* Circular queue of buffers. */
142 int16 oldest_buffer_index; /* Next pinned buffer to return */
143 int16 next_buffer_index; /* Index of next buffer to pin */
144 Buffer buffers[FLEXIBLE_ARRAY_MEMBER];
145 };
146
147 /*
148 * Return a pointer to the per-buffer data by index.
149 */
150 static inline void *
151 get_per_buffer_data(ReadStream *stream, int16 buffer_index)
152 {
153 return (char *) stream->per_buffer_data +
154 stream->per_buffer_data_size * buffer_index;
155 }
156
157 /*
158 * General-use ReadStreamBlockNumberCB for block range scans. Loops over the
159 * blocks [current_blocknum, last_exclusive).
160 */
161 BlockNumber
162 block_range_read_stream_cb(ReadStream *stream,
163 void *callback_private_data,
164 void *per_buffer_data)
165 {
166 BlockRangeReadStreamPrivate *p = callback_private_data;
167
168 if (p->current_blocknum < p->last_exclusive)
169 return p->current_blocknum++;
170
171 return InvalidBlockNumber;
172 }
173
174 /*
175 * Ask the callback which block it would like us to read next, with a one block
176 * buffer in front to allow read_stream_unget_block() to work.
177 */
178 static inline BlockNumber
179 read_stream_get_block(ReadStream *stream, void *per_buffer_data)
180 {
181 BlockNumber blocknum;
182
183 blocknum = stream->buffered_blocknum;
184 if (blocknum != InvalidBlockNumber)
185 stream->buffered_blocknum = InvalidBlockNumber;
186 else
187 {
188 /*
189 * Tell Valgrind that the per-buffer data is undefined. That replaces
190 * the "noaccess" state that was set when the consumer moved past this
191 * entry last time around the queue, and should also catch callbacks
192 * that fail to initialize data that the buffer consumer later
193 * accesses. On the first go around, it is undefined already.
194 */
195 VALGRIND_MAKE_MEM_UNDEFINED(per_buffer_data,
196 stream->per_buffer_data_size);
197 blocknum = stream->callback(stream,
198 stream->callback_private_data,
199 per_buffer_data);
200 }
201
202 return blocknum;
203 }
204
205 /*
206 * In order to deal with buffer shortages and I/O limits after short reads, we
207 * sometimes need to defer handling of a block we've already consumed from the
208 * registered callback until later.
209 */
210 static inline void
211 read_stream_unget_block(ReadStream *stream, BlockNumber blocknum)
212 {
213 /* We shouldn't ever unget more than one block. */
214 Assert(stream->buffered_blocknum == InvalidBlockNumber);
215 Assert(blocknum != InvalidBlockNumber);
216 stream->buffered_blocknum = blocknum;
217 }
218
219 /*
220 * Start as much of the current pending read as we can. If we have to split it
221 * because of the per-backend buffer limit, or the buffer manager decides to
222 * split it, then the pending read is adjusted to hold the remaining portion.
223 *
224 * We can always start a read of at least size one if we have no progress yet.
225 * Otherwise it's possible that we can't start a read at all because of a lack
226 * of buffers, and then false is returned. Buffer shortages also reduce the
227 * distance to a level that prevents look-ahead until buffers are released.
228 */
229 static bool
230 read_stream_start_pending_read(ReadStream *stream)
231 {
232 bool need_wait;
233 int requested_nblocks;
234 int nblocks;
235 int flags;
236 int forwarded;
237 int16 io_index;
238 int16 overflow;
239 int16 buffer_index;
240 int buffer_limit;
241
242 /* This should only be called with a pending read. */
243 Assert(stream->pending_read_nblocks > 0);
244 Assert(stream->pending_read_nblocks <= stream->io_combine_limit);
245
246 /* We had better not exceed the per-stream buffer limit with this read. */
247 Assert(stream->pinned_buffers + stream->pending_read_nblocks <=
248 stream->max_pinned_buffers);
249
250 #ifdef USE_ASSERT_CHECKING
251 /* We had better not be overwriting an existing pinned buffer. */
252 if (stream->pinned_buffers > 0)
253 Assert(stream->next_buffer_index != stream->oldest_buffer_index);
254 else
255 Assert(stream->next_buffer_index == stream->oldest_buffer_index);
256
257 /*
258 * Pinned buffers forwarded by a preceding StartReadBuffers() call that
259 * had to split the operation should match the leading blocks of this
260 * following StartReadBuffers() call.
261 */
262 Assert(stream->forwarded_buffers <= stream->pending_read_nblocks);
263 for (int i = 0; i < stream->forwarded_buffers; ++i)
264 Assert(BufferGetBlockNumber(stream->buffers[stream->next_buffer_index + i]) ==
265 stream->pending_read_blocknum + i);
266
267 /*
268 * Check that we've cleared the queue/overflow entries corresponding to
269 * the rest of the blocks covered by this read, unless it's the first go
270 * around and we haven't even initialized them yet.
271 */
272 for (int i = stream->forwarded_buffers; i < stream->pending_read_nblocks; ++i)
273 Assert(stream->next_buffer_index + i >= stream->initialized_buffers ||
274 stream->buffers[stream->next_buffer_index + i] == InvalidBuffer);
275 #endif
276
277 /* Do we need to issue read-ahead advice? */
278 flags = stream->read_buffers_flags;
279 if (stream->advice_enabled)
280 {
281 if (stream->pending_read_blocknum == stream->seq_blocknum)
282 {
283 /*
284 * Sequential: Issue advice until the preadv() calls have caught
285 * up with the first advice issued for this sequential region, and
286 * then stay of the way of the kernel's own read-ahead.
287 */
288 if (stream->seq_until_processed != InvalidBlockNumber)
289 flags |= READ_BUFFERS_ISSUE_ADVICE;
290 }
291 else
292 {
293 /*
294 * Random jump: Note the starting location of a new potential
295 * sequential region and start issuing advice. Skip it this time
296 * if the preadv() follows immediately, eg first block in stream.
297 */
298 stream->seq_until_processed = stream->pending_read_blocknum;
299 if (stream->pinned_buffers > 0)
300 flags |= READ_BUFFERS_ISSUE_ADVICE;
301 }
302 }
303
304 /*
305 * How many more buffers is this backend allowed?
306 *
307 * Forwarded buffers are already pinned and map to the leading blocks of
308 * the pending read (the remaining portion of an earlier short read that
309 * we're about to continue). They are not counted in pinned_buffers, but
310 * they are counted as pins already held by this backend according to the
311 * buffer manager, so they must be added to the limit it grants us.
312 */
313 if (stream->temporary)
314 buffer_limit = Min(GetAdditionalLocalPinLimit(), PG_INT16_MAX);
315 else
316 buffer_limit = Min(GetAdditionalPinLimit(), PG_INT16_MAX);
317 Assert(stream->forwarded_buffers <= stream->pending_read_nblocks);
318
319 buffer_limit += stream->forwarded_buffers;
320 buffer_limit = Min(buffer_limit, PG_INT16_MAX);
321
322 if (buffer_limit == 0 && stream->pinned_buffers == 0)
323 buffer_limit = 1; /* guarantee progress */
324
325 /* Does the per-backend limit affect this read? */
326 nblocks = stream->pending_read_nblocks;
327 if (buffer_limit < nblocks)
328 {
329 int16 new_distance;
330
331 /* Shrink distance: no more look-ahead until buffers are released. */
332 new_distance = stream->pinned_buffers + buffer_limit;
333 if (stream->distance > new_distance)
334 stream->distance = new_distance;
335
336 /* Unless we have nothing to give the consumer, stop here. */
337 if (stream->pinned_buffers > 0)
338 return false;
339
340 /* A short read is required to make progress. */
341 nblocks = buffer_limit;
342 }
343
344 /*
345 * We say how many blocks we want to read, but it may be smaller on return
346 * if the buffer manager decides to shorten the read. Initialize buffers
347 * to InvalidBuffer (= not a forwarded buffer) as input on first use only,
348 * and keep the original nblocks number so we can check for forwarded
349 * buffers as output, below.
350 */
351 buffer_index = stream->next_buffer_index;
352 io_index = stream->next_io_index;
353 while (stream->initialized_buffers < buffer_index + nblocks)
354 stream->buffers[stream->initialized_buffers++] = InvalidBuffer;
355 requested_nblocks = nblocks;
356 need_wait = StartReadBuffers(&stream->ios[io_index].op,
357 &stream->buffers[buffer_index],
358 stream->pending_read_blocknum,
359 &nblocks,
360 flags);
361 stream->pinned_buffers += nblocks;
362
363 /* Remember whether we need to wait before returning this buffer. */
364 if (!need_wait)
365 {
366 /* Look-ahead distance decays, no I/O necessary. */
367 if (stream->distance > 1)
368 stream->distance--;
369 }
370 else
371 {
372 /*
373 * Remember to call WaitReadBuffers() before returning head buffer.
374 * Look-ahead distance will be adjusted after waiting.
375 */
376 stream->ios[io_index].buffer_index = buffer_index;
377 if (++stream->next_io_index == stream->max_ios)
378 stream->next_io_index = 0;
379 Assert(stream->ios_in_progress < stream->max_ios);
380 stream->ios_in_progress++;
381 stream->seq_blocknum = stream->pending_read_blocknum + nblocks;
382 }
383
384 /*
385 * How many pins were acquired but forwarded to the next call? These need
386 * to be passed to the next StartReadBuffers() call by leaving them
387 * exactly where they are in the queue, or released if the stream ends
388 * early. We need the number for accounting purposes, since they are not
389 * counted in stream->pinned_buffers but we already hold them.
390 */
391 forwarded = 0;
392 while (nblocks + forwarded < requested_nblocks &&
393 stream->buffers[buffer_index + nblocks + forwarded] != InvalidBuffer)
394 forwarded++;
395 stream->forwarded_buffers = forwarded;
396
397 /*
398 * We gave a contiguous range of buffer space to StartReadBuffers(), but
399 * we want it to wrap around at queue_size. Copy overflowing buffers to
400 * the front of the array where they'll be consumed, but also leave a copy
401 * in the overflow zone which the I/O operation has a pointer to (it needs
402 * a contiguous array). Both copies will be cleared when the buffers are
403 * handed to the consumer.
404 */
405 overflow = (buffer_index + nblocks + forwarded) - stream->queue_size;
406 if (overflow > 0)
407 {
408 Assert(overflow < stream->queue_size); /* can't overlap */
409 memcpy(&stream->buffers[0],
410 &stream->buffers[stream->queue_size],
411 sizeof(stream->buffers[0]) * overflow);
412 }
413
414 /* Compute location of start of next read, without using % operator. */
415 buffer_index += nblocks;
416 if (buffer_index >= stream->queue_size)
417 buffer_index -= stream->queue_size;
418 Assert(buffer_index >= 0 && buffer_index < stream->queue_size);
419 stream->next_buffer_index = buffer_index;
420
421 /* Adjust the pending read to cover the remaining portion, if any. */
422 stream->pending_read_blocknum += nblocks;
423 stream->pending_read_nblocks -= nblocks;
424
425 return true;
426 }
427
428 static void
429 read_stream_look_ahead(ReadStream *stream)
430 {
431 /*
432 * Allow amortizing the cost of submitting IO over multiple IOs. This
433 * requires that we don't do any operations that could lead to a deadlock
434 * with staged-but-unsubmitted IO. The callback needs to opt-in to being
435 * careful.
436 */
437 if (stream->batch_mode)
438 pgaio_enter_batchmode();
439
440 while (stream->ios_in_progress < stream->max_ios &&
441 stream->pinned_buffers + stream->pending_read_nblocks < stream->distance)
442 {
443 BlockNumber blocknum;
444 int16 buffer_index;
445 void *per_buffer_data;
446
447 if (stream->pending_read_nblocks == stream->io_combine_limit)
448 {
449 read_stream_start_pending_read(stream);
450 continue;
451 }
452
453 /*
454 * See which block the callback wants next in the stream. We need to
455 * compute the index of the Nth block of the pending read including
456 * wrap-around, but we don't want to use the expensive % operator.
457 */
458 buffer_index = stream->next_buffer_index + stream->pending_read_nblocks;
459 if (buffer_index >= stream->queue_size)
460 buffer_index -= stream->queue_size;
461 Assert(buffer_index >= 0 && buffer_index < stream->queue_size);
462 per_buffer_data = get_per_buffer_data(stream, buffer_index);
463 blocknum = read_stream_get_block(stream, per_buffer_data);
464 if (blocknum == InvalidBlockNumber)
465 {
466 /* End of stream. */
467 stream->distance = 0;
468 break;
469 }
470
471 /* Can we merge it with the pending read? */
472 if (stream->pending_read_nblocks > 0 &&
473 stream->pending_read_blocknum + stream->pending_read_nblocks == blocknum)
474 {
475 stream->pending_read_nblocks++;
476 continue;
477 }
478
479 /* We have to start the pending read before we can build another. */
480 while (stream->pending_read_nblocks > 0)
481 {
482 if (!read_stream_start_pending_read(stream) ||
483 stream->ios_in_progress == stream->max_ios)
484 {
485 /* We've hit the buffer or I/O limit. Rewind and stop here. */
486 read_stream_unget_block(stream, blocknum);
487 if (stream->batch_mode)
488 pgaio_exit_batchmode();
489 return;
490 }
491 }
492
493 /* This is the start of a new pending read. */
494 stream->pending_read_blocknum = blocknum;
495 stream->pending_read_nblocks = 1;
496 }
497
498 /*
499 * We don't start the pending read just because we've hit the distance
500 * limit, preferring to give it another chance to grow to full
501 * io_combine_limit size once more buffers have been consumed. However,
502 * if we've already reached io_combine_limit, or we've reached the
503 * distance limit and there isn't anything pinned yet, or the callback has
504 * signaled end-of-stream, we start the read immediately. Note that the
505 * pending read can exceed the distance goal, if the latter was reduced
506 * after hitting the per-backend buffer limit.
507 */
508 if (stream->pending_read_nblocks > 0 &&
509 (stream->pending_read_nblocks == stream->io_combine_limit ||
510 (stream->pending_read_nblocks >= stream->distance &&
511 stream->pinned_buffers == 0) ||
512 stream->distance == 0) &&
513 stream->ios_in_progress < stream->max_ios)
514 read_stream_start_pending_read(stream);
515
516 /*
517 * There should always be something pinned when we leave this function,
518 * whether started by this call or not, unless we've hit the end of the
519 * stream. In the worst case we can always make progress one buffer at a
520 * time.
521 */
522 Assert(stream->pinned_buffers > 0 || stream->distance == 0);
523
524 if (stream->batch_mode)
525 pgaio_exit_batchmode();
526 }
527
528 /*
529 * Create a new read stream object that can be used to perform the equivalent
530 * of a series of ReadBuffer() calls for one fork of one relation.
531 * Internally, it generates larger vectored reads where possible by looking
532 * ahead. The callback should return block numbers or InvalidBlockNumber to
533 * signal end-of-stream, and if per_buffer_data_size is non-zero, it may also
534 * write extra data for each block into the space provided to it. It will
535 * also receive callback_private_data for its own purposes.
536 */
537 static ReadStream *
538 read_stream_begin_impl(int flags,
539 BufferAccessStrategy strategy,
540 Relation rel,
541 SMgrRelation smgr,
542 char persistence,
543 ForkNumber forknum,
544 ReadStreamBlockNumberCB callback,
545 void *callback_private_data,
546 size_t per_buffer_data_size)
547 {
548 ReadStream *stream;
549 size_t size;
550 int16 queue_size;
551 int16 queue_overflow;
552 int max_ios;
553 int strategy_pin_limit;
554 uint32 max_pinned_buffers;
555 uint32 max_possible_buffer_limit;
556 Oid tablespace_id;
557
558 /*
559 * Reject attempts to read non-local temporary relations; we would be
560 * likely to get wrong data since we have no visibility into the owning
561 * session's local buffers.
562 */
563 if (rel && RELATION_IS_OTHER_TEMP(rel))
564 ereport(ERROR,
565 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
566 errmsg("cannot access temporary tables of other sessions")));
567
568 /*
569 * Decide how many I/Os we will allow to run at the same time. This
570 * number also affects how far we look ahead for opportunities to start
571 * more I/Os.
572 */
573 tablespace_id = smgr->smgr_rlocator.locator.spcOid;
574 if (!OidIsValid(MyDatabaseId) ||
575 (rel && IsCatalogRelation(rel)) ||
576 IsCatalogRelationOid(smgr->smgr_rlocator.locator.relNumber))
577 {
578 /*
579 * Avoid circularity while trying to look up tablespace settings or
580 * before spccache.c is ready.
581 */
582 max_ios = effective_io_concurrency;
583 }
584 else if (flags & READ_STREAM_MAINTENANCE)
585 max_ios = get_tablespace_maintenance_io_concurrency(tablespace_id);
586 else
587 max_ios = get_tablespace_io_concurrency(tablespace_id);
588
589 /* Cap to INT16_MAX to avoid overflowing below */
590 max_ios = Min(max_ios, PG_INT16_MAX);
591
592 /*
593 * If starting a multi-block I/O near the end of the queue, we might
594 * temporarily need extra space for overflowing buffers before they are
595 * moved to regular circular position. This is the maximum extra space we
596 * could need.
597 */
598 queue_overflow = io_combine_limit - 1;
599
600 /*
601 * Choose the maximum number of buffers we're prepared to pin. We try to
602 * pin fewer if we can, though. We add one so that we can make progress
603 * even if max_ios is set to 0 (see also further down). For max_ios > 0,
604 * this also allows an extra full I/O's worth of buffers: after an I/O
605 * finishes we don't want to have to wait for its buffers to be consumed
606 * before starting a new one.
607 *
608 * Be careful not to allow int16 to overflow. That is possible with the
609 * current GUC range limits, so this is an artificial limit of ~32k
610 * buffers and we'd need to adjust the types to exceed that. We also have
611 * to allow for the spare entry and the overflow space.
612 */
613 max_pinned_buffers = (max_ios + 1) * io_combine_limit;
614 max_pinned_buffers = Min(max_pinned_buffers,
615 PG_INT16_MAX - queue_overflow - 1);
616
617 /* Give the strategy a chance to limit the number of buffers we pin. */
618 strategy_pin_limit = GetAccessStrategyPinLimit(strategy);
619 max_pinned_buffers = Min(strategy_pin_limit, max_pinned_buffers);
620
621 /*
622 * Also limit our queue to the maximum number of pins we could ever be
623 * allowed to acquire according to the buffer manager. We may not really
624 * be able to use them all due to other pins held by this backend, but
625 * we'll check that later in read_stream_start_pending_read().
626 */
627 if (SmgrIsTemp(smgr))
628 max_possible_buffer_limit = GetLocalPinLimit();
629 else
630 max_possible_buffer_limit = GetPinLimit();
631 max_pinned_buffers = Min(max_pinned_buffers, max_possible_buffer_limit);
632
633 /*
634 * The limit might be zero on a system configured with too few buffers for
635 * the number of connections. We need at least one to make progress.
636 */
637 max_pinned_buffers = Max(1, max_pinned_buffers);
638
639 /*
640 * We need one extra entry for buffers and per-buffer data, because users
641 * of per-buffer data have access to the object until the next call to
642 * read_stream_next_buffer(), so we need a gap between the head and tail
643 * of the queue so that we don't clobber it.
644 */
645 queue_size = max_pinned_buffers + 1;
646
647 /*
648 * Allocate the object, the buffers, the ios and per_buffer_data space in
649 * one big chunk. Though we have queue_size buffers, we want to be able
650 * to assume that all the buffers for a single read are contiguous (i.e.
651 * don't wrap around halfway through), so we allow temporary overflows of
652 * up to the maximum possible overflow size.
653 */
654 size = offsetof(ReadStream, buffers);
655 size += sizeof(Buffer) * (queue_size + queue_overflow);
656 size += sizeof(InProgressIO) * Max(1, max_ios);
657 size += per_buffer_data_size * queue_size;
658 size += MAXIMUM_ALIGNOF * 2;
659 stream = (ReadStream *) palloc(size);
660 memset(stream, 0, offsetof(ReadStream, buffers));
661 stream->ios = (InProgressIO *)
662 MAXALIGN(&stream->buffers[queue_size + queue_overflow]);
663 if (per_buffer_data_size > 0)
664 stream->per_buffer_data = (void *)
665 MAXALIGN(&stream->ios[Max(1, max_ios)]);
666
667 stream->sync_mode = io_method == IOMETHOD_SYNC;
668 stream->batch_mode = flags & READ_STREAM_USE_BATCHING;
669
670 #ifdef USE_PREFETCH
671
672 /*
673 * Read-ahead advice simulating asynchronous I/O with synchronous calls.
674 * Issue advice only if AIO is not used, direct I/O isn't enabled, the
675 * caller hasn't promised sequential access (overriding our detection
676 * heuristics), and max_ios hasn't been set to zero.
677 */
678 if (stream->sync_mode &&
679 (io_direct_flags & IO_DIRECT_DATA) == 0 &&
680 (flags & READ_STREAM_SEQUENTIAL) == 0 &&
681 max_ios > 0)
682 stream->advice_enabled = true;
683 #endif
684
685 /*
686 * Setting max_ios to zero disables AIO and advice-based pseudo AIO, but
687 * we still need to allocate space to combine and run one I/O. Bump it up
688 * to one, and remember to ask for synchronous I/O only.
689 */
690 if (max_ios == 0)
691 {
692 max_ios = 1;
693 stream->read_buffers_flags = READ_BUFFERS_SYNCHRONOUSLY;
694 }
695
696 /*
697 * Capture stable values for these two GUC-derived numbers for the
698 * lifetime of this stream, so we don't have to worry about the GUCs
699 * changing underneath us beyond this point.
700 */
701 stream->max_ios = max_ios;
702 stream->io_combine_limit = io_combine_limit;
703
704 stream->per_buffer_data_size = per_buffer_data_size;
705 stream->max_pinned_buffers = max_pinned_buffers;
706 stream->queue_size = queue_size;
707 stream->callback = callback;
708 stream->callback_private_data = callback_private_data;
709 stream->buffered_blocknum = InvalidBlockNumber;
710 stream->seq_blocknum = InvalidBlockNumber;
711 stream->seq_until_processed = InvalidBlockNumber;
712 stream->temporary = SmgrIsTemp(smgr);
713
714 /*
715 * Skip the initial ramp-up phase if the caller says we're going to be
716 * reading the whole relation. This way we start out assuming we'll be
717 * doing full io_combine_limit sized reads.
718 */
719 if (flags & READ_STREAM_FULL)
720 stream->distance = Min(max_pinned_buffers, stream->io_combine_limit);
721 else
722 stream->distance = 1;
723
724 /*
725 * Since we always access the same relation, we can initialize parts of
726 * the ReadBuffersOperation objects and leave them that way, to avoid
727 * wasting CPU cycles writing to them for each read.
728 */
729 for (int i = 0; i < max_ios; ++i)
730 {
731 stream->ios[i].op.rel = rel;
732 stream->ios[i].op.smgr = smgr;
733 stream->ios[i].op.persistence = persistence;
734 stream->ios[i].op.forknum = forknum;
735 stream->ios[i].op.strategy = strategy;
736 }
737
738 return stream;
739 }
740
741 /*
742 * Create a new read stream for reading a relation.
743 * See read_stream_begin_impl() for the detailed explanation.
744 */
745 ReadStream *
746 read_stream_begin_relation(int flags,
747 BufferAccessStrategy strategy,
748 Relation rel,
749 ForkNumber forknum,
750 ReadStreamBlockNumberCB callback,
751 void *callback_private_data,
752 size_t per_buffer_data_size)
753 {
754 return read_stream_begin_impl(flags,
755 strategy,
756 rel,
757 RelationGetSmgr(rel),
758 rel->rd_rel->relpersistence,
759 forknum,
760 callback,
761 callback_private_data,
762 per_buffer_data_size);
763 }
764
765 /*
766 * Create a new read stream for reading a SMgr relation.
767 * See read_stream_begin_impl() for the detailed explanation.
768 */
769 ReadStream *
770 read_stream_begin_smgr_relation(int flags,
771 BufferAccessStrategy strategy,
772 SMgrRelation smgr,
773 char smgr_persistence,
774 ForkNumber forknum,
775 ReadStreamBlockNumberCB callback,
776 void *callback_private_data,
777 size_t per_buffer_data_size)
778 {
779 return read_stream_begin_impl(flags,
780 strategy,
781 NULL,
782 smgr,
783 smgr_persistence,
784 forknum,
785 callback,
786 callback_private_data,
787 per_buffer_data_size);
788 }
789
790 /*
791 * Pull one pinned buffer out of a stream. Each call returns successive
792 * blocks in the order specified by the callback. If per_buffer_data_size was
793 * set to a non-zero size, *per_buffer_data receives a pointer to the extra
794 * per-buffer data that the callback had a chance to populate, which remains
795 * valid until the next call to read_stream_next_buffer(). When the stream
796 * runs out of data, InvalidBuffer is returned. The caller may decide to end
797 * the stream early at any time by calling read_stream_end().
798 */
799 Buffer
800 read_stream_next_buffer(ReadStream *stream, void **per_buffer_data)
801 {
802 Buffer buffer;
803 int16 oldest_buffer_index;
804
805 #ifndef READ_STREAM_DISABLE_FAST_PATH
806
807 /*
808 * A fast path for all-cached scans. This is the same as the usual
809 * algorithm, but it is specialized for no I/O and no per-buffer data, so
810 * we can skip the queue management code, stay in the same buffer slot and
811 * use singular StartReadBuffer().
812 */
813 if (likely(stream->fast_path))
814 {
815 BlockNumber next_blocknum;
816
817 /* Fast path assumptions. */
818 Assert(stream->ios_in_progress == 0);
819 Assert(stream->forwarded_buffers == 0);
820 Assert(stream->pinned_buffers == 1);
821 Assert(stream->distance == 1);
822 Assert(stream->pending_read_nblocks == 0);
823 Assert(stream->per_buffer_data_size == 0);
824 Assert(stream->initialized_buffers > stream->oldest_buffer_index);
825
826 /* We're going to return the buffer we pinned last time. */
827 oldest_buffer_index = stream->oldest_buffer_index;
828 Assert((oldest_buffer_index + 1) % stream->queue_size ==
829 stream->next_buffer_index);
830 buffer = stream->buffers[oldest_buffer_index];
831 Assert(buffer != InvalidBuffer);
832
833 /* Choose the next block to pin. */
834 next_blocknum = read_stream_get_block(stream, NULL);
835
836 if (likely(next_blocknum != InvalidBlockNumber))
837 {
838 int flags = stream->read_buffers_flags;
839
840 if (stream->advice_enabled)
841 flags |= READ_BUFFERS_ISSUE_ADVICE;
842
843 /*
844 * Pin a buffer for the next call. Same buffer entry, and
845 * arbitrary I/O entry (they're all free). We don't have to
846 * adjust pinned_buffers because we're transferring one to caller
847 * but pinning one more.
848 *
849 * In the fast path we don't need to check the pin limit. We're
850 * always allowed at least one pin so that progress can be made,
851 * and that's all we need here. Although two pins are momentarily
852 * held at the same time, the model used here is that the stream
853 * holds only one, and the other now belongs to the caller.
854 */
855 if (likely(!StartReadBuffer(&stream->ios[0].op,
856 &stream->buffers[oldest_buffer_index],
857 next_blocknum,
858 flags)))
859 {
860 /* Fast return. */
861 return buffer;
862 }
863
864 /* Next call must wait for I/O for the newly pinned buffer. */
865 stream->oldest_io_index = 0;
866 stream->next_io_index = stream->max_ios > 1 ? 1 : 0;
867 stream->ios_in_progress = 1;
868 stream->ios[0].buffer_index = oldest_buffer_index;
869 stream->seq_blocknum = next_blocknum + 1;
870 }
871 else
872 {
873 /* No more blocks, end of stream. */
874 stream->distance = 0;
875 stream->oldest_buffer_index = stream->next_buffer_index;
876 stream->pinned_buffers = 0;
877 stream->buffers[oldest_buffer_index] = InvalidBuffer;
878 }
879
880 stream->fast_path = false;
881 return buffer;
882 }
883 #endif
884
885 if (unlikely(stream->pinned_buffers == 0))
886 {
887 Assert(stream->oldest_buffer_index == stream->next_buffer_index);
888
889 /* End of stream reached? */
890 if (stream->distance == 0)
891 return InvalidBuffer;
892
893 /*
894 * The usual order of operations is that we look ahead at the bottom
895 * of this function after potentially finishing an I/O and making
896 * space for more, but if we're just starting up we'll need to crank
897 * the handle to get started.
898 */
899 read_stream_look_ahead(stream);
900
901 /* End of stream reached? */
902 if (stream->pinned_buffers == 0)
903 {
904 Assert(stream->distance == 0);
905 return InvalidBuffer;
906 }
907 }
908
909 /* Grab the oldest pinned buffer and associated per-buffer data. */
910 Assert(stream->pinned_buffers > 0);
911 oldest_buffer_index = stream->oldest_buffer_index;
912 Assert(oldest_buffer_index >= 0 &&
913 oldest_buffer_index < stream->queue_size);
914 buffer = stream->buffers[oldest_buffer_index];
915 if (per_buffer_data)
916 *per_buffer_data = get_per_buffer_data(stream, oldest_buffer_index);
917
918 Assert(BufferIsValid(buffer));
919
920 /* Do we have to wait for an associated I/O first? */
921 if (stream->ios_in_progress > 0 &&
922 stream->ios[stream->oldest_io_index].buffer_index == oldest_buffer_index)
923 {
924 int16 io_index = stream->oldest_io_index;
925 int32 distance; /* wider temporary value, clamped below */
926
927 /* Sanity check that we still agree on the buffers. */
928 Assert(stream->ios[io_index].op.buffers ==
929 &stream->buffers[oldest_buffer_index]);
930
931 WaitReadBuffers(&stream->ios[io_index].op);
932
933 Assert(stream->ios_in_progress > 0);
934 stream->ios_in_progress--;
935 if (++stream->oldest_io_index == stream->max_ios)
936 stream->oldest_io_index = 0;
937
938 /* Look-ahead distance ramps up rapidly after we do I/O. */
939 distance = stream->distance * 2;
940 distance = Min(distance, stream->max_pinned_buffers);
941 stream->distance = distance;
942
943 /*
944 * If we've reached the first block of a sequential region we're
945 * issuing advice for, cancel that until the next jump. The kernel
946 * will see the sequential preadv() pattern starting here.
947 */
948 if (stream->advice_enabled &&
949 stream->ios[io_index].op.blocknum == stream->seq_until_processed)
950 stream->seq_until_processed = InvalidBlockNumber;
951 }
952
953 /*
954 * We must zap this queue entry, or else it would appear as a forwarded
955 * buffer. If it's potentially in the overflow zone (ie from a
956 * multi-block I/O that wrapped around the queue), also zap the copy.
957 */
958 stream->buffers[oldest_buffer_index] = InvalidBuffer;
959 if (oldest_buffer_index < stream->io_combine_limit - 1)
960 stream->buffers[stream->queue_size + oldest_buffer_index] =
961 InvalidBuffer;
962
963 #if defined(CLOBBER_FREED_MEMORY) || defined(USE_VALGRIND)
964
965 /*
966 * The caller will get access to the per-buffer data, until the next call.
967 * We wipe the one before, which is never occupied because queue_size
968 * allowed one extra element. This will hopefully trip up client code
969 * that is holding a dangling pointer to it.
970 */
971 if (stream->per_buffer_data)
972 {
973 void *per_buffer_data;
974
975 per_buffer_data = get_per_buffer_data(stream,
976 oldest_buffer_index == 0 ?
977 stream->queue_size - 1 :
978 oldest_buffer_index - 1);
979
980 #if defined(CLOBBER_FREED_MEMORY)
981 /* This also tells Valgrind the memory is "noaccess". */
982 wipe_mem(per_buffer_data, stream->per_buffer_data_size);
983 #elif defined(USE_VALGRIND)
984 /* Tell it ourselves. */
985 VALGRIND_MAKE_MEM_NOACCESS(per_buffer_data,
986 stream->per_buffer_data_size);
987 #endif
988 }
989 #endif
990
991 /* Pin transferred to caller. */
992 Assert(stream->pinned_buffers > 0);
993 stream->pinned_buffers--;
994
995 /* Advance oldest buffer, with wrap-around. */
996 stream->oldest_buffer_index++;
997 if (stream->oldest_buffer_index == stream->queue_size)
998 stream->oldest_buffer_index = 0;
999
1000 /* Prepare for the next call. */
1001 read_stream_look_ahead(stream);
1002
1003 #ifndef READ_STREAM_DISABLE_FAST_PATH
1004 /* See if we can take the fast path for all-cached scans next time. */
1005 if (stream->ios_in_progress == 0 &&
1006 stream->forwarded_buffers == 0 &&
1007 stream->pinned_buffers == 1 &&
1008 stream->distance == 1 &&
1009 stream->pending_read_nblocks == 0 &&
1010 stream->per_buffer_data_size == 0)
1011 {
1012 /*
1013 * The fast path spins on one buffer entry repeatedly instead of
1014 * rotating through the whole queue and clearing the entries behind
1015 * it. If the buffer it starts with happened to be forwarded between
1016 * StartReadBuffers() calls and also wrapped around the circular queue
1017 * partway through, then a copy also exists in the overflow zone, and
1018 * it won't clear it out as the regular path would. Do that now, so
1019 * it doesn't need code for that.
1020 */
1021 if (stream->oldest_buffer_index < stream->io_combine_limit - 1)
1022 stream->buffers[stream->queue_size + stream->oldest_buffer_index] =
1023 InvalidBuffer;
1024
1025 stream->fast_path = true;
1026 }
1027 #endif
1028
1029 return buffer;
1030 }
1031
1032 /*
1033 * Transitional support for code that would like to perform or skip reads
1034 * itself, without using the stream. Returns, and consumes, the next block
1035 * number that would be read by the stream's look-ahead algorithm, or
1036 * InvalidBlockNumber if the end of the stream is reached. Also reports the
1037 * strategy that would be used to read it.
1038 */
1039 BlockNumber
1040 read_stream_next_block(ReadStream *stream, BufferAccessStrategy *strategy)
1041 {
1042 *strategy = stream->ios[0].op.strategy;
1043 return read_stream_get_block(stream, NULL);
1044 }
1045
1046
1047 /*
1048 * Stop using a buffer access strategy for reads from this stream.
1049 *
1050 * This clears the strategy for all of the stream's ReadBuffersOperations,
1051 * including those with in-progress IOs. The completion of an IO whose
1052 * strategy was cleared while it was in flight may have a small amount of its
1053 * read time attributed to IOCONTEXT_NORMAL instead of the strategy's
1054 * IOContext, because WaitReadBuffers() derives the IOContext from the (now
1055 * cleared) strategy. This is bounded by the stream's look-ahead window and
1056 * happens at most once, when the strategy is first cleared, so it is not worth
1057 * the complexity of preserving the original IOContext for those IOs.
1058 *
1059 * Note that the caller is responsible for freeing the strategy's memory.
1060 */
1061 void
1062 read_stream_clear_strategy(ReadStream *stream)
1063 {
1064 for (int i = 0; i < stream->max_ios; ++i)
1065 stream->ios[i].op.strategy = NULL;
1066 }
1067
1068 /*
1069 * Reset a read stream by releasing any queued up buffers, allowing the stream
1070 * to be used again for different blocks. This can be used to clear an
1071 * end-of-stream condition and start again, or to throw away blocks that were
1072 * speculatively read and read some different blocks instead.
1073 */
1074 void
1075 read_stream_reset(ReadStream *stream)
1076 {
1077 int16 index;
1078 Buffer buffer;
1079
1080 /* Stop looking ahead. */
1081 stream->distance = 0;
1082
1083 /* Forget buffered block number and fast path state. */
1084 stream->buffered_blocknum = InvalidBlockNumber;
1085 stream->fast_path = false;
1086
1087 /* Unpin anything that wasn't consumed. */
1088 while ((buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer)
1089 ReleaseBuffer(buffer);
1090
1091 /* Unpin any unused forwarded buffers. */
1092 index = stream->next_buffer_index;
1093 while (index < stream->initialized_buffers &&
1094 (buffer = stream->buffers[index]) != InvalidBuffer)
1095 {
1096 Assert(stream->forwarded_buffers > 0);
1097 stream->forwarded_buffers--;
1098 ReleaseBuffer(buffer);
1099
1100 stream->buffers[index] = InvalidBuffer;
1101 if (index < stream->io_combine_limit - 1)
1102 stream->buffers[stream->queue_size + index] = InvalidBuffer;
1103
1104 if (++index == stream->queue_size)
1105 index = 0;
1106 }
1107
1108 Assert(stream->forwarded_buffers == 0);
1109 Assert(stream->pinned_buffers == 0);
1110 Assert(stream->ios_in_progress == 0);
1111
1112 /* Start off assuming data is cached. */
1113 stream->distance = 1;
1114 }
1115
1116 /*
1117 * Release and free a read stream.
1118 */
1119 void
1120 read_stream_end(ReadStream *stream)
1121 {
1122 read_stream_reset(stream);
1123 pfree(stream);
1124 }