| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* read_stream.c |
| 4 |
* Mechanism for accessing buffered relation data with look-ahead |
| 5 |
* |
| 6 |
* Code that needs to access relation data typically pins blocks one at a |
| 7 |
* time, often in a predictable order that might be sequential or data-driven. |
| 8 |
* Calling the simple ReadBuffer() function for each block is inefficient, |
| 9 |
* because blocks that are not yet in the buffer pool require I/O operations |
| 10 |
* that are small and might stall waiting for storage. This mechanism looks |
| 11 |
* into the future and calls StartReadBuffers() and WaitReadBuffers() to read |
| 12 |
* neighboring blocks together and ahead of time, with an adaptive look-ahead |
| 13 |
* distance. |
| 14 |
* |
| 15 |
* A user-provided callback generates a stream of block numbers that is used |
| 16 |
* to form reads of up to io_combine_limit, by attempting to merge them with a |
| 17 |
* pending read. When that isn't possible, the existing pending read is sent |
| 18 |
* to StartReadBuffers() so that a new one can begin to form. |
| 19 |
* |
| 20 |
* The algorithm for controlling the look-ahead distance is based on recent |
| 21 |
* cache hit and miss history. When no I/O is necessary, there is no benefit |
| 22 |
* in looking ahead more than one block. This is the default initial |
| 23 |
* assumption, but when blocks needing I/O are streamed, the distance is |
| 24 |
* increased rapidly to try to benefit from I/O combining and concurrency. It |
| 25 |
* is reduced gradually when cached blocks are streamed. |
| 26 |
* |
| 27 |
* The main data structure is a circular queue of buffers of size |
| 28 |
* max_pinned_buffers plus some extra space for technical reasons, ready to be |
| 29 |
* returned by read_stream_next_buffer(). Each buffer also has an optional |
| 30 |
* variable sized object that is passed from the callback to the consumer of |
| 31 |
* buffers. |
| 32 |
* |
| 33 |
* Parallel to the queue of buffers, there is a circular queue of in-progress |
| 34 |
* I/Os that have been started with StartReadBuffers(), and for which |
| 35 |
* WaitReadBuffers() must be called before returning the buffer. |
| 36 |
* |
| 37 |
* For example, if the callback returns block numbers 10, 42, 43, 44, 60 in |
| 38 |
* successive calls, then these data structures might appear as follows: |
| 39 |
* |
| 40 |
* buffers buf/data ios |
| 41 |
* |
| 42 |
* +----+ +-----+ +--------+ |
| 43 |
* | | | | +----+ 42..44 | <- oldest_io_index |
| 44 |
* +----+ +-----+ | +--------+ |
| 45 |
* oldest_buffer_index -> | 10 | | ? | | +--+ 60..60 | |
| 46 |
* +----+ +-----+ | | +--------+ |
| 47 |
* | 42 | | ? |<-+ | | | <- next_io_index |
| 48 |
* +----+ +-----+ | +--------+ |
| 49 |
* | 43 | | ? | | | | |
| 50 |
* +----+ +-----+ | +--------+ |
| 51 |
* | 44 | | ? | | | | |
| 52 |
* +----+ +-----+ | +--------+ |
| 53 |
* | 60 | | ? |<---+ |
| 54 |
* +----+ +-----+ |
| 55 |
* next_buffer_index -> | | | | |
| 56 |
* +----+ +-----+ |
| 57 |
* |
| 58 |
* In the example, 5 buffers are pinned, and the next buffer to be streamed to |
| 59 |
* the client is block 10. Block 10 was a hit and has no associated I/O, but |
| 60 |
* the range 42..44 requires an I/O wait before its buffers are returned, as |
| 61 |
* does block 60. |
| 62 |
* |
| 63 |
* |
| 64 |
* Portions Copyright (c) 2024-2025, PostgreSQL Global Development Group |
| 65 |
* Portions Copyright (c) 1994, Regents of the University of California |
| 66 |
* |
| 67 |
* IDENTIFICATION |
| 68 |
* src/backend/storage/aio/read_stream.c |
| 69 |
* |
| 70 |
*------------------------------------------------------------------------- |
| 71 |
*/ |
| 72 |
#include "postgres.h" |
| 73 |
|
| 74 |
#include "miscadmin.h" |
| 75 |
#include "storage/aio.h" |
| 76 |
#include "storage/fd.h" |
| 77 |
#include "storage/smgr.h" |
| 78 |
#include "storage/read_stream.h" |
| 79 |
#include "utils/memdebug.h" |
| 80 |
#include "utils/rel.h" |
| 81 |
#include "utils/spccache.h" |
| 82 |
|
| 83 |
typedef struct InProgressIO |
| 84 |
{ |
| 85 |
int16 buffer_index; |
| 86 |
ReadBuffersOperation op; |
| 87 |
} InProgressIO; |
| 88 |
|
| 89 |
/* |
| 90 |
* State for managing a stream of reads. |
| 91 |
*/ |
| 92 |
struct ReadStream |
| 93 |
{ |
| 94 |
int16 max_ios; |
| 95 |
int16 io_combine_limit; |
| 96 |
int16 ios_in_progress; |
| 97 |
int16 queue_size; |
| 98 |
int16 max_pinned_buffers; |
| 99 |
int16 forwarded_buffers; |
| 100 |
int16 pinned_buffers; |
| 101 |
int16 distance; |
| 102 |
int16 initialized_buffers; |
| 103 |
int read_buffers_flags; |
| 104 |
bool sync_mode; /* using io_method=sync */ |
| 105 |
bool batch_mode; /* READ_STREAM_USE_BATCHING */ |
| 106 |
bool advice_enabled; |
| 107 |
bool temporary; |
| 108 |
|
| 109 |
/* |
| 110 |
* One-block buffer to support 'ungetting' a block number, to resolve flow |
| 111 |
* control problems when I/Os are split. |
| 112 |
*/ |
| 113 |
BlockNumber buffered_blocknum; |
| 114 |
|
| 115 |
/* |
| 116 |
* The callback that will tell us which block numbers to read, and an |
| 117 |
* opaque pointer that will be pass to it for its own purposes. |
| 118 |
*/ |
| 119 |
ReadStreamBlockNumberCB callback; |
| 120 |
void *callback_private_data; |
| 121 |
|
| 122 |
/* Next expected block, for detecting sequential access. */ |
| 123 |
BlockNumber seq_blocknum; |
| 124 |
BlockNumber seq_until_processed; |
| 125 |
|
| 126 |
/* The read operation we are currently preparing. */ |
| 127 |
BlockNumber pending_read_blocknum; |
| 128 |
int16 pending_read_nblocks; |
| 129 |
|
| 130 |
/* Space for buffers and optional per-buffer private data. */ |
| 131 |
size_t per_buffer_data_size; |
| 132 |
void *per_buffer_data; |
| 133 |
|
| 134 |
/* Read operations that have been started but not waited for yet. */ |
| 135 |
InProgressIO *ios; |
| 136 |
int16 oldest_io_index; |
| 137 |
int16 next_io_index; |
| 138 |
|
| 139 |
bool fast_path; |
| 140 |
|
| 141 |
/* Circular queue of buffers. */ |
| 142 |
int16 oldest_buffer_index; /* Next pinned buffer to return */ |
| 143 |
int16 next_buffer_index; /* Index of next buffer to pin */ |
| 144 |
Buffer buffers[FLEXIBLE_ARRAY_MEMBER]; |
| 145 |
}; |
| 146 |
|
| 147 |
/* |
| 148 |
* Return a pointer to the per-buffer data by index. |
| 149 |
*/ |
| 150 |
static inline void * |
| 151 |
get_per_buffer_data(ReadStream *stream, int16 buffer_index) |
| 152 |
{ |
| 153 |
return (char *) stream->per_buffer_data + |
| 154 |
stream->per_buffer_data_size * buffer_index; |
| 155 |
} |
| 156 |
|
| 157 |
/* |
| 158 |
* General-use ReadStreamBlockNumberCB for block range scans. Loops over the |
| 159 |
* blocks [current_blocknum, last_exclusive). |
| 160 |
*/ |
| 161 |
BlockNumber |
| 162 |
block_range_read_stream_cb(ReadStream *stream, |
| 163 |
void *callback_private_data, |
| 164 |
void *per_buffer_data) |
| 165 |
{ |
| 166 |
BlockRangeReadStreamPrivate *p = callback_private_data; |
| 167 |
|
| 168 |
if (p->current_blocknum < p->last_exclusive) |
| 169 |
return p->current_blocknum++; |
| 170 |
|
| 171 |
return InvalidBlockNumber; |
| 172 |
} |
| 173 |
|
| 174 |
/* |
| 175 |
* Ask the callback which block it would like us to read next, with a one block |
| 176 |
* buffer in front to allow read_stream_unget_block() to work. |
| 177 |
*/ |
| 178 |
static inline BlockNumber |
| 179 |
read_stream_get_block(ReadStream *stream, void *per_buffer_data) |
| 180 |
{ |
| 181 |
BlockNumber blocknum; |
| 182 |
|
| 183 |
blocknum = stream->buffered_blocknum; |
| 184 |
if (blocknum != InvalidBlockNumber) |
| 185 |
stream->buffered_blocknum = InvalidBlockNumber; |
| 186 |
else |
| 187 |
{ |
| 188 |
/* |
| 189 |
* Tell Valgrind that the per-buffer data is undefined. That replaces |
| 190 |
* the "noaccess" state that was set when the consumer moved past this |
| 191 |
* entry last time around the queue, and should also catch callbacks |
| 192 |
* that fail to initialize data that the buffer consumer later |
| 193 |
* accesses. On the first go around, it is undefined already. |
| 194 |
*/ |
| 195 |
VALGRIND_MAKE_MEM_UNDEFINED(per_buffer_data, |
| 196 |
stream->per_buffer_data_size); |
| 197 |
blocknum = stream->callback(stream, |
| 198 |
stream->callback_private_data, |
| 199 |
per_buffer_data); |
| 200 |
} |
| 201 |
|
| 202 |
return blocknum; |
| 203 |
} |
| 204 |
|
| 205 |
/* |
| 206 |
* In order to deal with buffer shortages and I/O limits after short reads, we |
| 207 |
* sometimes need to defer handling of a block we've already consumed from the |
| 208 |
* registered callback until later. |
| 209 |
*/ |
| 210 |
static inline void |
| 211 |
read_stream_unget_block(ReadStream *stream, BlockNumber blocknum) |
| 212 |
{ |
| 213 |
/* We shouldn't ever unget more than one block. */ |
| 214 |
Assert(stream->buffered_blocknum == InvalidBlockNumber); |
| 215 |
Assert(blocknum != InvalidBlockNumber); |
| 216 |
stream->buffered_blocknum = blocknum; |
| 217 |
} |
| 218 |
|
| 219 |
/* |
| 220 |
* Start as much of the current pending read as we can. If we have to split it |
| 221 |
* because of the per-backend buffer limit, or the buffer manager decides to |
| 222 |
* split it, then the pending read is adjusted to hold the remaining portion. |
| 223 |
* |
| 224 |
* We can always start a read of at least size one if we have no progress yet. |
| 225 |
* Otherwise it's possible that we can't start a read at all because of a lack |
| 226 |
* of buffers, and then false is returned. Buffer shortages also reduce the |
| 227 |
* distance to a level that prevents look-ahead until buffers are released. |
| 228 |
*/ |
| 229 |
static bool |
| 230 |
read_stream_start_pending_read(ReadStream *stream) |
| 231 |
{ |
| 232 |
bool need_wait; |
| 233 |
int requested_nblocks; |
| 234 |
int nblocks; |
| 235 |
int flags; |
| 236 |
int forwarded; |
| 237 |
int16 io_index; |
| 238 |
int16 overflow; |
| 239 |
int16 buffer_index; |
| 240 |
int buffer_limit; |
| 241 |
|
| 242 |
/* This should only be called with a pending read. */ |
| 243 |
Assert(stream->pending_read_nblocks > 0); |
| 244 |
Assert(stream->pending_read_nblocks <= stream->io_combine_limit); |
| 245 |
|
| 246 |
/* We had better not exceed the per-stream buffer limit with this read. */ |
| 247 |
Assert(stream->pinned_buffers + stream->pending_read_nblocks <= |
| 248 |
stream->max_pinned_buffers); |
| 249 |
|
| 250 |
#ifdef USE_ASSERT_CHECKING |
| 251 |
/* We had better not be overwriting an existing pinned buffer. */ |
| 252 |
if (stream->pinned_buffers > 0) |
| 253 |
Assert(stream->next_buffer_index != stream->oldest_buffer_index); |
| 254 |
else |
| 255 |
Assert(stream->next_buffer_index == stream->oldest_buffer_index); |
| 256 |
|
| 257 |
/* |
| 258 |
* Pinned buffers forwarded by a preceding StartReadBuffers() call that |
| 259 |
* had to split the operation should match the leading blocks of this |
| 260 |
* following StartReadBuffers() call. |
| 261 |
*/ |
| 262 |
Assert(stream->forwarded_buffers <= stream->pending_read_nblocks); |
| 263 |
for (int i = 0; i < stream->forwarded_buffers; ++i) |
| 264 |
Assert(BufferGetBlockNumber(stream->buffers[stream->next_buffer_index + i]) == |
| 265 |
stream->pending_read_blocknum + i); |
| 266 |
|
| 267 |
/* |
| 268 |
* Check that we've cleared the queue/overflow entries corresponding to |
| 269 |
* the rest of the blocks covered by this read, unless it's the first go |
| 270 |
* around and we haven't even initialized them yet. |
| 271 |
*/ |
| 272 |
for (int i = stream->forwarded_buffers; i < stream->pending_read_nblocks; ++i) |
| 273 |
Assert(stream->next_buffer_index + i >= stream->initialized_buffers || |
| 274 |
stream->buffers[stream->next_buffer_index + i] == InvalidBuffer); |
| 275 |
#endif |
| 276 |
|
| 277 |
/* Do we need to issue read-ahead advice? */ |
| 278 |
flags = stream->read_buffers_flags; |
| 279 |
if (stream->advice_enabled) |
| 280 |
{ |
| 281 |
if (stream->pending_read_blocknum == stream->seq_blocknum) |
| 282 |
{ |
| 283 |
/* |
| 284 |
* Sequential: Issue advice until the preadv() calls have caught |
| 285 |
* up with the first advice issued for this sequential region, and |
| 286 |
* then stay of the way of the kernel's own read-ahead. |
| 287 |
*/ |
| 288 |
if (stream->seq_until_processed != InvalidBlockNumber) |
| 289 |
flags |= READ_BUFFERS_ISSUE_ADVICE; |
| 290 |
} |
| 291 |
else |
| 292 |
{ |
| 293 |
/* |
| 294 |
* Random jump: Note the starting location of a new potential |
| 295 |
* sequential region and start issuing advice. Skip it this time |
| 296 |
* if the preadv() follows immediately, eg first block in stream. |
| 297 |
*/ |
| 298 |
stream->seq_until_processed = stream->pending_read_blocknum; |
| 299 |
if (stream->pinned_buffers > 0) |
| 300 |
flags |= READ_BUFFERS_ISSUE_ADVICE; |
| 301 |
} |
| 302 |
} |
| 303 |
|
| 304 |
/* |
| 305 |
* How many more buffers is this backend allowed? |
| 306 |
* |
| 307 |
* Forwarded buffers are already pinned and map to the leading blocks of |
| 308 |
* the pending read (the remaining portion of an earlier short read that |
| 309 |
* we're about to continue). They are not counted in pinned_buffers, but |
| 310 |
* they are counted as pins already held by this backend according to the |
| 311 |
* buffer manager, so they must be added to the limit it grants us. |
| 312 |
*/ |
| 313 |
if (stream->temporary) |
| 314 |
buffer_limit = Min(GetAdditionalLocalPinLimit(), PG_INT16_MAX); |
| 315 |
else |
| 316 |
buffer_limit = Min(GetAdditionalPinLimit(), PG_INT16_MAX); |
| 317 |
Assert(stream->forwarded_buffers <= stream->pending_read_nblocks); |
| 318 |
|
| 319 |
buffer_limit += stream->forwarded_buffers; |
| 320 |
buffer_limit = Min(buffer_limit, PG_INT16_MAX); |
| 321 |
|
| 322 |
if (buffer_limit == 0 && stream->pinned_buffers == 0) |
| 323 |
buffer_limit = 1; /* guarantee progress */ |
| 324 |
|
| 325 |
/* Does the per-backend limit affect this read? */ |
| 326 |
nblocks = stream->pending_read_nblocks; |
| 327 |
if (buffer_limit < nblocks) |
| 328 |
{ |
| 329 |
int16 new_distance; |
| 330 |
|
| 331 |
/* Shrink distance: no more look-ahead until buffers are released. */ |
| 332 |
new_distance = stream->pinned_buffers + buffer_limit; |
| 333 |
if (stream->distance > new_distance) |
| 334 |
stream->distance = new_distance; |
| 335 |
|
| 336 |
/* Unless we have nothing to give the consumer, stop here. */ |
| 337 |
if (stream->pinned_buffers > 0) |
| 338 |
return false; |
| 339 |
|
| 340 |
/* A short read is required to make progress. */ |
| 341 |
nblocks = buffer_limit; |
| 342 |
} |
| 343 |
|
| 344 |
/* |
| 345 |
* We say how many blocks we want to read, but it may be smaller on return |
| 346 |
* if the buffer manager decides to shorten the read. Initialize buffers |
| 347 |
* to InvalidBuffer (= not a forwarded buffer) as input on first use only, |
| 348 |
* and keep the original nblocks number so we can check for forwarded |
| 349 |
* buffers as output, below. |
| 350 |
*/ |
| 351 |
buffer_index = stream->next_buffer_index; |
| 352 |
io_index = stream->next_io_index; |
| 353 |
while (stream->initialized_buffers < buffer_index + nblocks) |
| 354 |
stream->buffers[stream->initialized_buffers++] = InvalidBuffer; |
| 355 |
requested_nblocks = nblocks; |
| 356 |
need_wait = StartReadBuffers(&stream->ios[io_index].op, |
| 357 |
&stream->buffers[buffer_index], |
| 358 |
stream->pending_read_blocknum, |
| 359 |
&nblocks, |
| 360 |
flags); |
| 361 |
stream->pinned_buffers += nblocks; |
| 362 |
|
| 363 |
/* Remember whether we need to wait before returning this buffer. */ |
| 364 |
if (!need_wait) |
| 365 |
{ |
| 366 |
/* Look-ahead distance decays, no I/O necessary. */ |
| 367 |
if (stream->distance > 1) |
| 368 |
stream->distance--; |
| 369 |
} |
| 370 |
else |
| 371 |
{ |
| 372 |
/* |
| 373 |
* Remember to call WaitReadBuffers() before returning head buffer. |
| 374 |
* Look-ahead distance will be adjusted after waiting. |
| 375 |
*/ |
| 376 |
stream->ios[io_index].buffer_index = buffer_index; |
| 377 |
if (++stream->next_io_index == stream->max_ios) |
| 378 |
stream->next_io_index = 0; |
| 379 |
Assert(stream->ios_in_progress < stream->max_ios); |
| 380 |
stream->ios_in_progress++; |
| 381 |
stream->seq_blocknum = stream->pending_read_blocknum + nblocks; |
| 382 |
} |
| 383 |
|
| 384 |
/* |
| 385 |
* How many pins were acquired but forwarded to the next call? These need |
| 386 |
* to be passed to the next StartReadBuffers() call by leaving them |
| 387 |
* exactly where they are in the queue, or released if the stream ends |
| 388 |
* early. We need the number for accounting purposes, since they are not |
| 389 |
* counted in stream->pinned_buffers but we already hold them. |
| 390 |
*/ |
| 391 |
forwarded = 0; |
| 392 |
while (nblocks + forwarded < requested_nblocks && |
| 393 |
stream->buffers[buffer_index + nblocks + forwarded] != InvalidBuffer) |
| 394 |
forwarded++; |
| 395 |
stream->forwarded_buffers = forwarded; |
| 396 |
|
| 397 |
/* |
| 398 |
* We gave a contiguous range of buffer space to StartReadBuffers(), but |
| 399 |
* we want it to wrap around at queue_size. Copy overflowing buffers to |
| 400 |
* the front of the array where they'll be consumed, but also leave a copy |
| 401 |
* in the overflow zone which the I/O operation has a pointer to (it needs |
| 402 |
* a contiguous array). Both copies will be cleared when the buffers are |
| 403 |
* handed to the consumer. |
| 404 |
*/ |
| 405 |
overflow = (buffer_index + nblocks + forwarded) - stream->queue_size; |
| 406 |
if (overflow > 0) |
| 407 |
{ |
| 408 |
Assert(overflow < stream->queue_size); /* can't overlap */ |
| 409 |
memcpy(&stream->buffers[0], |
| 410 |
&stream->buffers[stream->queue_size], |
| 411 |
sizeof(stream->buffers[0]) * overflow); |
| 412 |
} |
| 413 |
|
| 414 |
/* Compute location of start of next read, without using % operator. */ |
| 415 |
buffer_index += nblocks; |
| 416 |
if (buffer_index >= stream->queue_size) |
| 417 |
buffer_index -= stream->queue_size; |
| 418 |
Assert(buffer_index >= 0 && buffer_index < stream->queue_size); |
| 419 |
stream->next_buffer_index = buffer_index; |
| 420 |
|
| 421 |
/* Adjust the pending read to cover the remaining portion, if any. */ |
| 422 |
stream->pending_read_blocknum += nblocks; |
| 423 |
stream->pending_read_nblocks -= nblocks; |
| 424 |
|
| 425 |
return true; |
| 426 |
} |
| 427 |
|
| 428 |
static void |
| 429 |
read_stream_look_ahead(ReadStream *stream) |
| 430 |
{ |
| 431 |
/* |
| 432 |
* Allow amortizing the cost of submitting IO over multiple IOs. This |
| 433 |
* requires that we don't do any operations that could lead to a deadlock |
| 434 |
* with staged-but-unsubmitted IO. The callback needs to opt-in to being |
| 435 |
* careful. |
| 436 |
*/ |
| 437 |
if (stream->batch_mode) |
| 438 |
pgaio_enter_batchmode(); |
| 439 |
|
| 440 |
while (stream->ios_in_progress < stream->max_ios && |
| 441 |
stream->pinned_buffers + stream->pending_read_nblocks < stream->distance) |
| 442 |
{ |
| 443 |
BlockNumber blocknum; |
| 444 |
int16 buffer_index; |
| 445 |
void *per_buffer_data; |
| 446 |
|
| 447 |
if (stream->pending_read_nblocks == stream->io_combine_limit) |
| 448 |
{ |
| 449 |
read_stream_start_pending_read(stream); |
| 450 |
continue; |
| 451 |
} |
| 452 |
|
| 453 |
/* |
| 454 |
* See which block the callback wants next in the stream. We need to |
| 455 |
* compute the index of the Nth block of the pending read including |
| 456 |
* wrap-around, but we don't want to use the expensive % operator. |
| 457 |
*/ |
| 458 |
buffer_index = stream->next_buffer_index + stream->pending_read_nblocks; |
| 459 |
if (buffer_index >= stream->queue_size) |
| 460 |
buffer_index -= stream->queue_size; |
| 461 |
Assert(buffer_index >= 0 && buffer_index < stream->queue_size); |
| 462 |
per_buffer_data = get_per_buffer_data(stream, buffer_index); |
| 463 |
blocknum = read_stream_get_block(stream, per_buffer_data); |
| 464 |
if (blocknum == InvalidBlockNumber) |
| 465 |
{ |
| 466 |
/* End of stream. */ |
| 467 |
stream->distance = 0; |
| 468 |
break; |
| 469 |
} |
| 470 |
|
| 471 |
/* Can we merge it with the pending read? */ |
| 472 |
if (stream->pending_read_nblocks > 0 && |
| 473 |
stream->pending_read_blocknum + stream->pending_read_nblocks == blocknum) |
| 474 |
{ |
| 475 |
stream->pending_read_nblocks++; |
| 476 |
continue; |
| 477 |
} |
| 478 |
|
| 479 |
/* We have to start the pending read before we can build another. */ |
| 480 |
while (stream->pending_read_nblocks > 0) |
| 481 |
{ |
| 482 |
if (!read_stream_start_pending_read(stream) || |
| 483 |
stream->ios_in_progress == stream->max_ios) |
| 484 |
{ |
| 485 |
/* We've hit the buffer or I/O limit. Rewind and stop here. */ |
| 486 |
read_stream_unget_block(stream, blocknum); |
| 487 |
if (stream->batch_mode) |
| 488 |
pgaio_exit_batchmode(); |
| 489 |
return; |
| 490 |
} |
| 491 |
} |
| 492 |
|
| 493 |
/* This is the start of a new pending read. */ |
| 494 |
stream->pending_read_blocknum = blocknum; |
| 495 |
stream->pending_read_nblocks = 1; |
| 496 |
} |
| 497 |
|
| 498 |
/* |
| 499 |
* We don't start the pending read just because we've hit the distance |
| 500 |
* limit, preferring to give it another chance to grow to full |
| 501 |
* io_combine_limit size once more buffers have been consumed. However, |
| 502 |
* if we've already reached io_combine_limit, or we've reached the |
| 503 |
* distance limit and there isn't anything pinned yet, or the callback has |
| 504 |
* signaled end-of-stream, we start the read immediately. Note that the |
| 505 |
* pending read can exceed the distance goal, if the latter was reduced |
| 506 |
* after hitting the per-backend buffer limit. |
| 507 |
*/ |
| 508 |
if (stream->pending_read_nblocks > 0 && |
| 509 |
(stream->pending_read_nblocks == stream->io_combine_limit || |
| 510 |
(stream->pending_read_nblocks >= stream->distance && |
| 511 |
stream->pinned_buffers == 0) || |
| 512 |
stream->distance == 0) && |
| 513 |
stream->ios_in_progress < stream->max_ios) |
| 514 |
read_stream_start_pending_read(stream); |
| 515 |
|
| 516 |
/* |
| 517 |
* There should always be something pinned when we leave this function, |
| 518 |
* whether started by this call or not, unless we've hit the end of the |
| 519 |
* stream. In the worst case we can always make progress one buffer at a |
| 520 |
* time. |
| 521 |
*/ |
| 522 |
Assert(stream->pinned_buffers > 0 || stream->distance == 0); |
| 523 |
|
| 524 |
if (stream->batch_mode) |
| 525 |
pgaio_exit_batchmode(); |
| 526 |
} |
| 527 |
|
| 528 |
/* |
| 529 |
* Create a new read stream object that can be used to perform the equivalent |
| 530 |
* of a series of ReadBuffer() calls for one fork of one relation. |
| 531 |
* Internally, it generates larger vectored reads where possible by looking |
| 532 |
* ahead. The callback should return block numbers or InvalidBlockNumber to |
| 533 |
* signal end-of-stream, and if per_buffer_data_size is non-zero, it may also |
| 534 |
* write extra data for each block into the space provided to it. It will |
| 535 |
* also receive callback_private_data for its own purposes. |
| 536 |
*/ |
| 537 |
static ReadStream * |
| 538 |
read_stream_begin_impl(int flags, |
| 539 |
BufferAccessStrategy strategy, |
| 540 |
Relation rel, |
| 541 |
SMgrRelation smgr, |
| 542 |
char persistence, |
| 543 |
ForkNumber forknum, |
| 544 |
ReadStreamBlockNumberCB callback, |
| 545 |
void *callback_private_data, |
| 546 |
size_t per_buffer_data_size) |
| 547 |
{ |
| 548 |
ReadStream *stream; |
| 549 |
size_t size; |
| 550 |
int16 queue_size; |
| 551 |
int16 queue_overflow; |
| 552 |
int max_ios; |
| 553 |
int strategy_pin_limit; |
| 554 |
uint32 max_pinned_buffers; |
| 555 |
uint32 max_possible_buffer_limit; |
| 556 |
Oid tablespace_id; |
| 557 |
|
| 558 |
/* |
| 559 |
* Reject attempts to read non-local temporary relations; we would be |
| 560 |
* likely to get wrong data since we have no visibility into the owning |
| 561 |
* session's local buffers. |
| 562 |
*/ |
| 563 |
if (rel && RELATION_IS_OTHER_TEMP(rel)) |
| 564 |
ereport(ERROR, |
| 565 |
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
| 566 |
errmsg("cannot access temporary tables of other sessions"))); |
| 567 |
|
| 568 |
/* |
| 569 |
* Decide how many I/Os we will allow to run at the same time. This |
| 570 |
* number also affects how far we look ahead for opportunities to start |
| 571 |
* more I/Os. |
| 572 |
*/ |
| 573 |
tablespace_id = smgr->smgr_rlocator.locator.spcOid; |
| 574 |
if (!OidIsValid(MyDatabaseId) || |
| 575 |
(rel && IsCatalogRelation(rel)) || |
| 576 |
IsCatalogRelationOid(smgr->smgr_rlocator.locator.relNumber)) |
| 577 |
{ |
| 578 |
/* |
| 579 |
* Avoid circularity while trying to look up tablespace settings or |
| 580 |
* before spccache.c is ready. |
| 581 |
*/ |
| 582 |
max_ios = effective_io_concurrency; |
| 583 |
} |
| 584 |
else if (flags & READ_STREAM_MAINTENANCE) |
| 585 |
max_ios = get_tablespace_maintenance_io_concurrency(tablespace_id); |
| 586 |
else |
| 587 |
max_ios = get_tablespace_io_concurrency(tablespace_id); |
| 588 |
|
| 589 |
/* Cap to INT16_MAX to avoid overflowing below */ |
| 590 |
max_ios = Min(max_ios, PG_INT16_MAX); |
| 591 |
|
| 592 |
/* |
| 593 |
* If starting a multi-block I/O near the end of the queue, we might |
| 594 |
* temporarily need extra space for overflowing buffers before they are |
| 595 |
* moved to regular circular position. This is the maximum extra space we |
| 596 |
* could need. |
| 597 |
*/ |
| 598 |
queue_overflow = io_combine_limit - 1; |
| 599 |
|
| 600 |
/* |
| 601 |
* Choose the maximum number of buffers we're prepared to pin. We try to |
| 602 |
* pin fewer if we can, though. We add one so that we can make progress |
| 603 |
* even if max_ios is set to 0 (see also further down). For max_ios > 0, |
| 604 |
* this also allows an extra full I/O's worth of buffers: after an I/O |
| 605 |
* finishes we don't want to have to wait for its buffers to be consumed |
| 606 |
* before starting a new one. |
| 607 |
* |
| 608 |
* Be careful not to allow int16 to overflow. That is possible with the |
| 609 |
* current GUC range limits, so this is an artificial limit of ~32k |
| 610 |
* buffers and we'd need to adjust the types to exceed that. We also have |
| 611 |
* to allow for the spare entry and the overflow space. |
| 612 |
*/ |
| 613 |
max_pinned_buffers = (max_ios + 1) * io_combine_limit; |
| 614 |
max_pinned_buffers = Min(max_pinned_buffers, |
| 615 |
PG_INT16_MAX - queue_overflow - 1); |
| 616 |
|
| 617 |
/* Give the strategy a chance to limit the number of buffers we pin. */ |
| 618 |
strategy_pin_limit = GetAccessStrategyPinLimit(strategy); |
| 619 |
max_pinned_buffers = Min(strategy_pin_limit, max_pinned_buffers); |
| 620 |
|
| 621 |
/* |
| 622 |
* Also limit our queue to the maximum number of pins we could ever be |
| 623 |
* allowed to acquire according to the buffer manager. We may not really |
| 624 |
* be able to use them all due to other pins held by this backend, but |
| 625 |
* we'll check that later in read_stream_start_pending_read(). |
| 626 |
*/ |
| 627 |
if (SmgrIsTemp(smgr)) |
| 628 |
max_possible_buffer_limit = GetLocalPinLimit(); |
| 629 |
else |
| 630 |
max_possible_buffer_limit = GetPinLimit(); |
| 631 |
max_pinned_buffers = Min(max_pinned_buffers, max_possible_buffer_limit); |
| 632 |
|
| 633 |
/* |
| 634 |
* The limit might be zero on a system configured with too few buffers for |
| 635 |
* the number of connections. We need at least one to make progress. |
| 636 |
*/ |
| 637 |
max_pinned_buffers = Max(1, max_pinned_buffers); |
| 638 |
|
| 639 |
/* |
| 640 |
* We need one extra entry for buffers and per-buffer data, because users |
| 641 |
* of per-buffer data have access to the object until the next call to |
| 642 |
* read_stream_next_buffer(), so we need a gap between the head and tail |
| 643 |
* of the queue so that we don't clobber it. |
| 644 |
*/ |
| 645 |
queue_size = max_pinned_buffers + 1; |
| 646 |
|
| 647 |
/* |
| 648 |
* Allocate the object, the buffers, the ios and per_buffer_data space in |
| 649 |
* one big chunk. Though we have queue_size buffers, we want to be able |
| 650 |
* to assume that all the buffers for a single read are contiguous (i.e. |
| 651 |
* don't wrap around halfway through), so we allow temporary overflows of |
| 652 |
* up to the maximum possible overflow size. |
| 653 |
*/ |
| 654 |
size = offsetof(ReadStream, buffers); |
| 655 |
size += sizeof(Buffer) * (queue_size + queue_overflow); |
| 656 |
size += sizeof(InProgressIO) * Max(1, max_ios); |
| 657 |
size += per_buffer_data_size * queue_size; |
| 658 |
size += MAXIMUM_ALIGNOF * 2; |
| 659 |
stream = (ReadStream *) palloc(size); |
| 660 |
memset(stream, 0, offsetof(ReadStream, buffers)); |
| 661 |
stream->ios = (InProgressIO *) |
| 662 |
MAXALIGN(&stream->buffers[queue_size + queue_overflow]); |
| 663 |
if (per_buffer_data_size > 0) |
| 664 |
stream->per_buffer_data = (void *) |
| 665 |
MAXALIGN(&stream->ios[Max(1, max_ios)]); |
| 666 |
|
| 667 |
stream->sync_mode = io_method == IOMETHOD_SYNC; |
| 668 |
stream->batch_mode = flags & READ_STREAM_USE_BATCHING; |
| 669 |
|
| 670 |
#ifdef USE_PREFETCH |
| 671 |
|
| 672 |
/* |
| 673 |
* Read-ahead advice simulating asynchronous I/O with synchronous calls. |
| 674 |
* Issue advice only if AIO is not used, direct I/O isn't enabled, the |
| 675 |
* caller hasn't promised sequential access (overriding our detection |
| 676 |
* heuristics), and max_ios hasn't been set to zero. |
| 677 |
*/ |
| 678 |
if (stream->sync_mode && |
| 679 |
(io_direct_flags & IO_DIRECT_DATA) == 0 && |
| 680 |
(flags & READ_STREAM_SEQUENTIAL) == 0 && |
| 681 |
max_ios > 0) |
| 682 |
stream->advice_enabled = true; |
| 683 |
#endif |
| 684 |
|
| 685 |
/* |
| 686 |
* Setting max_ios to zero disables AIO and advice-based pseudo AIO, but |
| 687 |
* we still need to allocate space to combine and run one I/O. Bump it up |
| 688 |
* to one, and remember to ask for synchronous I/O only. |
| 689 |
*/ |
| 690 |
if (max_ios == 0) |
| 691 |
{ |
| 692 |
max_ios = 1; |
| 693 |
stream->read_buffers_flags = READ_BUFFERS_SYNCHRONOUSLY; |
| 694 |
} |
| 695 |
|
| 696 |
/* |
| 697 |
* Capture stable values for these two GUC-derived numbers for the |
| 698 |
* lifetime of this stream, so we don't have to worry about the GUCs |
| 699 |
* changing underneath us beyond this point. |
| 700 |
*/ |
| 701 |
stream->max_ios = max_ios; |
| 702 |
stream->io_combine_limit = io_combine_limit; |
| 703 |
|
| 704 |
stream->per_buffer_data_size = per_buffer_data_size; |
| 705 |
stream->max_pinned_buffers = max_pinned_buffers; |
| 706 |
stream->queue_size = queue_size; |
| 707 |
stream->callback = callback; |
| 708 |
stream->callback_private_data = callback_private_data; |
| 709 |
stream->buffered_blocknum = InvalidBlockNumber; |
| 710 |
stream->seq_blocknum = InvalidBlockNumber; |
| 711 |
stream->seq_until_processed = InvalidBlockNumber; |
| 712 |
stream->temporary = SmgrIsTemp(smgr); |
| 713 |
|
| 714 |
/* |
| 715 |
* Skip the initial ramp-up phase if the caller says we're going to be |
| 716 |
* reading the whole relation. This way we start out assuming we'll be |
| 717 |
* doing full io_combine_limit sized reads. |
| 718 |
*/ |
| 719 |
if (flags & READ_STREAM_FULL) |
| 720 |
stream->distance = Min(max_pinned_buffers, stream->io_combine_limit); |
| 721 |
else |
| 722 |
stream->distance = 1; |
| 723 |
|
| 724 |
/* |
| 725 |
* Since we always access the same relation, we can initialize parts of |
| 726 |
* the ReadBuffersOperation objects and leave them that way, to avoid |
| 727 |
* wasting CPU cycles writing to them for each read. |
| 728 |
*/ |
| 729 |
for (int i = 0; i < max_ios; ++i) |
| 730 |
{ |
| 731 |
stream->ios[i].op.rel = rel; |
| 732 |
stream->ios[i].op.smgr = smgr; |
| 733 |
stream->ios[i].op.persistence = persistence; |
| 734 |
stream->ios[i].op.forknum = forknum; |
| 735 |
stream->ios[i].op.strategy = strategy; |
| 736 |
} |
| 737 |
|
| 738 |
return stream; |
| 739 |
} |
| 740 |
|
| 741 |
/* |
| 742 |
* Create a new read stream for reading a relation. |
| 743 |
* See read_stream_begin_impl() for the detailed explanation. |
| 744 |
*/ |
| 745 |
ReadStream * |
| 746 |
read_stream_begin_relation(int flags, |
| 747 |
BufferAccessStrategy strategy, |
| 748 |
Relation rel, |
| 749 |
ForkNumber forknum, |
| 750 |
ReadStreamBlockNumberCB callback, |
| 751 |
void *callback_private_data, |
| 752 |
size_t per_buffer_data_size) |
| 753 |
{ |
| 754 |
return read_stream_begin_impl(flags, |
| 755 |
strategy, |
| 756 |
rel, |
| 757 |
RelationGetSmgr(rel), |
| 758 |
rel->rd_rel->relpersistence, |
| 759 |
forknum, |
| 760 |
callback, |
| 761 |
callback_private_data, |
| 762 |
per_buffer_data_size); |
| 763 |
} |
| 764 |
|
| 765 |
/* |
| 766 |
* Create a new read stream for reading a SMgr relation. |
| 767 |
* See read_stream_begin_impl() for the detailed explanation. |
| 768 |
*/ |
| 769 |
ReadStream * |
| 770 |
read_stream_begin_smgr_relation(int flags, |
| 771 |
BufferAccessStrategy strategy, |
| 772 |
SMgrRelation smgr, |
| 773 |
char smgr_persistence, |
| 774 |
ForkNumber forknum, |
| 775 |
ReadStreamBlockNumberCB callback, |
| 776 |
void *callback_private_data, |
| 777 |
size_t per_buffer_data_size) |
| 778 |
{ |
| 779 |
return read_stream_begin_impl(flags, |
| 780 |
strategy, |
| 781 |
NULL, |
| 782 |
smgr, |
| 783 |
smgr_persistence, |
| 784 |
forknum, |
| 785 |
callback, |
| 786 |
callback_private_data, |
| 787 |
per_buffer_data_size); |
| 788 |
} |
| 789 |
|
| 790 |
/* |
| 791 |
* Pull one pinned buffer out of a stream. Each call returns successive |
| 792 |
* blocks in the order specified by the callback. If per_buffer_data_size was |
| 793 |
* set to a non-zero size, *per_buffer_data receives a pointer to the extra |
| 794 |
* per-buffer data that the callback had a chance to populate, which remains |
| 795 |
* valid until the next call to read_stream_next_buffer(). When the stream |
| 796 |
* runs out of data, InvalidBuffer is returned. The caller may decide to end |
| 797 |
* the stream early at any time by calling read_stream_end(). |
| 798 |
*/ |
| 799 |
Buffer |
| 800 |
read_stream_next_buffer(ReadStream *stream, void **per_buffer_data) |
| 801 |
{ |
| 802 |
Buffer buffer; |
| 803 |
int16 oldest_buffer_index; |
| 804 |
|
| 805 |
#ifndef READ_STREAM_DISABLE_FAST_PATH |
| 806 |
|
| 807 |
/* |
| 808 |
* A fast path for all-cached scans. This is the same as the usual |
| 809 |
* algorithm, but it is specialized for no I/O and no per-buffer data, so |
| 810 |
* we can skip the queue management code, stay in the same buffer slot and |
| 811 |
* use singular StartReadBuffer(). |
| 812 |
*/ |
| 813 |
if (likely(stream->fast_path)) |
| 814 |
{ |
| 815 |
BlockNumber next_blocknum; |
| 816 |
|
| 817 |
/* Fast path assumptions. */ |
| 818 |
Assert(stream->ios_in_progress == 0); |
| 819 |
Assert(stream->forwarded_buffers == 0); |
| 820 |
Assert(stream->pinned_buffers == 1); |
| 821 |
Assert(stream->distance == 1); |
| 822 |
Assert(stream->pending_read_nblocks == 0); |
| 823 |
Assert(stream->per_buffer_data_size == 0); |
| 824 |
Assert(stream->initialized_buffers > stream->oldest_buffer_index); |
| 825 |
|
| 826 |
/* We're going to return the buffer we pinned last time. */ |
| 827 |
oldest_buffer_index = stream->oldest_buffer_index; |
| 828 |
Assert((oldest_buffer_index + 1) % stream->queue_size == |
| 829 |
stream->next_buffer_index); |
| 830 |
buffer = stream->buffers[oldest_buffer_index]; |
| 831 |
Assert(buffer != InvalidBuffer); |
| 832 |
|
| 833 |
/* Choose the next block to pin. */ |
| 834 |
next_blocknum = read_stream_get_block(stream, NULL); |
| 835 |
|
| 836 |
if (likely(next_blocknum != InvalidBlockNumber)) |
| 837 |
{ |
| 838 |
int flags = stream->read_buffers_flags; |
| 839 |
|
| 840 |
if (stream->advice_enabled) |
| 841 |
flags |= READ_BUFFERS_ISSUE_ADVICE; |
| 842 |
|
| 843 |
/* |
| 844 |
* Pin a buffer for the next call. Same buffer entry, and |
| 845 |
* arbitrary I/O entry (they're all free). We don't have to |
| 846 |
* adjust pinned_buffers because we're transferring one to caller |
| 847 |
* but pinning one more. |
| 848 |
* |
| 849 |
* In the fast path we don't need to check the pin limit. We're |
| 850 |
* always allowed at least one pin so that progress can be made, |
| 851 |
* and that's all we need here. Although two pins are momentarily |
| 852 |
* held at the same time, the model used here is that the stream |
| 853 |
* holds only one, and the other now belongs to the caller. |
| 854 |
*/ |
| 855 |
if (likely(!StartReadBuffer(&stream->ios[0].op, |
| 856 |
&stream->buffers[oldest_buffer_index], |
| 857 |
next_blocknum, |
| 858 |
flags))) |
| 859 |
{ |
| 860 |
/* Fast return. */ |
| 861 |
return buffer; |
| 862 |
} |
| 863 |
|
| 864 |
/* Next call must wait for I/O for the newly pinned buffer. */ |
| 865 |
stream->oldest_io_index = 0; |
| 866 |
stream->next_io_index = stream->max_ios > 1 ? 1 : 0; |
| 867 |
stream->ios_in_progress = 1; |
| 868 |
stream->ios[0].buffer_index = oldest_buffer_index; |
| 869 |
stream->seq_blocknum = next_blocknum + 1; |
| 870 |
} |
| 871 |
else |
| 872 |
{ |
| 873 |
/* No more blocks, end of stream. */ |
| 874 |
stream->distance = 0; |
| 875 |
stream->oldest_buffer_index = stream->next_buffer_index; |
| 876 |
stream->pinned_buffers = 0; |
| 877 |
stream->buffers[oldest_buffer_index] = InvalidBuffer; |
| 878 |
} |
| 879 |
|
| 880 |
stream->fast_path = false; |
| 881 |
return buffer; |
| 882 |
} |
| 883 |
#endif |
| 884 |
|
| 885 |
if (unlikely(stream->pinned_buffers == 0)) |
| 886 |
{ |
| 887 |
Assert(stream->oldest_buffer_index == stream->next_buffer_index); |
| 888 |
|
| 889 |
/* End of stream reached? */ |
| 890 |
if (stream->distance == 0) |
| 891 |
return InvalidBuffer; |
| 892 |
|
| 893 |
/* |
| 894 |
* The usual order of operations is that we look ahead at the bottom |
| 895 |
* of this function after potentially finishing an I/O and making |
| 896 |
* space for more, but if we're just starting up we'll need to crank |
| 897 |
* the handle to get started. |
| 898 |
*/ |
| 899 |
read_stream_look_ahead(stream); |
| 900 |
|
| 901 |
/* End of stream reached? */ |
| 902 |
if (stream->pinned_buffers == 0) |
| 903 |
{ |
| 904 |
Assert(stream->distance == 0); |
| 905 |
return InvalidBuffer; |
| 906 |
} |
| 907 |
} |
| 908 |
|
| 909 |
/* Grab the oldest pinned buffer and associated per-buffer data. */ |
| 910 |
Assert(stream->pinned_buffers > 0); |
| 911 |
oldest_buffer_index = stream->oldest_buffer_index; |
| 912 |
Assert(oldest_buffer_index >= 0 && |
| 913 |
oldest_buffer_index < stream->queue_size); |
| 914 |
buffer = stream->buffers[oldest_buffer_index]; |
| 915 |
if (per_buffer_data) |
| 916 |
*per_buffer_data = get_per_buffer_data(stream, oldest_buffer_index); |
| 917 |
|
| 918 |
Assert(BufferIsValid(buffer)); |
| 919 |
|
| 920 |
/* Do we have to wait for an associated I/O first? */ |
| 921 |
if (stream->ios_in_progress > 0 && |
| 922 |
stream->ios[stream->oldest_io_index].buffer_index == oldest_buffer_index) |
| 923 |
{ |
| 924 |
int16 io_index = stream->oldest_io_index; |
| 925 |
int32 distance; /* wider temporary value, clamped below */ |
| 926 |
|
| 927 |
/* Sanity check that we still agree on the buffers. */ |
| 928 |
Assert(stream->ios[io_index].op.buffers == |
| 929 |
&stream->buffers[oldest_buffer_index]); |
| 930 |
|
| 931 |
WaitReadBuffers(&stream->ios[io_index].op); |
| 932 |
|
| 933 |
Assert(stream->ios_in_progress > 0); |
| 934 |
stream->ios_in_progress--; |
| 935 |
if (++stream->oldest_io_index == stream->max_ios) |
| 936 |
stream->oldest_io_index = 0; |
| 937 |
|
| 938 |
/* Look-ahead distance ramps up rapidly after we do I/O. */ |
| 939 |
distance = stream->distance * 2; |
| 940 |
distance = Min(distance, stream->max_pinned_buffers); |
| 941 |
stream->distance = distance; |
| 942 |
|
| 943 |
/* |
| 944 |
* If we've reached the first block of a sequential region we're |
| 945 |
* issuing advice for, cancel that until the next jump. The kernel |
| 946 |
* will see the sequential preadv() pattern starting here. |
| 947 |
*/ |
| 948 |
if (stream->advice_enabled && |
| 949 |
stream->ios[io_index].op.blocknum == stream->seq_until_processed) |
| 950 |
stream->seq_until_processed = InvalidBlockNumber; |
| 951 |
} |
| 952 |
|
| 953 |
/* |
| 954 |
* We must zap this queue entry, or else it would appear as a forwarded |
| 955 |
* buffer. If it's potentially in the overflow zone (ie from a |
| 956 |
* multi-block I/O that wrapped around the queue), also zap the copy. |
| 957 |
*/ |
| 958 |
stream->buffers[oldest_buffer_index] = InvalidBuffer; |
| 959 |
if (oldest_buffer_index < stream->io_combine_limit - 1) |
| 960 |
stream->buffers[stream->queue_size + oldest_buffer_index] = |
| 961 |
InvalidBuffer; |
| 962 |
|
| 963 |
#if defined(CLOBBER_FREED_MEMORY) || defined(USE_VALGRIND) |
| 964 |
|
| 965 |
/* |
| 966 |
* The caller will get access to the per-buffer data, until the next call. |
| 967 |
* We wipe the one before, which is never occupied because queue_size |
| 968 |
* allowed one extra element. This will hopefully trip up client code |
| 969 |
* that is holding a dangling pointer to it. |
| 970 |
*/ |
| 971 |
if (stream->per_buffer_data) |
| 972 |
{ |
| 973 |
void *per_buffer_data; |
| 974 |
|
| 975 |
per_buffer_data = get_per_buffer_data(stream, |
| 976 |
oldest_buffer_index == 0 ? |
| 977 |
stream->queue_size - 1 : |
| 978 |
oldest_buffer_index - 1); |
| 979 |
|
| 980 |
#if defined(CLOBBER_FREED_MEMORY) |
| 981 |
/* This also tells Valgrind the memory is "noaccess". */ |
| 982 |
wipe_mem(per_buffer_data, stream->per_buffer_data_size); |
| 983 |
#elif defined(USE_VALGRIND) |
| 984 |
/* Tell it ourselves. */ |
| 985 |
VALGRIND_MAKE_MEM_NOACCESS(per_buffer_data, |
| 986 |
stream->per_buffer_data_size); |
| 987 |
#endif |
| 988 |
} |
| 989 |
#endif |
| 990 |
|
| 991 |
/* Pin transferred to caller. */ |
| 992 |
Assert(stream->pinned_buffers > 0); |
| 993 |
stream->pinned_buffers--; |
| 994 |
|
| 995 |
/* Advance oldest buffer, with wrap-around. */ |
| 996 |
stream->oldest_buffer_index++; |
| 997 |
if (stream->oldest_buffer_index == stream->queue_size) |
| 998 |
stream->oldest_buffer_index = 0; |
| 999 |
|
| 1000 |
/* Prepare for the next call. */ |
| 1001 |
read_stream_look_ahead(stream); |
| 1002 |
|
| 1003 |
#ifndef READ_STREAM_DISABLE_FAST_PATH |
| 1004 |
/* See if we can take the fast path for all-cached scans next time. */ |
| 1005 |
if (stream->ios_in_progress == 0 && |
| 1006 |
stream->forwarded_buffers == 0 && |
| 1007 |
stream->pinned_buffers == 1 && |
| 1008 |
stream->distance == 1 && |
| 1009 |
stream->pending_read_nblocks == 0 && |
| 1010 |
stream->per_buffer_data_size == 0) |
| 1011 |
{ |
| 1012 |
/* |
| 1013 |
* The fast path spins on one buffer entry repeatedly instead of |
| 1014 |
* rotating through the whole queue and clearing the entries behind |
| 1015 |
* it. If the buffer it starts with happened to be forwarded between |
| 1016 |
* StartReadBuffers() calls and also wrapped around the circular queue |
| 1017 |
* partway through, then a copy also exists in the overflow zone, and |
| 1018 |
* it won't clear it out as the regular path would. Do that now, so |
| 1019 |
* it doesn't need code for that. |
| 1020 |
*/ |
| 1021 |
if (stream->oldest_buffer_index < stream->io_combine_limit - 1) |
| 1022 |
stream->buffers[stream->queue_size + stream->oldest_buffer_index] = |
| 1023 |
InvalidBuffer; |
| 1024 |
|
| 1025 |
stream->fast_path = true; |
| 1026 |
} |
| 1027 |
#endif |
| 1028 |
|
| 1029 |
return buffer; |
| 1030 |
} |
| 1031 |
|
| 1032 |
/* |
| 1033 |
* Transitional support for code that would like to perform or skip reads |
| 1034 |
* itself, without using the stream. Returns, and consumes, the next block |
| 1035 |
* number that would be read by the stream's look-ahead algorithm, or |
| 1036 |
* InvalidBlockNumber if the end of the stream is reached. Also reports the |
| 1037 |
* strategy that would be used to read it. |
| 1038 |
*/ |
| 1039 |
BlockNumber |
| 1040 |
read_stream_next_block(ReadStream *stream, BufferAccessStrategy *strategy) |
| 1041 |
{ |
| 1042 |
*strategy = stream->ios[0].op.strategy; |
| 1043 |
return read_stream_get_block(stream, NULL); |
| 1044 |
} |
| 1045 |
|
| 1046 |
|
| 1047 |
/* |
| 1048 |
* Stop using a buffer access strategy for reads from this stream. |
| 1049 |
* |
| 1050 |
* This clears the strategy for all of the stream's ReadBuffersOperations, |
| 1051 |
* including those with in-progress IOs. The completion of an IO whose |
| 1052 |
* strategy was cleared while it was in flight may have a small amount of its |
| 1053 |
* read time attributed to IOCONTEXT_NORMAL instead of the strategy's |
| 1054 |
* IOContext, because WaitReadBuffers() derives the IOContext from the (now |
| 1055 |
* cleared) strategy. This is bounded by the stream's look-ahead window and |
| 1056 |
* happens at most once, when the strategy is first cleared, so it is not worth |
| 1057 |
* the complexity of preserving the original IOContext for those IOs. |
| 1058 |
* |
| 1059 |
* Note that the caller is responsible for freeing the strategy's memory. |
| 1060 |
*/ |
| 1061 |
void |
| 1062 |
read_stream_clear_strategy(ReadStream *stream) |
| 1063 |
{ |
| 1064 |
for (int i = 0; i < stream->max_ios; ++i) |
| 1065 |
stream->ios[i].op.strategy = NULL; |
| 1066 |
} |
| 1067 |
|
| 1068 |
/* |
| 1069 |
* Reset a read stream by releasing any queued up buffers, allowing the stream |
| 1070 |
* to be used again for different blocks. This can be used to clear an |
| 1071 |
* end-of-stream condition and start again, or to throw away blocks that were |
| 1072 |
* speculatively read and read some different blocks instead. |
| 1073 |
*/ |
| 1074 |
void |
| 1075 |
read_stream_reset(ReadStream *stream) |
| 1076 |
{ |
| 1077 |
int16 index; |
| 1078 |
Buffer buffer; |
| 1079 |
|
| 1080 |
/* Stop looking ahead. */ |
| 1081 |
stream->distance = 0; |
| 1082 |
|
| 1083 |
/* Forget buffered block number and fast path state. */ |
| 1084 |
stream->buffered_blocknum = InvalidBlockNumber; |
| 1085 |
stream->fast_path = false; |
| 1086 |
|
| 1087 |
/* Unpin anything that wasn't consumed. */ |
| 1088 |
while ((buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer) |
| 1089 |
ReleaseBuffer(buffer); |
| 1090 |
|
| 1091 |
/* Unpin any unused forwarded buffers. */ |
| 1092 |
index = stream->next_buffer_index; |
| 1093 |
while (index < stream->initialized_buffers && |
| 1094 |
(buffer = stream->buffers[index]) != InvalidBuffer) |
| 1095 |
{ |
| 1096 |
Assert(stream->forwarded_buffers > 0); |
| 1097 |
stream->forwarded_buffers--; |
| 1098 |
ReleaseBuffer(buffer); |
| 1099 |
|
| 1100 |
stream->buffers[index] = InvalidBuffer; |
| 1101 |
if (index < stream->io_combine_limit - 1) |
| 1102 |
stream->buffers[stream->queue_size + index] = InvalidBuffer; |
| 1103 |
|
| 1104 |
if (++index == stream->queue_size) |
| 1105 |
index = 0; |
| 1106 |
} |
| 1107 |
|
| 1108 |
Assert(stream->forwarded_buffers == 0); |
| 1109 |
Assert(stream->pinned_buffers == 0); |
| 1110 |
Assert(stream->ios_in_progress == 0); |
| 1111 |
|
| 1112 |
/* Start off assuming data is cached. */ |
| 1113 |
stream->distance = 1; |
| 1114 |
} |
| 1115 |
|
| 1116 |
/* |
| 1117 |
* Release and free a read stream. |
| 1118 |
*/ |
| 1119 |
void |
| 1120 |
read_stream_end(ReadStream *stream) |
| 1121 |
{ |
| 1122 |
read_stream_reset(stream); |
| 1123 |
pfree(stream); |
| 1124 |
} |