Debian Codegraph @

src/backend/storage/aio/method_io_uring.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * method_io_uring.c
4 * AIO - perform AIO using Linux' io_uring
5 *
6 * For now we create one io_uring instance for each backend. These io_uring
7 * instances have to be created in postmaster, during startup, to allow other
8 * backends to process IO completions, if the issuing backend is currently
9 * busy doing other things. Other backends may not use another backend's
10 * io_uring instance to submit IO, that'd require additional locking that
11 * would likely be harmful for performance.
12 *
13 * We likely will want to introduce a backend-local io_uring instance in the
14 * future, e.g. for FE/BE network IO.
15 *
16 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
17 * Portions Copyright (c) 1994, Regents of the University of California
18 *
19 * IDENTIFICATION
20 * src/backend/storage/aio/method_io_uring.c
21 *
22 *-------------------------------------------------------------------------
23 */
24
25 #include "postgres.h"
26
27 /* included early, for IOMETHOD_IO_URING_ENABLED */
28 #include "storage/aio.h"
29
30 #ifdef IOMETHOD_IO_URING_ENABLED
31
32 #include <sys/mman.h>
33 #include <unistd.h>
34
35 #include <liburing.h>
36
37 #include "miscadmin.h"
38 #include "storage/aio_internal.h"
39 #include "storage/fd.h"
40 #include "storage/proc.h"
41 #include "storage/shmem.h"
42 #include "storage/lwlock.h"
43 #include "storage/procnumber.h"
44 #include "utils/wait_event.h"
45
46
47 /* number of completions processed at once */
48 #define PGAIO_MAX_LOCAL_COMPLETED_IO 32
49
50
51 /* Entry points for IoMethodOps. */
52 static size_t pgaio_uring_shmem_size(void);
53 static void pgaio_uring_shmem_init(bool first_time);
54 static void pgaio_uring_init_backend(void);
55 static int pgaio_uring_submit(uint16 num_staged_ios, PgAioHandle **staged_ios);
56 static void pgaio_uring_wait_one(PgAioHandle *ioh, uint64 ref_generation);
57
58 /* helper functions */
59 static void pgaio_uring_sq_from_io(PgAioHandle *ioh, struct io_uring_sqe *sqe);
60
61
62 const IoMethodOps pgaio_uring_ops = {
63 /*
64 * While io_uring mostly is OK with FDs getting closed while the IO is in
65 * flight, that is not true for IOs submitted with IOSQE_ASYNC.
66 *
67 * See
68 * https://postgr.es/m/5ons2rtmwarqqhhexb3dnqulw5rjgwgoct57vpdau4rujlrffj%403fls6d2mkiwc
69 */
70 .wait_on_fd_before_close = true,
71
72 .shmem_size = pgaio_uring_shmem_size,
73 .shmem_init = pgaio_uring_shmem_init,
74 .init_backend = pgaio_uring_init_backend,
75
76 .submit = pgaio_uring_submit,
77 .wait_one = pgaio_uring_wait_one,
78 };
79
80 /*
81 * Per-backend state when using io_method=io_uring
82 *
83 * Align the whole struct to a cacheline boundary, to prevent false sharing
84 * between completion_lock and prior backend's io_uring_ring.
85 */
86 typedef struct pg_attribute_aligned (PG_CACHE_LINE_SIZE)
87 PgAioUringContext
88 {
89 /*
90 * Multiple backends can process completions for this backend's io_uring
91 * instance (e.g. when the backend issuing IO is busy doing something
92 * else). To make that safe we have to ensure that only a single backend
93 * gets io completions from the io_uring instance at a time.
94 */
95 LWLock completion_lock;
96
97 struct io_uring io_uring_ring;
98 } PgAioUringContext;
99
100 /*
101 * Information about the capabilities that io_uring has.
102 *
103 * Depending on liburing and kernel version different features are
104 * supported. At least for the kernel a kernel version check does not suffice
105 * as various vendors do backport features to older kernels :(.
106 */
107 typedef struct PgAioUringCaps
108 {
109 bool checked;
110 /* -1 if io_uring_queue_init_mem() is unsupported */
111 int mem_init_size;
112 } PgAioUringCaps;
113
114
115 /* PgAioUringContexts for all backends */
116 static PgAioUringContext *pgaio_uring_contexts;
117
118 /* the current backend's context */
119 static PgAioUringContext *pgaio_my_uring_context;
120
121 static PgAioUringCaps pgaio_uring_caps =
122 {
123 .checked = false,
124 .mem_init_size = -1,
125 };
126
127 static uint32
128 pgaio_uring_procs(void)
129 {
130 /*
131 * We can subtract MAX_IO_WORKERS here as io workers are never used at the
132 * same time as io_method=io_uring.
133 */
134 return MaxBackends + NUM_AUXILIARY_PROCS - MAX_IO_WORKERS;
135 }
136
137 /*
138 * Initializes pgaio_uring_caps, unless that's already done.
139 */
140 static void
141 pgaio_uring_check_capabilities(void)
142 {
143 if (pgaio_uring_caps.checked)
144 return;
145
146 /*
147 * By default io_uring creates a shared memory mapping for each io_uring
148 * instance, leading to a large number of memory mappings. Unfortunately a
149 * large number of memory mappings slows things down, backend exit is
150 * particularly affected. To address that, newer kernels (6.5) support
151 * using user-provided memory for the memory, by putting the relevant
152 * memory into shared memory we don't need any additional mappings.
153 *
154 * To know whether this is supported, we unfortunately need to probe the
155 * kernel by trying to create a ring with userspace-provided memory. This
156 * also has a secondary benefit: We can determine precisely how much
157 * memory we need for each io_uring instance.
158 */
159 #if defined(HAVE_IO_URING_QUEUE_INIT_MEM) && defined(IORING_SETUP_NO_MMAP)
160 {
161 struct io_uring test_ring;
162 size_t ring_size;
163 void *ring_ptr;
164 struct io_uring_params p = {0};
165 int ret;
166
167 /*
168 * Liburing does not yet provide an API to query how much memory a
169 * ring will need. So we over-estimate it here. As the memory is freed
170 * just below that's small temporary waste of memory.
171 *
172 * 1MB is more than enough for rings within io_max_concurrency's
173 * range.
174 */
175 ring_size = 1024 * 1024;
176
177 /*
178 * Hard to believe a system exists where 1MB would not be a multiple
179 * of the page size. But it's cheap to ensure...
180 */
181 ring_size -= ring_size % sysconf(_SC_PAGESIZE);
182
183 ring_ptr = mmap(NULL, ring_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
184 if (ring_ptr == MAP_FAILED)
185 elog(ERROR,
186 "mmap(%zu) to determine io_uring_queue_init_mem() support failed: %m",
187 ring_size);
188
189 ret = io_uring_queue_init_mem(io_max_concurrency, &test_ring, &p, ring_ptr, ring_size);
190 if (ret > 0)
191 {
192 pgaio_uring_caps.mem_init_size = ret;
193
194 elog(DEBUG1,
195 "can use combined memory mapping for io_uring, each ring needs %d bytes",
196 ret);
197
198 /* clean up the created ring, it was just for a test */
199 io_uring_queue_exit(&test_ring);
200 }
201 else
202 {
203 /*
204 * There are different reasons for ring creation to fail, but it's
205 * ok to treat that just as io_uring_queue_init_mem() not being
206 * supported. We'll report a more detailed error in
207 * pgaio_uring_shmem_init().
208 */
209 errno = -ret;
210 elog(DEBUG1,
211 "cannot use combined memory mapping for io_uring, ring creation failed: %m");
212
213 }
214
215 if (munmap(ring_ptr, ring_size) != 0)
216 elog(ERROR, "munmap() failed: %m");
217 }
218 #else
219 {
220 elog(DEBUG1,
221 "can't use combined memory mapping for io_uring, kernel or liburing too old");
222 }
223 #endif
224
225 pgaio_uring_caps.checked = true;
226 }
227
228 /*
229 * Memory for all PgAioUringContext instances
230 */
231 static size_t
232 pgaio_uring_context_shmem_size(void)
233 {
234 return mul_size(pgaio_uring_procs(), sizeof(PgAioUringContext));
235 }
236
237 /*
238 * Memory for the combined memory used by io_uring instances. Returns 0 if
239 * that is not supported by kernel/liburing.
240 */
241 static size_t
242 pgaio_uring_ring_shmem_size(void)
243 {
244 size_t sz = 0;
245
246 if (pgaio_uring_caps.mem_init_size > 0)
247 {
248 /*
249 * Memory for rings needs to be allocated to the page boundary,
250 * reserve space. Luckily it does not need to be aligned to hugepage
251 * boundaries, even if huge pages are used.
252 */
253 sz = add_size(sz, sysconf(_SC_PAGESIZE));
254 sz = add_size(sz, mul_size(pgaio_uring_procs(),
255 pgaio_uring_caps.mem_init_size));
256 }
257
258 return sz;
259 }
260
261 static size_t
262 pgaio_uring_shmem_size(void)
263 {
264 size_t sz;
265
266 /*
267 * Kernel and liburing support for various features influences how much
268 * shmem we need, perform the necessary checks.
269 */
270 pgaio_uring_check_capabilities();
271
272 sz = pgaio_uring_context_shmem_size();
273 sz = add_size(sz, pgaio_uring_ring_shmem_size());
274
275 return sz;
276 }
277
278 static void
279 pgaio_uring_shmem_init(bool first_time)
280 {
281 int TotalProcs = pgaio_uring_procs();
282 bool found;
283 char *shmem;
284 size_t ring_mem_remain = 0;
285 char *ring_mem_next = 0;
286
287 /*
288 * We allocate memory for all PgAioUringContext instances and, if
289 * supported, the memory required for each of the io_uring instances, in
290 * one ShmemInitStruct().
291 */
292 shmem = ShmemInitStruct("AioUringContext", pgaio_uring_shmem_size(), &found);
293 if (found)
294 return;
295
296 pgaio_uring_contexts = (PgAioUringContext *) shmem;
297 shmem += pgaio_uring_context_shmem_size();
298
299 /* if supported, handle memory alignment / sizing for io_uring memory */
300 if (pgaio_uring_caps.mem_init_size > 0)
301 {
302 ring_mem_remain = pgaio_uring_ring_shmem_size();
303 ring_mem_next = (char *) shmem;
304
305 /* align to page boundary, see also pgaio_uring_ring_shmem_size() */
306 ring_mem_next = (char *) TYPEALIGN(sysconf(_SC_PAGESIZE), ring_mem_next);
307
308 /* account for alignment */
309 ring_mem_remain -= ring_mem_next - shmem;
310 shmem += ring_mem_next - shmem;
311
312 shmem += ring_mem_remain;
313 }
314
315 for (int contextno = 0; contextno < TotalProcs; contextno++)
316 {
317 PgAioUringContext *context = &pgaio_uring_contexts[contextno];
318 int ret;
319
320 /*
321 * Right now a high TotalProcs will cause problems in two ways:
322 *
323 * - RLIMIT_NOFILE needs to be big enough to allow all
324 * io_uring_queue_init() calls to succeed.
325 *
326 * - RLIMIT_NOFILE needs to be big enough to still have enough file
327 * descriptors to satisfy set_max_safe_fds() left over. Or, even
328 * better, have max_files_per_process left over FDs.
329 *
330 * We probably should adjust the soft RLIMIT_NOFILE to ensure that.
331 *
332 *
333 * XXX: Newer versions of io_uring support sharing the workers that
334 * execute some asynchronous IOs between io_uring instances. It might
335 * be worth using that - also need to evaluate if that causes
336 * noticeable additional contention?
337 */
338
339 /*
340 * If supported (c.f. pgaio_uring_check_capabilities()), create ring
341 * with its data in shared memory. Otherwise fall back io_uring
342 * creating a memory mapping for each ring.
343 */
344 #if defined(HAVE_IO_URING_QUEUE_INIT_MEM) && defined(IORING_SETUP_NO_MMAP)
345 if (pgaio_uring_caps.mem_init_size > 0)
346 {
347 struct io_uring_params p = {0};
348
349 ret = io_uring_queue_init_mem(io_max_concurrency, &context->io_uring_ring, &p, ring_mem_next, ring_mem_remain);
350
351 ring_mem_remain -= ret;
352 ring_mem_next += ret;
353 }
354 else
355 #endif
356 {
357 ret = io_uring_queue_init(io_max_concurrency, &context->io_uring_ring, 0);
358 }
359
360 if (ret < 0)
361 {
362 char *hint = NULL;
363 int err = ERRCODE_INTERNAL_ERROR;
364
365 /* add hints for some failures that errno explains sufficiently */
366 if (-ret == EPERM)
367 {
368 err = ERRCODE_INSUFFICIENT_PRIVILEGE;
369 hint = _("Check if io_uring is disabled via /proc/sys/kernel/io_uring_disabled.");
370 }
371 else if (-ret == EMFILE)
372 {
373 err = ERRCODE_INSUFFICIENT_RESOURCES;
374 hint = psprintf(_("Consider increasing \"ulimit -n\" to at least %d."),
375 TotalProcs + max_files_per_process);
376 }
377 else if (-ret == ENOSYS)
378 {
379 err = ERRCODE_FEATURE_NOT_SUPPORTED;
380 hint = _("The kernel does not support io_uring.");
381 }
382
383 /* update errno to allow %m to work */
384 errno = -ret;
385
386 ereport(ERROR,
387 errcode(err),
388 errmsg("could not setup io_uring queue: %m"),
389 hint != NULL ? errhint("%s", hint) : 0);
390 }
391
392 LWLockInitialize(&context->completion_lock, LWTRANCHE_AIO_URING_COMPLETION);
393 }
394 }
395
396 static void
397 pgaio_uring_init_backend(void)
398 {
399 Assert(MyProcNumber < pgaio_uring_procs());
400
401 pgaio_my_uring_context = &pgaio_uring_contexts[MyProcNumber];
402 }
403
404 static int
405 pgaio_uring_submit(uint16 num_staged_ios, PgAioHandle **staged_ios)
406 {
407 struct io_uring *uring_instance = &pgaio_my_uring_context->io_uring_ring;
408 int in_flight_before = dclist_count(&pgaio_my_backend->in_flight_ios);
409
410 Assert(num_staged_ios <= PGAIO_SUBMIT_BATCH_SIZE);
411
412 for (int i = 0; i < num_staged_ios; i++)
413 {
414 PgAioHandle *ioh = staged_ios[i];
415 struct io_uring_sqe *sqe;
416
417 sqe = io_uring_get_sqe(uring_instance);
418
419 if (!sqe)
420 elog(ERROR, "io_uring submission queue is unexpectedly full");
421
422 pgaio_io_prepare_submit(ioh);
423 pgaio_uring_sq_from_io(ioh, sqe);
424
425 /*
426 * io_uring executes IO in process context if possible. That's
427 * generally good, as it reduces context switching. When performing a
428 * lot of buffered IO that means that copying between page cache and
429 * userspace memory happens in the foreground, as it can't be
430 * offloaded to DMA hardware as is possible when using direct IO. When
431 * executing a lot of buffered IO this causes io_uring to be slower
432 * than worker mode, as worker mode parallelizes the copying. io_uring
433 * can be told to offload work to worker threads instead.
434 *
435 * If an IO is buffered IO and we already have IOs in flight or
436 * multiple IOs are being submitted, we thus tell io_uring to execute
437 * the IO in the background. We don't do so for the first few IOs
438 * being submitted as executing in this process' context has lower
439 * latency.
440 */
441 if (in_flight_before > 4 && (ioh->flags & PGAIO_HF_BUFFERED))
442 io_uring_sqe_set_flags(sqe, IOSQE_ASYNC);
443
444 in_flight_before++;
445 }
446
447 while (true)
448 {
449 int ret;
450
451 pgstat_report_wait_start(WAIT_EVENT_AIO_IO_URING_SUBMIT);
452 ret = io_uring_submit(uring_instance);
453 pgstat_report_wait_end();
454
455 if (ret == -EINTR)
456 {
457 pgaio_debug(DEBUG3,
458 "aio method uring: submit EINTR, nios: %d",
459 num_staged_ios);
460 }
461 else if (ret < 0)
462 {
463 /*
464 * The io_uring_enter() manpage suggests that the appropriate
465 * reaction to EAGAIN is:
466 *
467 * "The application should wait for some completions and try
468 * again"
469 *
470 * However, it seems unlikely that that would help in our case, as
471 * we apply a low limit to the number of outstanding IOs and thus
472 * also outstanding completions, making it unlikely that we'd get
473 * EAGAIN while the OS is in good working order.
474 *
475 * Additionally, it would be problematic to just wait here, our
476 * caller might hold critical locks. It'd possibly lead to
477 * delaying the crash-restart that seems likely to occur when the
478 * kernel is under such heavy memory pressure.
479 *
480 * Update errno to allow %m to work.
481 */
482 errno = -ret;
483 elog(PANIC, "io_uring submit failed: %m");
484 }
485 else if (ret != num_staged_ios)
486 {
487 /* likely unreachable, but if it is, we would need to re-submit */
488 elog(PANIC, "io_uring submit submitted only %d of %d",
489 ret, num_staged_ios);
490 }
491 else
492 {
493 pgaio_debug(DEBUG4,
494 "aio method uring: submitted %d IOs",
495 num_staged_ios);
496 break;
497 }
498 }
499
500 return num_staged_ios;
501 }
502
503 static void
504 pgaio_uring_completion_error_callback(void *arg)
505 {
506 ProcNumber owner;
507 PGPROC *owner_proc;
508 int32 owner_pid;
509 PgAioHandle *ioh = arg;
510
511 if (!ioh)
512 return;
513
514 /* No need for context if a backend is completing the IO for itself */
515 if (ioh->owner_procno == MyProcNumber)
516 return;
517
518 owner = ioh->owner_procno;
519 owner_proc = GetPGProcByNumber(owner);
520 owner_pid = owner_proc->pid;
521
522 errcontext("completing I/O on behalf of process %d", owner_pid);
523 }
524
525 static void
526 pgaio_uring_drain_locked(PgAioUringContext *context)
527 {
528 int ready;
529 int orig_ready;
530 ErrorContextCallback errcallback = {0};
531
532 Assert(LWLockHeldByMeInMode(&context->completion_lock, LW_EXCLUSIVE));
533
534 errcallback.callback = pgaio_uring_completion_error_callback;
535 errcallback.previous = error_context_stack;
536 error_context_stack = &errcallback;
537
538 /*
539 * Don't drain more events than available right now. Otherwise it's
540 * plausible that one backend could get stuck, for a while, receiving CQEs
541 * without actually processing them.
542 */
543 orig_ready = ready = io_uring_cq_ready(&context->io_uring_ring);
544
545 while (ready > 0)
546 {
547 struct io_uring_cqe *cqes[PGAIO_MAX_LOCAL_COMPLETED_IO];
548 uint32 ncqes;
549
550 START_CRIT_SECTION();
551 ncqes =
552 io_uring_peek_batch_cqe(&context->io_uring_ring,
553 cqes,
554 Min(PGAIO_MAX_LOCAL_COMPLETED_IO, ready));
555 Assert(ncqes <= ready);
556
557 ready -= ncqes;
558
559 for (int i = 0; i < ncqes; i++)
560 {
561 struct io_uring_cqe *cqe = cqes[i];
562 PgAioHandle *ioh = io_uring_cqe_get_data(cqe);
563 int result = cqe->res;
564
565 errcallback.arg = ioh;
566
567 io_uring_cqe_seen(&context->io_uring_ring, cqe);
568
569 pgaio_io_process_completion(ioh, result);
570 errcallback.arg = NULL;
571 }
572
573 END_CRIT_SECTION();
574
575 pgaio_debug(DEBUG3,
576 "drained %d/%d, now expecting %d",
577 ncqes, orig_ready, io_uring_cq_ready(&context->io_uring_ring));
578 }
579
580 error_context_stack = errcallback.previous;
581 }
582
583 static void
584 pgaio_uring_wait_one(PgAioHandle *ioh, uint64 ref_generation)
585 {
586 PgAioHandleState state;
587 ProcNumber owner_procno = ioh->owner_procno;
588 PgAioUringContext *owner_context = &pgaio_uring_contexts[owner_procno];
589 bool expect_cqe;
590 int waited = 0;
591
592 /*
593 * XXX: It would be nice to have a smarter locking scheme, nearly all the
594 * time the backend owning the ring will consume the completions, making
595 * the locking unnecessarily expensive.
596 */
597 LWLockAcquire(&owner_context->completion_lock, LW_EXCLUSIVE);
598
599 while (true)
600 {
601 pgaio_debug_io(DEBUG3, ioh,
602 "wait_one io_gen: %" PRIu64 ", ref_gen: %" PRIu64 ", cycle %d",
603 ioh->generation,
604 ref_generation,
605 waited);
606
607 if (pgaio_io_was_recycled(ioh, ref_generation, &state) ||
608 state != PGAIO_HS_SUBMITTED)
609 {
610 /* the IO was completed by another backend */
611 break;
612 }
613 else if (io_uring_cq_ready(&owner_context->io_uring_ring))
614 {
615 /* no need to wait in the kernel, io_uring has a completion */
616 expect_cqe = true;
617 }
618 else
619 {
620 int ret;
621 struct io_uring_cqe *cqes;
622
623 /* need to wait in the kernel */
624 pgstat_report_wait_start(WAIT_EVENT_AIO_IO_URING_EXECUTION);
625 ret = io_uring_wait_cqes(&owner_context->io_uring_ring, &cqes, 1, NULL, NULL);
626 pgstat_report_wait_end();
627
628 if (ret == -EINTR)
629 {
630 continue;
631 }
632 else if (ret != 0)
633 {
634 /* see comment after io_uring_submit() */
635 errno = -ret;
636 elog(PANIC, "io_uring wait failed: %m");
637 }
638 else
639 {
640 Assert(cqes != NULL);
641 expect_cqe = true;
642 waited++;
643 }
644 }
645
646 if (expect_cqe)
647 {
648 pgaio_uring_drain_locked(owner_context);
649 }
650 }
651
652 LWLockRelease(&owner_context->completion_lock);
653
654 pgaio_debug(DEBUG3,
655 "wait_one with %d sleeps",
656 waited);
657 }
658
659 static void
660 pgaio_uring_sq_from_io(PgAioHandle *ioh, struct io_uring_sqe *sqe)
661 {
662 struct iovec *iov;
663
664 switch ((PgAioOp) ioh->op)
665 {
666 case PGAIO_OP_READV:
667 iov = &pgaio_ctl->iovecs[ioh->iovec_off];
668 if (ioh->op_data.read.iov_length == 1)
669 {
670 io_uring_prep_read(sqe,
671 ioh->op_data.read.fd,
672 iov->iov_base,
673 iov->iov_len,
674 ioh->op_data.read.offset);
675 }
676 else
677 {
678 io_uring_prep_readv(sqe,
679 ioh->op_data.read.fd,
680 iov,
681 ioh->op_data.read.iov_length,
682 ioh->op_data.read.offset);
683
684 }
685 break;
686
687 case PGAIO_OP_WRITEV:
688 iov = &pgaio_ctl->iovecs[ioh->iovec_off];
689 if (ioh->op_data.write.iov_length == 1)
690 {
691 io_uring_prep_write(sqe,
692 ioh->op_data.write.fd,
693 iov->iov_base,
694 iov->iov_len,
695 ioh->op_data.write.offset);
696 }
697 else
698 {
699 io_uring_prep_writev(sqe,
700 ioh->op_data.write.fd,
701 iov,
702 ioh->op_data.write.iov_length,
703 ioh->op_data.write.offset);
704 }
705 break;
706
707 case PGAIO_OP_INVALID:
708 elog(ERROR, "trying to prepare invalid IO operation for execution");
709 }
710
711 io_uring_sqe_set_data(sqe, ioh);
712 }
713
714 #endif /* IOMETHOD_IO_URING_ENABLED */