Debian Codegraph @

fs/fuse/dev.c

raw
1 /*
2 FUSE: Filesystem in Userspace
3 Copyright (C) 2001-2008 Miklos Szeredi <[email protected]>
4
5 This program can be distributed under the terms of the GNU GPL.
6 See the file COPYING.
7 */
8
9 #include "dev_uring_i.h"
10 #include "fuse_i.h"
11 #include "fuse_dev_i.h"
12
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/poll.h>
16 #include <linux/sched/signal.h>
17 #include <linux/uio.h>
18 #include <linux/miscdevice.h>
19 #include <linux/pagemap.h>
20 #include <linux/file.h>
21 #include <linux/slab.h>
22 #include <linux/pipe_fs_i.h>
23 #include <linux/swap.h>
24 #include <linux/splice.h>
25 #include <linux/sched.h>
26 #include <linux/seq_file.h>
27
28 #include "fuse_trace.h"
29
30 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
31 MODULE_ALIAS("devname:fuse");
32
33 static struct kmem_cache *fuse_req_cachep;
34
35 const unsigned long fuse_timeout_timer_freq =
36 secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ);
37
38 bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list)
39 {
40 struct fuse_req *req;
41
42 req = list_first_entry_or_null(list, struct fuse_req, list);
43 if (!req)
44 return false;
45 return time_is_before_jiffies(req->create_time + fc->timeout.req_timeout);
46 }
47
48 static bool fuse_fpq_processing_expired(struct fuse_conn *fc, struct list_head *processing)
49 {
50 int i;
51
52 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
53 if (fuse_request_expired(fc, &processing[i]))
54 return true;
55
56 return false;
57 }
58
59 /*
60 * Check if any requests aren't being completed by the time the request timeout
61 * elapses. To do so, we:
62 * - check the fiq pending list
63 * - check the bg queue
64 * - check the fpq io and processing lists
65 *
66 * To make this fast, we only check against the head request on each list since
67 * these are generally queued in order of creation time (eg newer requests get
68 * queued to the tail). We might miss a few edge cases (eg requests transitioning
69 * between lists, re-sent requests at the head of the pending list having a
70 * later creation time than other requests on that list, etc.) but that is fine
71 * since if the request never gets fulfilled, it will eventually be caught.
72 */
73 void fuse_check_timeout(struct work_struct *work)
74 {
75 struct delayed_work *dwork = to_delayed_work(work);
76 struct fuse_conn *fc = container_of(dwork, struct fuse_conn,
77 timeout.work);
78 struct fuse_iqueue *fiq = &fc->iq;
79 struct fuse_dev *fud;
80 struct fuse_pqueue *fpq;
81 bool expired = false;
82
83 if (!atomic_read(&fc->num_waiting))
84 goto out;
85
86 spin_lock(&fiq->lock);
87 expired = fuse_request_expired(fc, &fiq->pending);
88 spin_unlock(&fiq->lock);
89 if (expired)
90 goto abort_conn;
91
92 spin_lock(&fc->bg_lock);
93 expired = fuse_request_expired(fc, &fc->bg_queue);
94 spin_unlock(&fc->bg_lock);
95 if (expired)
96 goto abort_conn;
97
98 spin_lock(&fc->lock);
99 if (!fc->connected) {
100 spin_unlock(&fc->lock);
101 return;
102 }
103 list_for_each_entry(fud, &fc->devices, entry) {
104 fpq = &fud->pq;
105 spin_lock(&fpq->lock);
106 if (fuse_request_expired(fc, &fpq->io) ||
107 fuse_fpq_processing_expired(fc, fpq->processing)) {
108 spin_unlock(&fpq->lock);
109 spin_unlock(&fc->lock);
110 goto abort_conn;
111 }
112
113 spin_unlock(&fpq->lock);
114 }
115 spin_unlock(&fc->lock);
116
117 if (fuse_uring_request_expired(fc))
118 goto abort_conn;
119
120 out:
121 queue_delayed_work(system_percpu_wq, &fc->timeout.work,
122 fuse_timeout_timer_freq);
123 return;
124
125 abort_conn:
126 fuse_abort_conn(fc);
127 }
128
129 static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
130 {
131 INIT_LIST_HEAD(&req->list);
132 INIT_LIST_HEAD(&req->intr_entry);
133 init_waitqueue_head(&req->waitq);
134 refcount_set(&req->count, 1);
135 __set_bit(FR_PENDING, &req->flags);
136 req->fm = fm;
137 req->create_time = jiffies;
138 }
139
140 static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
141 {
142 struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);
143 if (req)
144 fuse_request_init(fm, req);
145
146 return req;
147 }
148
149 static void fuse_request_free(struct fuse_req *req)
150 {
151 kmem_cache_free(fuse_req_cachep, req);
152 }
153
154 static void __fuse_get_request(struct fuse_req *req)
155 {
156 refcount_inc(&req->count);
157 }
158
159 /* Must be called with > 1 refcount */
160 static void __fuse_put_request(struct fuse_req *req)
161 {
162 refcount_dec(&req->count);
163 }
164
165 void fuse_set_initialized(struct fuse_conn *fc)
166 {
167 /* Make sure stores before this are seen on another CPU */
168 smp_wmb();
169 fc->initialized = 1;
170 }
171
172 static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
173 {
174 return !fc->initialized || (for_background && fc->blocked) ||
175 (fc->io_uring && fc->connected && !fuse_uring_ready(fc));
176 }
177
178 static void fuse_drop_waiting(struct fuse_conn *fc)
179 {
180 /*
181 * lockess check of fc->connected is okay, because atomic_dec_and_test()
182 * provides a memory barrier matched with the one in fuse_wait_aborted()
183 * to ensure no wake-up is missed.
184 */
185 if (atomic_dec_and_test(&fc->num_waiting) &&
186 !READ_ONCE(fc->connected)) {
187 /* wake up aborters */
188 wake_up_all(&fc->blocked_waitq);
189 }
190 }
191
192 static void fuse_put_request(struct fuse_req *req);
193
194 static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap,
195 struct fuse_mount *fm,
196 bool for_background)
197 {
198 struct fuse_conn *fc = fm->fc;
199 struct fuse_req *req;
200 bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP);
201 kuid_t fsuid;
202 kgid_t fsgid;
203 int err;
204
205 atomic_inc(&fc->num_waiting);
206
207 if (fuse_block_alloc(fc, for_background)) {
208 err = -EINTR;
209 if (wait_event_state_exclusive(fc->blocked_waitq,
210 !fuse_block_alloc(fc, for_background),
211 (TASK_KILLABLE | TASK_FREEZABLE)))
212 goto out;
213 }
214 /* Matches smp_wmb() in fuse_set_initialized() */
215 smp_rmb();
216
217 err = -ENOTCONN;
218 if (!fc->connected)
219 goto out;
220
221 err = -ECONNREFUSED;
222 if (fc->conn_error)
223 goto out;
224
225 req = fuse_request_alloc(fm, GFP_KERNEL);
226 err = -ENOMEM;
227 if (!req) {
228 if (for_background)
229 wake_up(&fc->blocked_waitq);
230 goto out;
231 }
232
233 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
234
235 __set_bit(FR_WAITING, &req->flags);
236 if (for_background)
237 __set_bit(FR_BACKGROUND, &req->flags);
238
239 /*
240 * Keep the old behavior when idmappings support was not
241 * declared by a FUSE server.
242 *
243 * For those FUSE servers who support idmapped mounts,
244 * we send UID/GID only along with "inode creation"
245 * fuse requests, otherwise idmap == &invalid_mnt_idmap and
246 * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID.
247 */
248 fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns);
249 fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns);
250 req->in.h.uid = from_kuid(fc->user_ns, fsuid);
251 req->in.h.gid = from_kgid(fc->user_ns, fsgid);
252
253 if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) ||
254 req->in.h.gid == ((gid_t)-1))) {
255 fuse_put_request(req);
256 return ERR_PTR(-EOVERFLOW);
257 }
258
259 return req;
260
261 out:
262 fuse_drop_waiting(fc);
263 return ERR_PTR(err);
264 }
265
266 static void fuse_put_request(struct fuse_req *req)
267 {
268 struct fuse_conn *fc = req->fm->fc;
269
270 if (refcount_dec_and_test(&req->count)) {
271 if (test_bit(FR_BACKGROUND, &req->flags)) {
272 /*
273 * We get here in the unlikely case that a background
274 * request was allocated but not sent
275 */
276 spin_lock(&fc->bg_lock);
277 if (!fc->blocked)
278 wake_up(&fc->blocked_waitq);
279 spin_unlock(&fc->bg_lock);
280 }
281
282 if (test_bit(FR_WAITING, &req->flags)) {
283 __clear_bit(FR_WAITING, &req->flags);
284 fuse_drop_waiting(fc);
285 }
286
287 fuse_request_free(req);
288 }
289 }
290
291 unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args)
292 {
293 unsigned nbytes = 0;
294 unsigned i;
295
296 for (i = 0; i < numargs; i++)
297 nbytes += args[i].size;
298
299 return nbytes;
300 }
301 EXPORT_SYMBOL_GPL(fuse_len_args);
302
303 static u64 fuse_get_unique_locked(struct fuse_iqueue *fiq)
304 {
305 fiq->reqctr += FUSE_REQ_ID_STEP;
306 return fiq->reqctr;
307 }
308
309 u64 fuse_get_unique(struct fuse_iqueue *fiq)
310 {
311 u64 ret;
312
313 spin_lock(&fiq->lock);
314 ret = fuse_get_unique_locked(fiq);
315 spin_unlock(&fiq->lock);
316
317 return ret;
318 }
319 EXPORT_SYMBOL_GPL(fuse_get_unique);
320
321 unsigned int fuse_req_hash(u64 unique)
322 {
323 return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS);
324 }
325 EXPORT_SYMBOL_GPL(fuse_req_hash);
326
327 /*
328 * A new request is available, wake fiq->waitq
329 */
330 static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
331 __releases(fiq->lock)
332 {
333 wake_up(&fiq->waitq);
334 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
335 spin_unlock(&fiq->lock);
336 }
337
338 void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
339 struct fuse_forget_link *forget)
340 {
341 spin_lock(&fiq->lock);
342 if (fiq->connected) {
343 fiq->forget_list_tail->next = forget;
344 fiq->forget_list_tail = forget;
345 fuse_dev_wake_and_unlock(fiq);
346 } else {
347 kfree(forget);
348 spin_unlock(&fiq->lock);
349 }
350 }
351
352 void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
353 {
354 spin_lock(&fiq->lock);
355 if (list_empty(&req->intr_entry)) {
356 list_add_tail(&req->intr_entry, &fiq->interrupts);
357 /*
358 * Pairs with smp_mb() implied by test_and_set_bit()
359 * from fuse_request_end().
360 */
361 smp_mb();
362 if (test_bit(FR_FINISHED, &req->flags)) {
363 list_del_init(&req->intr_entry);
364 spin_unlock(&fiq->lock);
365 } else {
366 fuse_dev_wake_and_unlock(fiq);
367 }
368 } else {
369 spin_unlock(&fiq->lock);
370 }
371 }
372
373 static inline void fuse_request_assign_unique_locked(struct fuse_iqueue *fiq,
374 struct fuse_req *req)
375 {
376 if (req->in.h.opcode != FUSE_NOTIFY_REPLY)
377 req->in.h.unique = fuse_get_unique_locked(fiq);
378
379 /* tracepoint captures in.h.unique and in.h.len */
380 trace_fuse_request_send(req);
381 }
382
383 inline void fuse_request_assign_unique(struct fuse_iqueue *fiq,
384 struct fuse_req *req)
385 {
386 if (req->in.h.opcode != FUSE_NOTIFY_REPLY)
387 req->in.h.unique = fuse_get_unique(fiq);
388
389 /* tracepoint captures in.h.unique and in.h.len */
390 trace_fuse_request_send(req);
391 }
392 EXPORT_SYMBOL_GPL(fuse_request_assign_unique);
393
394 static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req)
395 {
396 spin_lock(&fiq->lock);
397 if (fiq->connected) {
398 fuse_request_assign_unique_locked(fiq, req);
399 list_add_tail(&req->list, &fiq->pending);
400 fuse_dev_wake_and_unlock(fiq);
401 } else {
402 spin_unlock(&fiq->lock);
403 req->out.h.error = -ENOTCONN;
404 clear_bit(FR_PENDING, &req->flags);
405 fuse_request_end(req);
406 }
407 }
408
409 const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
410 .send_forget = fuse_dev_queue_forget,
411 .send_interrupt = fuse_dev_queue_interrupt,
412 .send_req = fuse_dev_queue_req,
413 };
414 EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
415
416 static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req)
417 {
418 req->in.h.len = sizeof(struct fuse_in_header) +
419 fuse_len_args(req->args->in_numargs,
420 (struct fuse_arg *) req->args->in_args);
421 fiq->ops->send_req(fiq, req);
422 }
423
424 void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
425 u64 nodeid, u64 nlookup)
426 {
427 struct fuse_iqueue *fiq = &fc->iq;
428
429 forget->forget_one.nodeid = nodeid;
430 forget->forget_one.nlookup = nlookup;
431
432 fiq->ops->send_forget(fiq, forget);
433 }
434
435 static void flush_bg_queue(struct fuse_conn *fc)
436 {
437 struct fuse_iqueue *fiq = &fc->iq;
438
439 while (fc->active_background < fc->max_background &&
440 !list_empty(&fc->bg_queue)) {
441 struct fuse_req *req;
442
443 req = list_first_entry(&fc->bg_queue, struct fuse_req, list);
444 list_del(&req->list);
445 fc->active_background++;
446 fuse_send_one(fiq, req);
447 }
448 }
449
450 /*
451 * This function is called when a request is finished. Either a reply
452 * has arrived or it was aborted (and not yet sent) or some error
453 * occurred during communication with userspace, or the device file
454 * was closed. The requester thread is woken up (if still waiting),
455 * the 'end' callback is called if given, else the reference to the
456 * request is released
457 */
458 void fuse_request_end(struct fuse_req *req)
459 {
460 struct fuse_mount *fm = req->fm;
461 struct fuse_conn *fc = fm->fc;
462 struct fuse_iqueue *fiq = &fc->iq;
463
464 if (test_and_set_bit(FR_FINISHED, &req->flags))
465 goto put_request;
466
467 trace_fuse_request_end(req);
468 /*
469 * test_and_set_bit() implies smp_mb() between bit
470 * changing and below FR_INTERRUPTED check. Pairs with
471 * smp_mb() from queue_interrupt().
472 */
473 if (test_bit(FR_INTERRUPTED, &req->flags)) {
474 spin_lock(&fiq->lock);
475 list_del_init(&req->intr_entry);
476 spin_unlock(&fiq->lock);
477 }
478 WARN_ON(test_bit(FR_PENDING, &req->flags));
479 WARN_ON(test_bit(FR_SENT, &req->flags));
480 if (test_bit(FR_BACKGROUND, &req->flags)) {
481 spin_lock(&fc->bg_lock);
482 clear_bit(FR_BACKGROUND, &req->flags);
483 if (fc->num_background == fc->max_background) {
484 fc->blocked = 0;
485 wake_up(&fc->blocked_waitq);
486 } else if (!fc->blocked) {
487 /*
488 * Wake up next waiter, if any. It's okay to use
489 * waitqueue_active(), as we've already synced up
490 * fc->blocked with waiters with the wake_up() call
491 * above.
492 */
493 if (waitqueue_active(&fc->blocked_waitq))
494 wake_up(&fc->blocked_waitq);
495 }
496
497 fc->num_background--;
498 fc->active_background--;
499 flush_bg_queue(fc);
500 spin_unlock(&fc->bg_lock);
501 } else {
502 /* Wake up waiter sleeping in request_wait_answer() */
503 wake_up(&req->waitq);
504 }
505
506 if (test_bit(FR_ASYNC, &req->flags))
507 req->args->end(fm, req->args, req->out.h.error);
508 put_request:
509 fuse_put_request(req);
510 }
511 EXPORT_SYMBOL_GPL(fuse_request_end);
512
513 static int queue_interrupt(struct fuse_req *req)
514 {
515 struct fuse_iqueue *fiq = &req->fm->fc->iq;
516
517 /* Check for we've sent request to interrupt this req */
518 if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags)))
519 return -EINVAL;
520
521 fiq->ops->send_interrupt(fiq, req);
522
523 return 0;
524 }
525
526 bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock)
527 {
528 spin_lock(lock);
529 if (test_bit(FR_PENDING, &req->flags)) {
530 /*
531 * FR_PENDING does not get cleared as the request will end
532 * up in destruction anyway.
533 */
534 list_del(&req->list);
535 spin_unlock(lock);
536 __fuse_put_request(req);
537 req->out.h.error = -EINTR;
538 return true;
539 }
540 spin_unlock(lock);
541 return false;
542 }
543
544 static void request_wait_answer(struct fuse_req *req)
545 {
546 struct fuse_conn *fc = req->fm->fc;
547 struct fuse_iqueue *fiq = &fc->iq;
548 int err;
549
550 if (!fc->no_interrupt) {
551 /* Any signal may interrupt this */
552 err = wait_event_interruptible(req->waitq,
553 test_bit(FR_FINISHED, &req->flags));
554 if (!err)
555 return;
556
557 set_bit(FR_INTERRUPTED, &req->flags);
558 /* matches barrier in fuse_dev_do_read() */
559 smp_mb__after_atomic();
560 if (test_bit(FR_SENT, &req->flags))
561 queue_interrupt(req);
562 }
563
564 if (!test_bit(FR_FORCE, &req->flags)) {
565 bool removed;
566
567 /* Only fatal signals may interrupt this */
568 err = wait_event_killable(req->waitq,
569 test_bit(FR_FINISHED, &req->flags));
570 if (!err)
571 return;
572
573 if (req->args->abort_on_kill) {
574 fuse_abort_conn(fc);
575 return;
576 }
577
578 if (test_bit(FR_URING, &req->flags))
579 removed = fuse_uring_remove_pending_req(req);
580 else
581 removed = fuse_remove_pending_req(req, &fiq->lock);
582 if (removed)
583 return;
584 }
585
586 /*
587 * Either request is already in userspace, or it was forced.
588 * Wait it out.
589 */
590 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
591 }
592
593 static void __fuse_request_send(struct fuse_req *req)
594 {
595 struct fuse_iqueue *fiq = &req->fm->fc->iq;
596
597 BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
598
599 /* acquire extra reference, since request is still needed after
600 fuse_request_end() */
601 __fuse_get_request(req);
602 fuse_send_one(fiq, req);
603
604 request_wait_answer(req);
605 /* Pairs with smp_wmb() in fuse_request_end() */
606 smp_rmb();
607 }
608
609 static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
610 {
611 if (fc->minor < 4 && args->opcode == FUSE_STATFS)
612 args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE;
613
614 if (fc->minor < 9) {
615 switch (args->opcode) {
616 case FUSE_LOOKUP:
617 case FUSE_CREATE:
618 case FUSE_MKNOD:
619 case FUSE_MKDIR:
620 case FUSE_SYMLINK:
621 case FUSE_LINK:
622 args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
623 break;
624 case FUSE_GETATTR:
625 case FUSE_SETATTR:
626 args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
627 break;
628 }
629 }
630 if (fc->minor < 12) {
631 switch (args->opcode) {
632 case FUSE_CREATE:
633 args->in_args[0].size = sizeof(struct fuse_open_in);
634 break;
635 case FUSE_MKNOD:
636 args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
637 break;
638 }
639 }
640 }
641
642 static void fuse_force_creds(struct fuse_req *req)
643 {
644 struct fuse_conn *fc = req->fm->fc;
645
646 if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) {
647 req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
648 req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
649 } else {
650 req->in.h.uid = FUSE_INVALID_UIDGID;
651 req->in.h.gid = FUSE_INVALID_UIDGID;
652 }
653
654 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
655 }
656
657 static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
658 {
659 req->in.h.opcode = args->opcode;
660 req->in.h.nodeid = args->nodeid;
661 req->args = args;
662 if (args->is_ext)
663 req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8;
664 if (args->end)
665 __set_bit(FR_ASYNC, &req->flags);
666 }
667
668 ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
669 struct fuse_mount *fm,
670 struct fuse_args *args)
671 {
672 struct fuse_conn *fc = fm->fc;
673 struct fuse_req *req;
674 ssize_t ret;
675
676 if (args->force) {
677 atomic_inc(&fc->num_waiting);
678 req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL);
679
680 if (!args->nocreds)
681 fuse_force_creds(req);
682
683 __set_bit(FR_WAITING, &req->flags);
684 if (!args->abort_on_kill)
685 __set_bit(FR_FORCE, &req->flags);
686 } else {
687 WARN_ON(args->nocreds);
688 req = fuse_get_req(idmap, fm, false);
689 if (IS_ERR(req))
690 return PTR_ERR(req);
691 }
692
693 /* Needs to be done after fuse_get_req() so that fc->minor is valid */
694 fuse_adjust_compat(fc, args);
695 fuse_args_to_req(req, args);
696
697 if (!args->noreply)
698 __set_bit(FR_ISREPLY, &req->flags);
699 __fuse_request_send(req);
700 ret = req->out.h.error;
701 if (!ret && args->out_argvar) {
702 BUG_ON(args->out_numargs == 0);
703 ret = args->out_args[args->out_numargs - 1].size;
704 }
705 fuse_put_request(req);
706
707 return ret;
708 }
709
710 #ifdef CONFIG_FUSE_IO_URING
711 static bool fuse_request_queue_background_uring(struct fuse_conn *fc,
712 struct fuse_req *req)
713 {
714 struct fuse_iqueue *fiq = &fc->iq;
715
716 req->in.h.len = sizeof(struct fuse_in_header) +
717 fuse_len_args(req->args->in_numargs,
718 (struct fuse_arg *) req->args->in_args);
719 fuse_request_assign_unique(fiq, req);
720
721 return fuse_uring_queue_bq_req(req);
722 }
723 #endif
724
725 /*
726 * @return true if queued
727 */
728 static int fuse_request_queue_background(struct fuse_req *req)
729 {
730 struct fuse_mount *fm = req->fm;
731 struct fuse_conn *fc = fm->fc;
732 bool queued = false;
733
734 WARN_ON(!test_bit(FR_BACKGROUND, &req->flags));
735 if (!test_bit(FR_WAITING, &req->flags)) {
736 __set_bit(FR_WAITING, &req->flags);
737 atomic_inc(&fc->num_waiting);
738 }
739 __set_bit(FR_ISREPLY, &req->flags);
740
741 #ifdef CONFIG_FUSE_IO_URING
742 if (fuse_uring_ready(fc))
743 return fuse_request_queue_background_uring(fc, req);
744 #endif
745
746 spin_lock(&fc->bg_lock);
747 if (likely(fc->connected)) {
748 fc->num_background++;
749 if (fc->num_background == fc->max_background)
750 fc->blocked = 1;
751 list_add_tail(&req->list, &fc->bg_queue);
752 flush_bg_queue(fc);
753 queued = true;
754 }
755 spin_unlock(&fc->bg_lock);
756
757 return queued;
758 }
759
760 int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
761 gfp_t gfp_flags)
762 {
763 struct fuse_req *req;
764
765 if (args->force) {
766 WARN_ON(!args->nocreds);
767 req = fuse_request_alloc(fm, gfp_flags);
768 if (!req)
769 return -ENOMEM;
770 __set_bit(FR_BACKGROUND, &req->flags);
771 } else {
772 WARN_ON(args->nocreds);
773 req = fuse_get_req(&invalid_mnt_idmap, fm, true);
774 if (IS_ERR(req))
775 return PTR_ERR(req);
776 }
777
778 fuse_args_to_req(req, args);
779
780 if (!fuse_request_queue_background(req)) {
781 fuse_put_request(req);
782 return -ENOTCONN;
783 }
784
785 return 0;
786 }
787 EXPORT_SYMBOL_GPL(fuse_simple_background);
788
789 static int fuse_simple_notify_reply(struct fuse_mount *fm,
790 struct fuse_args *args, u64 unique)
791 {
792 struct fuse_req *req;
793 struct fuse_iqueue *fiq = &fm->fc->iq;
794
795 req = fuse_get_req(&invalid_mnt_idmap, fm, false);
796 if (IS_ERR(req))
797 return PTR_ERR(req);
798
799 __clear_bit(FR_ISREPLY, &req->flags);
800 req->in.h.unique = unique;
801
802 fuse_args_to_req(req, args);
803
804 fuse_send_one(fiq, req);
805
806 return 0;
807 }
808
809 /*
810 * Lock the request. Up to the next unlock_request() there mustn't be
811 * anything that could cause a page-fault. If the request was already
812 * aborted bail out.
813 */
814 static int lock_request(struct fuse_req *req)
815 {
816 int err = 0;
817 if (req) {
818 spin_lock(&req->waitq.lock);
819 if (test_bit(FR_ABORTED, &req->flags))
820 err = -ENOENT;
821 else
822 set_bit(FR_LOCKED, &req->flags);
823 spin_unlock(&req->waitq.lock);
824 }
825 return err;
826 }
827
828 /*
829 * Unlock request. If it was aborted while locked, caller is responsible
830 * for unlocking and ending the request.
831 */
832 static int unlock_request(struct fuse_req *req)
833 {
834 int err = 0;
835 if (req) {
836 spin_lock(&req->waitq.lock);
837 if (test_bit(FR_ABORTED, &req->flags))
838 err = -ENOENT;
839 else
840 clear_bit(FR_LOCKED, &req->flags);
841 spin_unlock(&req->waitq.lock);
842 }
843 return err;
844 }
845
846 void fuse_copy_init(struct fuse_copy_state *cs, bool write,
847 struct iov_iter *iter)
848 {
849 memset(cs, 0, sizeof(*cs));
850 cs->write = write;
851 cs->iter = iter;
852 }
853
854 /* Unmap and put previous page of userspace buffer */
855 void fuse_copy_finish(struct fuse_copy_state *cs)
856 {
857 if (cs->currbuf) {
858 struct pipe_buffer *buf = cs->currbuf;
859
860 if (cs->write)
861 buf->len = PAGE_SIZE - cs->len;
862 cs->currbuf = NULL;
863 } else if (cs->pg) {
864 if (cs->write) {
865 flush_dcache_page(cs->pg);
866 set_page_dirty_lock(cs->pg);
867 }
868 put_page(cs->pg);
869 }
870 cs->pg = NULL;
871 }
872
873 /*
874 * Get another pagefull of userspace buffer, and map it to kernel
875 * address space, and lock request
876 */
877 static int fuse_copy_fill(struct fuse_copy_state *cs)
878 {
879 struct page *page;
880 int err;
881
882 err = unlock_request(cs->req);
883 if (err)
884 return err;
885
886 fuse_copy_finish(cs);
887 if (cs->pipebufs) {
888 struct pipe_buffer *buf = cs->pipebufs;
889
890 if (!cs->write) {
891 err = pipe_buf_confirm(cs->pipe, buf);
892 if (err)
893 return err;
894
895 BUG_ON(!cs->nr_segs);
896 cs->currbuf = buf;
897 cs->pg = buf->page;
898 cs->offset = buf->offset;
899 cs->len = buf->len;
900 cs->pipebufs++;
901 cs->nr_segs--;
902 } else {
903 if (cs->nr_segs >= cs->pipe->max_usage)
904 return -EIO;
905
906 page = alloc_page(GFP_HIGHUSER);
907 if (!page)
908 return -ENOMEM;
909
910 buf->page = page;
911 buf->offset = 0;
912 buf->len = 0;
913
914 cs->currbuf = buf;
915 cs->pg = page;
916 cs->offset = 0;
917 cs->len = PAGE_SIZE;
918 cs->pipebufs++;
919 cs->nr_segs++;
920 }
921 } else {
922 size_t off;
923 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off);
924 if (err < 0)
925 return err;
926 BUG_ON(!err);
927 cs->len = err;
928 cs->offset = off;
929 cs->pg = page;
930 }
931
932 return lock_request(cs->req);
933 }
934
935 /* Do as much copy to/from userspace buffer as we can */
936 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
937 {
938 unsigned ncpy = min(*size, cs->len);
939 if (val) {
940 void *pgaddr = kmap_local_page(cs->pg);
941 void *buf = pgaddr + cs->offset;
942
943 if (cs->write)
944 memcpy(buf, *val, ncpy);
945 else
946 memcpy(*val, buf, ncpy);
947
948 kunmap_local(pgaddr);
949 *val += ncpy;
950 }
951 *size -= ncpy;
952 cs->len -= ncpy;
953 cs->offset += ncpy;
954 if (cs->is_uring)
955 cs->ring.copied_sz += ncpy;
956
957 return ncpy;
958 }
959
960 static int fuse_check_folio(struct folio *folio)
961 {
962 if (folio_mapped(folio) ||
963 folio->mapping != NULL ||
964 (folio->flags.f & PAGE_FLAGS_CHECK_AT_PREP &
965 ~(1 << PG_locked |
966 1 << PG_referenced |
967 1 << PG_lru |
968 1 << PG_active |
969 1 << PG_workingset |
970 1 << PG_reclaim |
971 1 << PG_waiters |
972 LRU_GEN_MASK | LRU_REFS_MASK))) {
973 dump_page(&folio->page, "fuse: trying to steal weird page");
974 return 1;
975 }
976 return 0;
977 }
978
979 /*
980 * Attempt to steal a page from the splice() pipe and move it into the
981 * pagecache. If successful, the pointer in @pagep will be updated. The
982 * folio that was originally in @pagep will lose a reference and the new
983 * folio returned in @pagep will carry a reference.
984 */
985 static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop)
986 {
987 int err;
988 struct folio *oldfolio = *foliop;
989 struct folio *newfolio;
990 struct pipe_buffer *buf = cs->pipebufs;
991
992 folio_get(oldfolio);
993 err = unlock_request(cs->req);
994 if (err)
995 goto out_put_old;
996
997 fuse_copy_finish(cs);
998
999 err = pipe_buf_confirm(cs->pipe, buf);
1000 if (err)
1001 goto out_put_old;
1002
1003 BUG_ON(!cs->nr_segs);
1004 cs->currbuf = buf;
1005 cs->len = buf->len;
1006 cs->pipebufs++;
1007 cs->nr_segs--;
1008
1009 if (cs->len != folio_size(oldfolio))
1010 goto out_fallback;
1011
1012 if (!pipe_buf_try_steal(cs->pipe, buf))
1013 goto out_fallback;
1014
1015 newfolio = page_folio(buf->page);
1016
1017 folio_clear_uptodate(newfolio);
1018 folio_clear_mappedtodisk(newfolio);
1019
1020 if (folio_test_large(newfolio))
1021 goto out_fallback_unlock;
1022
1023 if (fuse_check_folio(newfolio) != 0)
1024 goto out_fallback_unlock;
1025
1026 /*
1027 * This is a new and locked page, it shouldn't be mapped or
1028 * have any special flags on it
1029 */
1030 if (WARN_ON(folio_mapped(oldfolio)))
1031 goto out_fallback_unlock;
1032 if (WARN_ON(folio_has_private(oldfolio)))
1033 goto out_fallback_unlock;
1034 if (WARN_ON(folio_test_dirty(oldfolio) ||
1035 folio_test_writeback(oldfolio)))
1036 goto out_fallback_unlock;
1037 if (WARN_ON(folio_test_mlocked(oldfolio)))
1038 goto out_fallback_unlock;
1039
1040 replace_page_cache_folio(oldfolio, newfolio);
1041
1042 folio_get(newfolio);
1043
1044 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
1045 folio_add_lru(newfolio);
1046
1047 /*
1048 * Release while we have extra ref on stolen page. Otherwise
1049 * anon_pipe_buf_release() might think the page can be reused.
1050 */
1051 pipe_buf_release(cs->pipe, buf);
1052
1053 err = 0;
1054 spin_lock(&cs->req->waitq.lock);
1055 if (test_bit(FR_ABORTED, &cs->req->flags))
1056 err = -ENOENT;
1057 else
1058 *foliop = newfolio;
1059 spin_unlock(&cs->req->waitq.lock);
1060
1061 if (err) {
1062 folio_unlock(newfolio);
1063 folio_put(newfolio);
1064 goto out_put_old;
1065 }
1066
1067 folio_unlock(oldfolio);
1068 /* Drop ref for ap->pages[] array */
1069 folio_put(oldfolio);
1070 cs->len = 0;
1071
1072 err = 0;
1073 out_put_old:
1074 /* Drop ref obtained in this function */
1075 folio_put(oldfolio);
1076 return err;
1077
1078 out_fallback_unlock:
1079 folio_unlock(newfolio);
1080 out_fallback:
1081 cs->pg = buf->page;
1082 cs->offset = buf->offset;
1083
1084 err = lock_request(cs->req);
1085 if (!err)
1086 err = 1;
1087
1088 goto out_put_old;
1089 }
1090
1091 static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,
1092 unsigned offset, unsigned count)
1093 {
1094 struct pipe_buffer *buf;
1095 int err;
1096
1097 if (cs->nr_segs >= cs->pipe->max_usage)
1098 return -EIO;
1099
1100 folio_get(folio);
1101 err = unlock_request(cs->req);
1102 if (err) {
1103 folio_put(folio);
1104 return err;
1105 }
1106
1107 fuse_copy_finish(cs);
1108
1109 buf = cs->pipebufs;
1110 buf->page = &folio->page;
1111 buf->offset = offset;
1112 buf->len = count;
1113
1114 cs->pipebufs++;
1115 cs->nr_segs++;
1116 cs->len = 0;
1117
1118 return 0;
1119 }
1120
1121 /*
1122 * Copy a folio in the request to/from the userspace buffer. Must be
1123 * done atomically
1124 */
1125 static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
1126 unsigned offset, unsigned count, int zeroing)
1127 {
1128 int err;
1129 struct folio *folio = *foliop;
1130 size_t size;
1131
1132 if (folio) {
1133 size = folio_size(folio);
1134 if (zeroing && count < size)
1135 folio_zero_range(folio, 0, size);
1136 }
1137
1138 while (count) {
1139 if (cs->write && cs->pipebufs && folio) {
1140 /*
1141 * Can't control lifetime of pipe buffers, so always
1142 * copy user pages.
1143 */
1144 if (cs->req->args->user_pages) {
1145 err = fuse_copy_fill(cs);
1146 if (err)
1147 return err;
1148 } else {
1149 return fuse_ref_folio(cs, folio, offset, count);
1150 }
1151 } else if (!cs->len) {
1152 if (cs->move_folios && folio &&
1153 offset == 0 && count == size) {
1154 err = fuse_try_move_folio(cs, foliop);
1155 if (err <= 0)
1156 return err;
1157 } else {
1158 err = fuse_copy_fill(cs);
1159 if (err)
1160 return err;
1161 }
1162 }
1163 if (folio) {
1164 void *mapaddr = kmap_local_folio(folio, offset);
1165 void *buf = mapaddr;
1166 unsigned int copy = count;
1167 unsigned int bytes_copied;
1168
1169 if (folio_test_highmem(folio) && count > PAGE_SIZE - offset_in_page(offset))
1170 copy = PAGE_SIZE - offset_in_page(offset);
1171
1172 bytes_copied = fuse_copy_do(cs, &buf, &copy);
1173 kunmap_local(mapaddr);
1174 offset += bytes_copied;
1175 count -= bytes_copied;
1176 } else
1177 offset += fuse_copy_do(cs, NULL, &count);
1178 }
1179 if (folio && !cs->write)
1180 flush_dcache_folio(folio);
1181 return 0;
1182 }
1183
1184 /* Copy folios in the request to/from userspace buffer */
1185 static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes,
1186 int zeroing)
1187 {
1188 unsigned i;
1189 struct fuse_req *req = cs->req;
1190 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
1191
1192 for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) {
1193 int err;
1194 unsigned int offset = ap->descs[i].offset;
1195 unsigned int count = min(nbytes, ap->descs[i].length);
1196
1197 err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing);
1198 if (err)
1199 return err;
1200
1201 nbytes -= count;
1202 }
1203 return 0;
1204 }
1205
1206 /* Copy a single argument in the request to/from userspace buffer */
1207 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1208 {
1209 while (size) {
1210 if (!cs->len) {
1211 int err = fuse_copy_fill(cs);
1212 if (err)
1213 return err;
1214 }
1215 fuse_copy_do(cs, &val, &size);
1216 }
1217 return 0;
1218 }
1219
1220 /* Copy request arguments to/from userspace buffer */
1221 int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1222 unsigned argpages, struct fuse_arg *args,
1223 int zeroing)
1224 {
1225 int err = 0;
1226 unsigned i;
1227
1228 for (i = 0; !err && i < numargs; i++) {
1229 struct fuse_arg *arg = &args[i];
1230 if (i == numargs - 1 && argpages)
1231 err = fuse_copy_folios(cs, arg->size, zeroing);
1232 else
1233 err = fuse_copy_one(cs, arg->value, arg->size);
1234 }
1235 return err;
1236 }
1237
1238 static int forget_pending(struct fuse_iqueue *fiq)
1239 {
1240 return fiq->forget_list_head.next != NULL;
1241 }
1242
1243 static int request_pending(struct fuse_iqueue *fiq)
1244 {
1245 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1246 forget_pending(fiq);
1247 }
1248
1249 /*
1250 * Transfer an interrupt request to userspace
1251 *
1252 * Unlike other requests this is assembled on demand, without a need
1253 * to allocate a separate fuse_req structure.
1254 *
1255 * Called with fiq->lock held, releases it
1256 */
1257 static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1258 struct fuse_copy_state *cs,
1259 size_t nbytes, struct fuse_req *req)
1260 __releases(fiq->lock)
1261 {
1262 struct fuse_in_header ih;
1263 struct fuse_interrupt_in arg;
1264 unsigned reqsize = sizeof(ih) + sizeof(arg);
1265 int err;
1266
1267 list_del_init(&req->intr_entry);
1268 memset(&ih, 0, sizeof(ih));
1269 memset(&arg, 0, sizeof(arg));
1270 ih.len = reqsize;
1271 ih.opcode = FUSE_INTERRUPT;
1272 ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT);
1273 arg.unique = req->in.h.unique;
1274
1275 spin_unlock(&fiq->lock);
1276 if (nbytes < reqsize)
1277 return -EINVAL;
1278
1279 err = fuse_copy_one(cs, &ih, sizeof(ih));
1280 if (!err)
1281 err = fuse_copy_one(cs, &arg, sizeof(arg));
1282 fuse_copy_finish(cs);
1283
1284 return err ? err : reqsize;
1285 }
1286
1287 static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
1288 unsigned int max,
1289 unsigned int *countp)
1290 {
1291 struct fuse_forget_link *head = fiq->forget_list_head.next;
1292 struct fuse_forget_link **newhead = &head;
1293 unsigned count;
1294
1295 for (count = 0; *newhead != NULL && count < max; count++)
1296 newhead = &(*newhead)->next;
1297
1298 fiq->forget_list_head.next = *newhead;
1299 *newhead = NULL;
1300 if (fiq->forget_list_head.next == NULL)
1301 fiq->forget_list_tail = &fiq->forget_list_head;
1302
1303 if (countp != NULL)
1304 *countp = count;
1305
1306 return head;
1307 }
1308
1309 static int fuse_read_single_forget(struct fuse_iqueue *fiq,
1310 struct fuse_copy_state *cs,
1311 size_t nbytes)
1312 __releases(fiq->lock)
1313 {
1314 int err;
1315 struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
1316 struct fuse_forget_in arg = {
1317 .nlookup = forget->forget_one.nlookup,
1318 };
1319 struct fuse_in_header ih = {
1320 .opcode = FUSE_FORGET,
1321 .nodeid = forget->forget_one.nodeid,
1322 .unique = fuse_get_unique_locked(fiq),
1323 .len = sizeof(ih) + sizeof(arg),
1324 };
1325
1326 spin_unlock(&fiq->lock);
1327 kfree(forget);
1328 if (nbytes < ih.len)
1329 return -EINVAL;
1330
1331 err = fuse_copy_one(cs, &ih, sizeof(ih));
1332 if (!err)
1333 err = fuse_copy_one(cs, &arg, sizeof(arg));
1334 fuse_copy_finish(cs);
1335
1336 if (err)
1337 return err;
1338
1339 return ih.len;
1340 }
1341
1342 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
1343 struct fuse_copy_state *cs, size_t nbytes)
1344 __releases(fiq->lock)
1345 {
1346 int err;
1347 unsigned max_forgets;
1348 unsigned count;
1349 struct fuse_forget_link *head;
1350 struct fuse_batch_forget_in arg = { .count = 0 };
1351 struct fuse_in_header ih = {
1352 .opcode = FUSE_BATCH_FORGET,
1353 .unique = fuse_get_unique_locked(fiq),
1354 .len = sizeof(ih) + sizeof(arg),
1355 };
1356
1357 if (nbytes < ih.len) {
1358 spin_unlock(&fiq->lock);
1359 return -EINVAL;
1360 }
1361
1362 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1363 head = fuse_dequeue_forget(fiq, max_forgets, &count);
1364 spin_unlock(&fiq->lock);
1365
1366 arg.count = count;
1367 ih.len += count * sizeof(struct fuse_forget_one);
1368 err = fuse_copy_one(cs, &ih, sizeof(ih));
1369 if (!err)
1370 err = fuse_copy_one(cs, &arg, sizeof(arg));
1371
1372 while (head) {
1373 struct fuse_forget_link *forget = head;
1374
1375 if (!err) {
1376 err = fuse_copy_one(cs, &forget->forget_one,
1377 sizeof(forget->forget_one));
1378 }
1379 head = forget->next;
1380 kfree(forget);
1381 }
1382
1383 fuse_copy_finish(cs);
1384
1385 if (err)
1386 return err;
1387
1388 return ih.len;
1389 }
1390
1391 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1392 struct fuse_copy_state *cs,
1393 size_t nbytes)
1394 __releases(fiq->lock)
1395 {
1396 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
1397 return fuse_read_single_forget(fiq, cs, nbytes);
1398 else
1399 return fuse_read_batch_forget(fiq, cs, nbytes);
1400 }
1401
1402 /*
1403 * Read a single request into the userspace filesystem's buffer. This
1404 * function waits until a request is available, then removes it from
1405 * the pending list and copies request data to userspace buffer. If
1406 * no reply is needed (FORGET) or request has been aborted or there
1407 * was an error during the copying then it's finished by calling
1408 * fuse_request_end(). Otherwise add it to the processing list, and set
1409 * the 'sent' flag.
1410 */
1411 static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
1412 struct fuse_copy_state *cs, size_t nbytes)
1413 {
1414 ssize_t err;
1415 struct fuse_conn *fc = fud->fc;
1416 struct fuse_iqueue *fiq = &fc->iq;
1417 struct fuse_pqueue *fpq = &fud->pq;
1418 struct fuse_req *req;
1419 struct fuse_args *args;
1420 unsigned reqsize;
1421 unsigned int hash;
1422
1423 /*
1424 * Require sane minimum read buffer - that has capacity for fixed part
1425 * of any request header + negotiated max_write room for data.
1426 *
1427 * Historically libfuse reserves 4K for fixed header room, but e.g.
1428 * GlusterFS reserves only 80 bytes
1429 *
1430 * = `sizeof(fuse_in_header) + sizeof(fuse_write_in)`
1431 *
1432 * which is the absolute minimum any sane filesystem should be using
1433 * for header room.
1434 */
1435 if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER,
1436 sizeof(struct fuse_in_header) +
1437 sizeof(struct fuse_write_in) +
1438 fc->max_write))
1439 return -EINVAL;
1440
1441 restart:
1442 for (;;) {
1443 spin_lock(&fiq->lock);
1444 if (!fiq->connected || request_pending(fiq))
1445 break;
1446 spin_unlock(&fiq->lock);
1447
1448 if (file->f_flags & O_NONBLOCK)
1449 return -EAGAIN;
1450 err = wait_event_interruptible_exclusive(fiq->waitq,
1451 !fiq->connected || request_pending(fiq));
1452 if (err)
1453 return err;
1454 }
1455
1456 if (!fiq->connected) {
1457 err = fc->aborted ? -ECONNABORTED : -ENODEV;
1458 goto err_unlock;
1459 }
1460
1461 if (!list_empty(&fiq->interrupts)) {
1462 req = list_entry(fiq->interrupts.next, struct fuse_req,
1463 intr_entry);
1464 return fuse_read_interrupt(fiq, cs, nbytes, req);
1465 }
1466
1467 if (forget_pending(fiq)) {
1468 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
1469 return fuse_read_forget(fc, fiq, cs, nbytes);
1470
1471 if (fiq->forget_batch <= -8)
1472 fiq->forget_batch = 16;
1473 }
1474
1475 req = list_entry(fiq->pending.next, struct fuse_req, list);
1476 clear_bit(FR_PENDING, &req->flags);
1477 list_del_init(&req->list);
1478 spin_unlock(&fiq->lock);
1479
1480 args = req->args;
1481 reqsize = req->in.h.len;
1482
1483 /* If request is too large, reply with an error and restart the read */
1484 if (nbytes < reqsize) {
1485 req->out.h.error = -EIO;
1486 /* SETXATTR is special, since it may contain too large data */
1487 if (args->opcode == FUSE_SETXATTR)
1488 req->out.h.error = -E2BIG;
1489 fuse_request_end(req);
1490 goto restart;
1491 }
1492 spin_lock(&fpq->lock);
1493 /*
1494 * Must not put request on fpq->io queue after having been shut down by
1495 * fuse_abort_conn()
1496 */
1497 if (!fpq->connected) {
1498 req->out.h.error = err = -ECONNABORTED;
1499 goto out_end;
1500
1501 }
1502 list_add(&req->list, &fpq->io);
1503 spin_unlock(&fpq->lock);
1504 cs->req = req;
1505 err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h));
1506 if (!err)
1507 err = fuse_copy_args(cs, args->in_numargs, args->in_pages,
1508 (struct fuse_arg *) args->in_args, 0);
1509 fuse_copy_finish(cs);
1510 spin_lock(&fpq->lock);
1511 clear_bit(FR_LOCKED, &req->flags);
1512 if (!fpq->connected) {
1513 err = fc->aborted ? -ECONNABORTED : -ENODEV;
1514 goto out_end;
1515 }
1516 if (err) {
1517 req->out.h.error = -EIO;
1518 goto out_end;
1519 }
1520 if (!test_bit(FR_ISREPLY, &req->flags)) {
1521 err = reqsize;
1522 goto out_end;
1523 }
1524 hash = fuse_req_hash(req->in.h.unique);
1525 list_move_tail(&req->list, &fpq->processing[hash]);
1526 __fuse_get_request(req);
1527 set_bit(FR_SENT, &req->flags);
1528 spin_unlock(&fpq->lock);
1529 /* matches barrier in request_wait_answer() */
1530 smp_mb__after_atomic();
1531 if (test_bit(FR_INTERRUPTED, &req->flags))
1532 queue_interrupt(req);
1533 fuse_put_request(req);
1534
1535 return reqsize;
1536
1537 out_end:
1538 if (!test_bit(FR_PRIVATE, &req->flags))
1539 list_del_init(&req->list);
1540 spin_unlock(&fpq->lock);
1541 fuse_request_end(req);
1542 return err;
1543
1544 err_unlock:
1545 spin_unlock(&fiq->lock);
1546 return err;
1547 }
1548
1549 static int fuse_dev_open(struct inode *inode, struct file *file)
1550 {
1551 /*
1552 * The fuse device's file's private_data is used to hold
1553 * the fuse_conn(ection) when it is mounted, and is used to
1554 * keep track of whether the file has been mounted already.
1555 */
1556 file->private_data = NULL;
1557 return 0;
1558 }
1559
1560 struct fuse_dev *fuse_get_dev(struct file *file)
1561 {
1562 struct fuse_dev *fud = __fuse_get_dev(file);
1563 int err;
1564
1565 if (likely(fud))
1566 return fud;
1567
1568 err = wait_event_interruptible(fuse_dev_waitq,
1569 READ_ONCE(file->private_data) != FUSE_DEV_SYNC_INIT);
1570 if (err)
1571 return ERR_PTR(err);
1572
1573 fud = __fuse_get_dev(file);
1574 if (!fud)
1575 return ERR_PTR(-EPERM);
1576
1577 return fud;
1578 }
1579
1580 static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
1581 {
1582 struct fuse_copy_state cs;
1583 struct file *file = iocb->ki_filp;
1584 struct fuse_dev *fud = fuse_get_dev(file);
1585
1586 if (IS_ERR(fud))
1587 return PTR_ERR(fud);
1588
1589 if (!user_backed_iter(to))
1590 return -EINVAL;
1591
1592 fuse_copy_init(&cs, true, to);
1593
1594 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
1595 }
1596
1597 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1598 struct pipe_inode_info *pipe,
1599 size_t len, unsigned int flags)
1600 {
1601 int total, ret;
1602 int page_nr = 0;
1603 struct pipe_buffer *bufs;
1604 struct fuse_copy_state cs;
1605 struct fuse_dev *fud = fuse_get_dev(in);
1606
1607 if (IS_ERR(fud))
1608 return PTR_ERR(fud);
1609
1610 bufs = kvmalloc_objs(struct pipe_buffer, pipe->max_usage);
1611 if (!bufs)
1612 return -ENOMEM;
1613
1614 fuse_copy_init(&cs, true, NULL);
1615 cs.pipebufs = bufs;
1616 cs.pipe = pipe;
1617 ret = fuse_dev_do_read(fud, in, &cs, len);
1618 if (ret < 0)
1619 goto out;
1620
1621 if (pipe_buf_usage(pipe) + cs.nr_segs > pipe->max_usage) {
1622 ret = -EIO;
1623 goto out;
1624 }
1625
1626 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
1627 /*
1628 * Need to be careful about this. Having buf->ops in module
1629 * code can Oops if the buffer persists after module unload.
1630 */
1631 bufs[page_nr].ops = &nosteal_pipe_buf_ops;
1632 bufs[page_nr].flags = 0;
1633 ret = add_to_pipe(pipe, &bufs[page_nr++]);
1634 if (unlikely(ret < 0))
1635 break;
1636 }
1637 if (total)
1638 ret = total;
1639 out:
1640 for (; page_nr < cs.nr_segs; page_nr++)
1641 put_page(bufs[page_nr].page);
1642
1643 kvfree(bufs);
1644 return ret;
1645 }
1646
1647 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1648 struct fuse_copy_state *cs)
1649 {
1650 struct fuse_notify_poll_wakeup_out outarg;
1651 int err;
1652
1653 if (size != sizeof(outarg))
1654 return -EINVAL;
1655
1656 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1657 if (err)
1658 return err;
1659
1660 fuse_copy_finish(cs);
1661 return fuse_notify_poll_wakeup(fc, &outarg);
1662 }
1663
1664 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1665 struct fuse_copy_state *cs)
1666 {
1667 struct fuse_notify_inval_inode_out outarg;
1668 int err;
1669
1670 if (size != sizeof(outarg))
1671 return -EINVAL;
1672
1673 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1674 if (err)
1675 return err;
1676 fuse_copy_finish(cs);
1677
1678 down_read(&fc->killsb);
1679 err = fuse_reverse_inval_inode(fc, outarg.ino,
1680 outarg.off, outarg.len);
1681 up_read(&fc->killsb);
1682 return err;
1683 }
1684
1685 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1686 struct fuse_copy_state *cs)
1687 {
1688 struct fuse_notify_inval_entry_out outarg;
1689 int err;
1690 char *buf;
1691 struct qstr name;
1692
1693 if (size < sizeof(outarg))
1694 return -EINVAL;
1695
1696 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1697 if (err)
1698 return err;
1699
1700 if (outarg.namelen > fc->name_max)
1701 return -ENAMETOOLONG;
1702
1703 err = -EINVAL;
1704 if (size != sizeof(outarg) + outarg.namelen + 1)
1705 return -EINVAL;
1706
1707 buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
1708 if (!buf)
1709 return -ENOMEM;
1710
1711 name.name = buf;
1712 name.len = outarg.namelen;
1713 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1714 if (err)
1715 goto err;
1716 fuse_copy_finish(cs);
1717 buf[outarg.namelen] = 0;
1718
1719 down_read(&fc->killsb);
1720 err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags);
1721 up_read(&fc->killsb);
1722 err:
1723 kfree(buf);
1724 return err;
1725 }
1726
1727 static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1728 struct fuse_copy_state *cs)
1729 {
1730 struct fuse_notify_delete_out outarg;
1731 int err;
1732 char *buf;
1733 struct qstr name;
1734
1735 if (size < sizeof(outarg))
1736 return -EINVAL;
1737
1738 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1739 if (err)
1740 return err;
1741
1742 if (outarg.namelen > fc->name_max)
1743 return -ENAMETOOLONG;
1744
1745 if (size != sizeof(outarg) + outarg.namelen + 1)
1746 return -EINVAL;
1747
1748 buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
1749 if (!buf)
1750 return -ENOMEM;
1751
1752 name.name = buf;
1753 name.len = outarg.namelen;
1754 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1755 if (err)
1756 goto err;
1757 fuse_copy_finish(cs);
1758 buf[outarg.namelen] = 0;
1759
1760 down_read(&fc->killsb);
1761 err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0);
1762 up_read(&fc->killsb);
1763 err:
1764 kfree(buf);
1765 return err;
1766 }
1767
1768 static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1769 struct fuse_copy_state *cs)
1770 {
1771 struct fuse_notify_store_out outarg;
1772 struct inode *inode;
1773 struct address_space *mapping;
1774 u64 nodeid;
1775 int err;
1776 pgoff_t index;
1777 unsigned int offset;
1778 unsigned int num;
1779 loff_t file_size;
1780 loff_t end;
1781
1782 if (size < sizeof(outarg))
1783 return -EINVAL;
1784
1785 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1786 if (err)
1787 return err;
1788
1789 if (size - sizeof(outarg) != outarg.size)
1790 return -EINVAL;
1791
1792 nodeid = outarg.nodeid;
1793
1794 down_read(&fc->killsb);
1795
1796 err = -ENOENT;
1797 inode = fuse_ilookup(fc, nodeid, NULL);
1798 if (!inode)
1799 goto out_up_killsb;
1800
1801 mapping = inode->i_mapping;
1802 index = outarg.offset >> PAGE_SHIFT;
1803 offset = outarg.offset & ~PAGE_MASK;
1804 file_size = i_size_read(inode);
1805 end = outarg.offset + outarg.size;
1806 if (end > file_size) {
1807 file_size = end;
1808 fuse_write_update_attr(inode, file_size, outarg.size);
1809 }
1810
1811 num = outarg.size;
1812 while (num) {
1813 struct folio *folio;
1814 unsigned int folio_offset;
1815 unsigned int nr_bytes;
1816 unsigned int nr_pages;
1817
1818 folio = filemap_grab_folio(mapping, index);
1819 err = PTR_ERR(folio);
1820 if (IS_ERR(folio))
1821 goto out_iput;
1822
1823 folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
1824 nr_bytes = min(num, folio_size(folio) - folio_offset);
1825 nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT;
1826
1827 err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
1828 if (!folio_test_uptodate(folio) && !err && offset == 0 &&
1829 (nr_bytes == folio_size(folio) || file_size == end)) {
1830 folio_zero_segment(folio, nr_bytes, folio_size(folio));
1831 folio_mark_uptodate(folio);
1832 }
1833 folio_unlock(folio);
1834 folio_put(folio);
1835
1836 if (err)
1837 goto out_iput;
1838
1839 num -= nr_bytes;
1840 offset = 0;
1841 index += nr_pages;
1842 }
1843
1844 err = 0;
1845
1846 out_iput:
1847 iput(inode);
1848 out_up_killsb:
1849 up_read(&fc->killsb);
1850 return err;
1851 }
1852
1853 struct fuse_retrieve_args {
1854 struct fuse_args_pages ap;
1855 struct fuse_notify_retrieve_in inarg;
1856 };
1857
1858 static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args,
1859 int error)
1860 {
1861 struct fuse_retrieve_args *ra =
1862 container_of(args, typeof(*ra), ap.args);
1863
1864 release_pages(ra->ap.folios, ra->ap.num_folios);
1865 kfree(ra);
1866 }
1867
1868 static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
1869 struct fuse_notify_retrieve_out *outarg)
1870 {
1871 int err;
1872 struct address_space *mapping = inode->i_mapping;
1873 pgoff_t index;
1874 loff_t file_size;
1875 unsigned int num;
1876 unsigned int offset;
1877 size_t total_len = 0;
1878 unsigned int num_pages;
1879 struct fuse_conn *fc = fm->fc;
1880 struct fuse_retrieve_args *ra;
1881 size_t args_size = sizeof(*ra);
1882 struct fuse_args_pages *ap;
1883 struct fuse_args *args;
1884
1885 offset = outarg->offset & ~PAGE_MASK;
1886 file_size = i_size_read(inode);
1887
1888 num = min(outarg->size, fc->max_write);
1889 if (outarg->offset > file_size)
1890 num = 0;
1891 else if (outarg->offset + num > file_size)
1892 num = file_size - outarg->offset;
1893
1894 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1895 num_pages = min(num_pages, fc->max_pages);
1896 num = min(num, num_pages << PAGE_SHIFT);
1897
1898 args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0]));
1899
1900 ra = kzalloc(args_size, GFP_KERNEL);
1901 if (!ra)
1902 return -ENOMEM;
1903
1904 ap = &ra->ap;
1905 ap->folios = (void *) (ra + 1);
1906 ap->descs = (void *) (ap->folios + num_pages);
1907
1908 args = &ap->args;
1909 args->nodeid = outarg->nodeid;
1910 args->opcode = FUSE_NOTIFY_REPLY;
1911 args->in_numargs = 3;
1912 args->in_pages = true;
1913 args->end = fuse_retrieve_end;
1914
1915 index = outarg->offset >> PAGE_SHIFT;
1916
1917 while (num && ap->num_folios < num_pages) {
1918 struct folio *folio;
1919 unsigned int folio_offset;
1920 unsigned int nr_bytes;
1921 unsigned int nr_pages;
1922
1923 folio = filemap_get_folio(mapping, index);
1924 if (IS_ERR(folio))
1925 break;
1926
1927 folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
1928 nr_bytes = min(folio_size(folio) - folio_offset, num);
1929 nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT;
1930
1931 ap->folios[ap->num_folios] = folio;
1932 ap->descs[ap->num_folios].offset = folio_offset;
1933 ap->descs[ap->num_folios].length = nr_bytes;
1934 ap->num_folios++;
1935
1936 offset = 0;
1937 num -= nr_bytes;
1938 total_len += nr_bytes;
1939 index += nr_pages;
1940 }
1941 ra->inarg.offset = outarg->offset;
1942 ra->inarg.size = total_len;
1943 fuse_set_zero_arg0(args);
1944 args->in_args[1].size = sizeof(ra->inarg);
1945 args->in_args[1].value = &ra->inarg;
1946 args->in_args[2].size = total_len;
1947
1948 err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
1949 if (err)
1950 fuse_retrieve_end(fm, args, err);
1951
1952 return err;
1953 }
1954
1955 static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1956 struct fuse_copy_state *cs)
1957 {
1958 struct fuse_notify_retrieve_out outarg;
1959 struct fuse_mount *fm;
1960 struct inode *inode;
1961 u64 nodeid;
1962 int err;
1963
1964 if (size != sizeof(outarg))
1965 return -EINVAL;
1966
1967 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1968 if (err)
1969 return err;
1970
1971 fuse_copy_finish(cs);
1972
1973 down_read(&fc->killsb);
1974 err = -ENOENT;
1975 nodeid = outarg.nodeid;
1976
1977 inode = fuse_ilookup(fc, nodeid, &fm);
1978 if (inode) {
1979 err = fuse_retrieve(fm, inode, &outarg);
1980 iput(inode);
1981 }
1982 up_read(&fc->killsb);
1983
1984 return err;
1985 }
1986
1987 /*
1988 * Resending all processing queue requests.
1989 *
1990 * During a FUSE daemon panics and failover, it is possible for some inflight
1991 * requests to be lost and never returned. As a result, applications awaiting
1992 * replies would become stuck forever. To address this, we can use notification
1993 * to trigger resending of these pending requests to the FUSE daemon, ensuring
1994 * they are properly processed again.
1995 *
1996 * Please note that this strategy is applicable only to idempotent requests or
1997 * if the FUSE daemon takes careful measures to avoid processing duplicated
1998 * non-idempotent requests.
1999 */
2000 static void fuse_resend(struct fuse_conn *fc)
2001 {
2002 struct fuse_dev *fud;
2003 struct fuse_req *req, *next;
2004 struct fuse_iqueue *fiq = &fc->iq;
2005 LIST_HEAD(to_queue);
2006 unsigned int i;
2007
2008 spin_lock(&fc->lock);
2009 if (!fc->connected) {
2010 spin_unlock(&fc->lock);
2011 return;
2012 }
2013
2014 list_for_each_entry(fud, &fc->devices, entry) {
2015 struct fuse_pqueue *fpq = &fud->pq;
2016
2017 spin_lock(&fpq->lock);
2018 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2019 list_splice_tail_init(&fpq->processing[i], &to_queue);
2020 spin_unlock(&fpq->lock);
2021 }
2022 spin_unlock(&fc->lock);
2023
2024 list_for_each_entry_safe(req, next, &to_queue, list) {
2025 set_bit(FR_PENDING, &req->flags);
2026 clear_bit(FR_SENT, &req->flags);
2027 /* mark the request as resend request */
2028 req->in.h.unique |= FUSE_UNIQUE_RESEND;
2029 }
2030
2031 spin_lock(&fiq->lock);
2032 if (!fiq->connected) {
2033 spin_unlock(&fiq->lock);
2034 list_for_each_entry(req, &to_queue, list)
2035 clear_bit(FR_PENDING, &req->flags);
2036 fuse_dev_end_requests(&to_queue);
2037 return;
2038 }
2039 /* iq and pq requests are both oldest to newest */
2040 list_splice(&to_queue, &fiq->pending);
2041 fuse_dev_wake_and_unlock(fiq);
2042 }
2043
2044 static int fuse_notify_resend(struct fuse_conn *fc)
2045 {
2046 fuse_resend(fc);
2047 return 0;
2048 }
2049
2050 /*
2051 * Increments the fuse connection epoch. This will result of dentries from
2052 * previous epochs to be invalidated. Additionally, if inval_wq is set, a work
2053 * queue is scheduled to trigger the invalidation.
2054 */
2055 static int fuse_notify_inc_epoch(struct fuse_conn *fc)
2056 {
2057 atomic_inc(&fc->epoch);
2058 if (inval_wq)
2059 schedule_work(&fc->epoch_work);
2060
2061 return 0;
2062 }
2063
2064 static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size,
2065 struct fuse_copy_state *cs)
2066 {
2067 struct fuse_notify_prune_out outarg;
2068 const unsigned int batch = 512;
2069 u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL);
2070 unsigned int num, i;
2071 int err;
2072
2073 if (!nodeids)
2074 return -ENOMEM;
2075
2076 if (size < sizeof(outarg))
2077 return -EINVAL;
2078
2079 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
2080 if (err)
2081 return err;
2082
2083 if (size - sizeof(outarg) != outarg.count * sizeof(u64))
2084 return -EINVAL;
2085
2086 for (; outarg.count; outarg.count -= num) {
2087 num = min(batch, outarg.count);
2088 err = fuse_copy_one(cs, nodeids, num * sizeof(u64));
2089 if (err)
2090 return err;
2091
2092 scoped_guard(rwsem_read, &fc->killsb) {
2093 for (i = 0; i < num; i++)
2094 fuse_try_prune_one_inode(fc, nodeids[i]);
2095 }
2096 }
2097 return 0;
2098 }
2099
2100 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
2101 unsigned int size, struct fuse_copy_state *cs)
2102 {
2103 /* Don't try to move folios (yet) */
2104 cs->move_folios = false;
2105
2106 switch (code) {
2107 case FUSE_NOTIFY_POLL:
2108 return fuse_notify_poll(fc, size, cs);
2109
2110 case FUSE_NOTIFY_INVAL_INODE:
2111 return fuse_notify_inval_inode(fc, size, cs);
2112
2113 case FUSE_NOTIFY_INVAL_ENTRY:
2114 return fuse_notify_inval_entry(fc, size, cs);
2115
2116 case FUSE_NOTIFY_STORE:
2117 return fuse_notify_store(fc, size, cs);
2118
2119 case FUSE_NOTIFY_RETRIEVE:
2120 return fuse_notify_retrieve(fc, size, cs);
2121
2122 case FUSE_NOTIFY_DELETE:
2123 return fuse_notify_delete(fc, size, cs);
2124
2125 case FUSE_NOTIFY_RESEND:
2126 return fuse_notify_resend(fc);
2127
2128 case FUSE_NOTIFY_INC_EPOCH:
2129 return fuse_notify_inc_epoch(fc);
2130
2131 case FUSE_NOTIFY_PRUNE:
2132 return fuse_notify_prune(fc, size, cs);
2133
2134 default:
2135 return -EINVAL;
2136 }
2137 }
2138
2139 /* Look up request on processing list by unique ID */
2140 struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique)
2141 {
2142 unsigned int hash = fuse_req_hash(unique);
2143 struct fuse_req *req;
2144
2145 list_for_each_entry(req, &fpq->processing[hash], list) {
2146 if (req->in.h.unique == unique)
2147 return req;
2148 }
2149 return NULL;
2150 }
2151
2152 int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
2153 unsigned nbytes)
2154 {
2155
2156 unsigned int reqsize = 0;
2157
2158 /*
2159 * Uring has all headers separated from args - args is payload only
2160 */
2161 if (!cs->is_uring)
2162 reqsize = sizeof(struct fuse_out_header);
2163
2164 reqsize += fuse_len_args(args->out_numargs, args->out_args);
2165
2166 if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
2167 return -EINVAL;
2168 else if (reqsize > nbytes) {
2169 struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
2170 unsigned diffsize = reqsize - nbytes;
2171
2172 if (diffsize > lastarg->size)
2173 return -EINVAL;
2174 lastarg->size -= diffsize;
2175 }
2176 return fuse_copy_args(cs, args->out_numargs, args->out_pages,
2177 args->out_args, args->page_zeroing);
2178 }
2179
2180 /*
2181 * Write a single reply to a request. First the header is copied from
2182 * the write buffer. The request is then searched on the processing
2183 * list by the unique ID found in the header. If found, then remove
2184 * it from the list and copy the rest of the buffer to the request.
2185 * The request is finished by calling fuse_request_end().
2186 */
2187 static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
2188 struct fuse_copy_state *cs, size_t nbytes)
2189 {
2190 int err;
2191 struct fuse_conn *fc = fud->fc;
2192 struct fuse_pqueue *fpq = &fud->pq;
2193 struct fuse_req *req;
2194 struct fuse_out_header oh;
2195
2196 err = -EINVAL;
2197 if (nbytes < sizeof(struct fuse_out_header))
2198 goto out;
2199
2200 err = fuse_copy_one(cs, &oh, sizeof(oh));
2201 if (err)
2202 goto copy_finish;
2203
2204 err = -EINVAL;
2205 if (oh.len != nbytes)
2206 goto copy_finish;
2207
2208 /*
2209 * Zero oh.unique indicates unsolicited notification message
2210 * and error contains notification code.
2211 */
2212 if (!oh.unique) {
2213 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
2214 goto copy_finish;
2215 }
2216
2217 err = -EINVAL;
2218 if (oh.error <= -512 || oh.error > 0)
2219 goto copy_finish;
2220
2221 spin_lock(&fpq->lock);
2222 req = NULL;
2223 if (fpq->connected)
2224 req = fuse_request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT);
2225
2226 err = -ENOENT;
2227 if (!req) {
2228 spin_unlock(&fpq->lock);
2229 goto copy_finish;
2230 }
2231
2232 /* Is it an interrupt reply ID? */
2233 if (oh.unique & FUSE_INT_REQ_BIT) {
2234 __fuse_get_request(req);
2235 spin_unlock(&fpq->lock);
2236
2237 err = 0;
2238 if (nbytes != sizeof(struct fuse_out_header))
2239 err = -EINVAL;
2240 else if (oh.error == -ENOSYS)
2241 fc->no_interrupt = 1;
2242 else if (oh.error == -EAGAIN)
2243 err = queue_interrupt(req);
2244
2245 fuse_put_request(req);
2246
2247 goto copy_finish;
2248 }
2249
2250 clear_bit(FR_SENT, &req->flags);
2251 list_move(&req->list, &fpq->io);
2252 req->out.h = oh;
2253 set_bit(FR_LOCKED, &req->flags);
2254 spin_unlock(&fpq->lock);
2255 cs->req = req;
2256 if (!req->args->page_replace)
2257 cs->move_folios = false;
2258
2259 if (oh.error)
2260 err = nbytes != sizeof(oh) ? -EINVAL : 0;
2261 else
2262 err = fuse_copy_out_args(cs, req->args, nbytes);
2263 fuse_copy_finish(cs);
2264
2265 spin_lock(&fpq->lock);
2266 clear_bit(FR_LOCKED, &req->flags);
2267 if (!fpq->connected)
2268 err = -ENOENT;
2269 else if (err)
2270 req->out.h.error = -EIO;
2271 if (!test_bit(FR_PRIVATE, &req->flags))
2272 list_del_init(&req->list);
2273 spin_unlock(&fpq->lock);
2274
2275 fuse_request_end(req);
2276 out:
2277 return err ? err : nbytes;
2278
2279 copy_finish:
2280 fuse_copy_finish(cs);
2281 goto out;
2282 }
2283
2284 static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
2285 {
2286 struct fuse_copy_state cs;
2287 struct fuse_dev *fud = __fuse_get_dev(iocb->ki_filp);
2288
2289 if (!fud)
2290 return -EPERM;
2291
2292 if (!user_backed_iter(from))
2293 return -EINVAL;
2294
2295 fuse_copy_init(&cs, false, from);
2296
2297 return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
2298 }
2299
2300 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
2301 struct file *out, loff_t *ppos,
2302 size_t len, unsigned int flags)
2303 {
2304 unsigned int head, tail, count;
2305 unsigned nbuf;
2306 unsigned idx;
2307 struct pipe_buffer *bufs;
2308 struct fuse_copy_state cs;
2309 struct fuse_dev *fud = __fuse_get_dev(out);
2310 size_t rem;
2311 ssize_t ret;
2312
2313 if (!fud)
2314 return -EPERM;
2315
2316 pipe_lock(pipe);
2317
2318 head = pipe->head;
2319 tail = pipe->tail;
2320 count = pipe_occupancy(head, tail);
2321
2322 bufs = kvmalloc_objs(struct pipe_buffer, count);
2323 if (!bufs) {
2324 pipe_unlock(pipe);
2325 return -ENOMEM;
2326 }
2327
2328 nbuf = 0;
2329 rem = 0;
2330 for (idx = tail; !pipe_empty(head, idx) && rem < len; idx++)
2331 rem += pipe_buf(pipe, idx)->len;
2332
2333 ret = -EINVAL;
2334 if (rem < len)
2335 goto out_free;
2336
2337 rem = len;
2338 while (rem) {
2339 struct pipe_buffer *ibuf;
2340 struct pipe_buffer *obuf;
2341
2342 if (WARN_ON(nbuf >= count || pipe_empty(head, tail)))
2343 goto out_free;
2344
2345 ibuf = pipe_buf(pipe, tail);
2346 obuf = &bufs[nbuf];
2347
2348 if (rem >= ibuf->len) {
2349 *obuf = *ibuf;
2350 ibuf->ops = NULL;
2351 tail++;
2352 pipe->tail = tail;
2353 } else {
2354 if (!pipe_buf_get(pipe, ibuf))
2355 goto out_free;
2356
2357 *obuf = *ibuf;
2358 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2359 obuf->len = rem;
2360 ibuf->offset += obuf->len;
2361 ibuf->len -= obuf->len;
2362 }
2363 nbuf++;
2364 rem -= obuf->len;
2365 }
2366 pipe_unlock(pipe);
2367
2368 fuse_copy_init(&cs, false, NULL);
2369 cs.pipebufs = bufs;
2370 cs.nr_segs = nbuf;
2371 cs.pipe = pipe;
2372
2373 if (flags & SPLICE_F_MOVE)
2374 cs.move_folios = true;
2375
2376 ret = fuse_dev_do_write(fud, &cs, len);
2377
2378 pipe_lock(pipe);
2379 out_free:
2380 for (idx = 0; idx < nbuf; idx++) {
2381 struct pipe_buffer *buf = &bufs[idx];
2382
2383 if (buf->ops)
2384 pipe_buf_release(pipe, buf);
2385 }
2386 pipe_unlock(pipe);
2387
2388 kvfree(bufs);
2389 return ret;
2390 }
2391
2392 static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
2393 {
2394 __poll_t mask = EPOLLOUT | EPOLLWRNORM;
2395 struct fuse_iqueue *fiq;
2396 struct fuse_dev *fud = fuse_get_dev(file);
2397
2398 if (IS_ERR(fud))
2399 return EPOLLERR;
2400
2401 fiq = &fud->fc->iq;
2402 poll_wait(file, &fiq->waitq, wait);
2403
2404 spin_lock(&fiq->lock);
2405 if (!fiq->connected)
2406 mask = EPOLLERR;
2407 else if (request_pending(fiq))
2408 mask |= EPOLLIN | EPOLLRDNORM;
2409 spin_unlock(&fiq->lock);
2410
2411 return mask;
2412 }
2413
2414 /* Abort all requests on the given list (pending or processing) */
2415 void fuse_dev_end_requests(struct list_head *head)
2416 {
2417 while (!list_empty(head)) {
2418 struct fuse_req *req;
2419 req = list_entry(head->next, struct fuse_req, list);
2420 req->out.h.error = -ECONNABORTED;
2421 clear_bit(FR_SENT, &req->flags);
2422 list_del_init(&req->list);
2423 fuse_request_end(req);
2424 }
2425 }
2426
2427 static void end_polls(struct fuse_conn *fc)
2428 {
2429 struct rb_node *p;
2430
2431 p = rb_first(&fc->polled_files);
2432
2433 while (p) {
2434 struct fuse_file *ff;
2435 ff = rb_entry(p, struct fuse_file, polled_node);
2436 wake_up_interruptible_all(&ff->poll_wait);
2437
2438 p = rb_next(p);
2439 }
2440 }
2441
2442 /*
2443 * Abort all requests.
2444 *
2445 * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2446 * filesystem.
2447 *
2448 * The same effect is usually achievable through killing the filesystem daemon
2449 * and all users of the filesystem. The exception is the combination of an
2450 * asynchronous request and the tricky deadlock (see
2451 * Documentation/filesystems/fuse/fuse.rst).
2452 *
2453 * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2454 * requests, they should be finished off immediately. Locked requests will be
2455 * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2456 * requests. It is possible that some request will finish before we can. This
2457 * is OK, the request will in that case be removed from the list before we touch
2458 * it.
2459 */
2460 void fuse_abort_conn(struct fuse_conn *fc)
2461 {
2462 struct fuse_iqueue *fiq = &fc->iq;
2463
2464 spin_lock(&fc->lock);
2465 if (fc->connected) {
2466 struct fuse_dev *fud;
2467 struct fuse_req *req, *next;
2468 LIST_HEAD(to_end);
2469 unsigned int i;
2470
2471 if (fc->timeout.req_timeout)
2472 cancel_delayed_work(&fc->timeout.work);
2473
2474 /* Background queuing checks fc->connected under bg_lock */
2475 spin_lock(&fc->bg_lock);
2476 fc->connected = 0;
2477 spin_unlock(&fc->bg_lock);
2478
2479 fuse_set_initialized(fc);
2480 list_for_each_entry(fud, &fc->devices, entry) {
2481 struct fuse_pqueue *fpq = &fud->pq;
2482
2483 spin_lock(&fpq->lock);
2484 fpq->connected = 0;
2485 list_for_each_entry_safe(req, next, &fpq->io, list) {
2486 req->out.h.error = -ECONNABORTED;
2487 spin_lock(&req->waitq.lock);
2488 set_bit(FR_ABORTED, &req->flags);
2489 if (!test_bit(FR_LOCKED, &req->flags)) {
2490 set_bit(FR_PRIVATE, &req->flags);
2491 __fuse_get_request(req);
2492 list_move(&req->list, &to_end);
2493 }
2494 spin_unlock(&req->waitq.lock);
2495 }
2496 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2497 list_splice_tail_init(&fpq->processing[i],
2498 &to_end);
2499 spin_unlock(&fpq->lock);
2500 }
2501 spin_lock(&fc->bg_lock);
2502 fc->blocked = 0;
2503 fc->max_background = UINT_MAX;
2504 flush_bg_queue(fc);
2505 spin_unlock(&fc->bg_lock);
2506
2507 spin_lock(&fiq->lock);
2508 fiq->connected = 0;
2509 list_for_each_entry(req, &fiq->pending, list)
2510 clear_bit(FR_PENDING, &req->flags);
2511 list_splice_tail_init(&fiq->pending, &to_end);
2512 while (forget_pending(fiq))
2513 kfree(fuse_dequeue_forget(fiq, 1, NULL));
2514 wake_up_all(&fiq->waitq);
2515 spin_unlock(&fiq->lock);
2516 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
2517 end_polls(fc);
2518 wake_up_all(&fc->blocked_waitq);
2519 spin_unlock(&fc->lock);
2520
2521 fuse_dev_end_requests(&to_end);
2522
2523 /*
2524 * fc->lock must not be taken to avoid conflicts with io-uring
2525 * locks
2526 */
2527 fuse_uring_abort(fc);
2528 } else {
2529 spin_unlock(&fc->lock);
2530 }
2531 }
2532 EXPORT_SYMBOL_GPL(fuse_abort_conn);
2533
2534 void fuse_wait_aborted(struct fuse_conn *fc)
2535 {
2536 /* matches implicit memory barrier in fuse_drop_waiting() */
2537 smp_mb();
2538 wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
2539
2540 fuse_uring_wait_stopped_queues(fc);
2541 }
2542
2543 int fuse_dev_release(struct inode *inode, struct file *file)
2544 {
2545 struct fuse_dev *fud = __fuse_get_dev(file);
2546
2547 if (fud) {
2548 struct fuse_conn *fc = fud->fc;
2549 struct fuse_pqueue *fpq = &fud->pq;
2550 LIST_HEAD(to_end);
2551 unsigned int i;
2552
2553 spin_lock(&fpq->lock);
2554 WARN_ON(!list_empty(&fpq->io));
2555 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2556 list_splice_init(&fpq->processing[i], &to_end);
2557 spin_unlock(&fpq->lock);
2558
2559 fuse_dev_end_requests(&to_end);
2560
2561 /* Are we the last open device? */
2562 if (atomic_dec_and_test(&fc->dev_count)) {
2563 WARN_ON(fc->iq.fasync != NULL);
2564 fuse_abort_conn(fc);
2565 }
2566 fuse_dev_free(fud);
2567 }
2568 return 0;
2569 }
2570 EXPORT_SYMBOL_GPL(fuse_dev_release);
2571
2572 static int fuse_dev_fasync(int fd, struct file *file, int on)
2573 {
2574 struct fuse_dev *fud = fuse_get_dev(file);
2575
2576 if (IS_ERR(fud))
2577 return PTR_ERR(fud);
2578
2579 /* No locking - fasync_helper does its own locking */
2580 return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
2581 }
2582
2583 static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2584 {
2585 struct fuse_dev *fud;
2586
2587 if (__fuse_get_dev(new))
2588 return -EINVAL;
2589
2590 fud = fuse_dev_alloc_install(fc);
2591 if (!fud)
2592 return -ENOMEM;
2593
2594 new->private_data = fud;
2595 atomic_inc(&fc->dev_count);
2596
2597 return 0;
2598 }
2599
2600 static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp)
2601 {
2602 int oldfd;
2603 struct fuse_dev *fud;
2604
2605 if (get_user(oldfd, argp))
2606 return -EFAULT;
2607
2608 CLASS(fd, f)(oldfd);
2609 if (fd_empty(f))
2610 return -EINVAL;
2611
2612 /*
2613 * Check against file->f_op because CUSE
2614 * uses the same ioctl handler.
2615 */
2616 if (fd_file(f)->f_op != file->f_op)
2617 return -EINVAL;
2618
2619 fud = fuse_get_dev(fd_file(f));
2620 if (IS_ERR(fud))
2621 return PTR_ERR(fud);
2622
2623 guard(mutex)(&fuse_mutex);
2624 return fuse_device_clone(fud->fc, file);
2625 }
2626
2627 static long fuse_dev_ioctl_backing_open(struct file *file,
2628 struct fuse_backing_map __user *argp)
2629 {
2630 struct fuse_dev *fud = fuse_get_dev(file);
2631 struct fuse_backing_map map;
2632
2633 if (IS_ERR(fud))
2634 return PTR_ERR(fud);
2635
2636 if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
2637 return -EOPNOTSUPP;
2638
2639 if (copy_from_user(&map, argp, sizeof(map)))
2640 return -EFAULT;
2641
2642 return fuse_backing_open(fud->fc, &map);
2643 }
2644
2645 static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp)
2646 {
2647 struct fuse_dev *fud = fuse_get_dev(file);
2648 int backing_id;
2649
2650 if (IS_ERR(fud))
2651 return PTR_ERR(fud);
2652
2653 if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
2654 return -EOPNOTSUPP;
2655
2656 if (get_user(backing_id, argp))
2657 return -EFAULT;
2658
2659 return fuse_backing_close(fud->fc, backing_id);
2660 }
2661
2662 static long fuse_dev_ioctl_sync_init(struct file *file)
2663 {
2664 int err = -EINVAL;
2665
2666 mutex_lock(&fuse_mutex);
2667 if (!__fuse_get_dev(file)) {
2668 WRITE_ONCE(file->private_data, FUSE_DEV_SYNC_INIT);
2669 err = 0;
2670 }
2671 mutex_unlock(&fuse_mutex);
2672 return err;
2673 }
2674
2675 static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2676 unsigned long arg)
2677 {
2678 void __user *argp = (void __user *)arg;
2679
2680 switch (cmd) {
2681 case FUSE_DEV_IOC_CLONE:
2682 return fuse_dev_ioctl_clone(file, argp);
2683
2684 case FUSE_DEV_IOC_BACKING_OPEN:
2685 return fuse_dev_ioctl_backing_open(file, argp);
2686
2687 case FUSE_DEV_IOC_BACKING_CLOSE:
2688 return fuse_dev_ioctl_backing_close(file, argp);
2689
2690 case FUSE_DEV_IOC_SYNC_INIT:
2691 return fuse_dev_ioctl_sync_init(file);
2692
2693 default:
2694 return -ENOTTY;
2695 }
2696 }
2697
2698 #ifdef CONFIG_PROC_FS
2699 static void fuse_dev_show_fdinfo(struct seq_file *seq, struct file *file)
2700 {
2701 struct fuse_dev *fud = __fuse_get_dev(file);
2702 if (!fud)
2703 return;
2704
2705 seq_printf(seq, "fuse_connection:\t%u\n", fud->fc->dev);
2706 }
2707 #endif
2708
2709 const struct file_operations fuse_dev_operations = {
2710 .owner = THIS_MODULE,
2711 .open = fuse_dev_open,
2712 .read_iter = fuse_dev_read,
2713 .splice_read = fuse_dev_splice_read,
2714 .write_iter = fuse_dev_write,
2715 .splice_write = fuse_dev_splice_write,
2716 .poll = fuse_dev_poll,
2717 .release = fuse_dev_release,
2718 .fasync = fuse_dev_fasync,
2719 .unlocked_ioctl = fuse_dev_ioctl,
2720 .compat_ioctl = compat_ptr_ioctl,
2721 #ifdef CONFIG_FUSE_IO_URING
2722 .uring_cmd = fuse_uring_cmd,
2723 #endif
2724 #ifdef CONFIG_PROC_FS
2725 .show_fdinfo = fuse_dev_show_fdinfo,
2726 #endif
2727 };
2728 EXPORT_SYMBOL_GPL(fuse_dev_operations);
2729
2730 static struct miscdevice fuse_miscdevice = {
2731 .minor = FUSE_MINOR,
2732 .name = "fuse",
2733 .fops = &fuse_dev_operations,
2734 };
2735
2736 int __init fuse_dev_init(void)
2737 {
2738 int err = -ENOMEM;
2739 fuse_req_cachep = kmem_cache_create("fuse_request",
2740 sizeof(struct fuse_req),
2741 0, 0, NULL);
2742 if (!fuse_req_cachep)
2743 goto out;
2744
2745 err = misc_register(&fuse_miscdevice);
2746 if (err)
2747 goto out_cache_clean;
2748
2749 return 0;
2750
2751 out_cache_clean:
2752 kmem_cache_destroy(fuse_req_cachep);
2753 out:
2754 return err;
2755 }
2756
2757 void fuse_dev_cleanup(void)
2758 {
2759 misc_deregister(&fuse_miscdevice);
2760 kmem_cache_destroy(fuse_req_cachep);
2761 }