| 1 |
/* |
| 2 |
FUSE: Filesystem in Userspace |
| 3 |
Copyright (C) 2001-2008 Miklos Szeredi <[email protected]> |
| 4 |
|
| 5 |
This program can be distributed under the terms of the GNU GPL. |
| 6 |
See the file COPYING. |
| 7 |
*/ |
| 8 |
|
| 9 |
#include "fuse_i.h" |
| 10 |
|
| 11 |
#include <linux/pagemap.h> |
| 12 |
#include <linux/slab.h> |
| 13 |
#include <linux/kernel.h> |
| 14 |
#include <linux/sched.h> |
| 15 |
#include <linux/sched/signal.h> |
| 16 |
#include <linux/module.h> |
| 17 |
#include <linux/swap.h> |
| 18 |
#include <linux/falloc.h> |
| 19 |
#include <linux/uio.h> |
| 20 |
#include <linux/fs.h> |
| 21 |
#include <linux/filelock.h> |
| 22 |
#include <linux/splice.h> |
| 23 |
#include <linux/task_io_accounting_ops.h> |
| 24 |
#include <linux/iomap.h> |
| 25 |
|
| 26 |
static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, |
| 27 |
unsigned int open_flags, int opcode, |
| 28 |
struct fuse_open_out *outargp) |
| 29 |
{ |
| 30 |
struct fuse_open_in inarg; |
| 31 |
FUSE_ARGS(args); |
| 32 |
|
| 33 |
memset(&inarg, 0, sizeof(inarg)); |
| 34 |
inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY); |
| 35 |
if (!fm->fc->atomic_o_trunc) |
| 36 |
inarg.flags &= ~O_TRUNC; |
| 37 |
|
| 38 |
if (fm->fc->handle_killpriv_v2 && |
| 39 |
(inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { |
| 40 |
inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; |
| 41 |
} |
| 42 |
|
| 43 |
args.opcode = opcode; |
| 44 |
args.nodeid = nodeid; |
| 45 |
args.in_numargs = 1; |
| 46 |
args.in_args[0].size = sizeof(inarg); |
| 47 |
args.in_args[0].value = &inarg; |
| 48 |
args.out_numargs = 1; |
| 49 |
args.out_args[0].size = sizeof(*outargp); |
| 50 |
args.out_args[0].value = outargp; |
| 51 |
|
| 52 |
return fuse_simple_request(fm, &args); |
| 53 |
} |
| 54 |
|
| 55 |
struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release) |
| 56 |
{ |
| 57 |
struct fuse_file *ff; |
| 58 |
|
| 59 |
ff = kzalloc_obj(struct fuse_file, GFP_KERNEL_ACCOUNT); |
| 60 |
if (unlikely(!ff)) |
| 61 |
return NULL; |
| 62 |
|
| 63 |
ff->fm = fm; |
| 64 |
if (release) { |
| 65 |
ff->args = kzalloc_obj(*ff->args, GFP_KERNEL_ACCOUNT); |
| 66 |
if (!ff->args) { |
| 67 |
kfree(ff); |
| 68 |
return NULL; |
| 69 |
} |
| 70 |
} |
| 71 |
|
| 72 |
INIT_LIST_HEAD(&ff->write_entry); |
| 73 |
refcount_set(&ff->count, 1); |
| 74 |
RB_CLEAR_NODE(&ff->polled_node); |
| 75 |
init_waitqueue_head(&ff->poll_wait); |
| 76 |
|
| 77 |
ff->kh = atomic64_inc_return(&fm->fc->khctr); |
| 78 |
|
| 79 |
return ff; |
| 80 |
} |
| 81 |
|
| 82 |
void fuse_file_free(struct fuse_file *ff) |
| 83 |
{ |
| 84 |
kfree(ff->args); |
| 85 |
kfree(ff); |
| 86 |
} |
| 87 |
|
| 88 |
static struct fuse_file *fuse_file_get(struct fuse_file *ff) |
| 89 |
{ |
| 90 |
refcount_inc(&ff->count); |
| 91 |
return ff; |
| 92 |
} |
| 93 |
|
| 94 |
static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args, |
| 95 |
int error) |
| 96 |
{ |
| 97 |
struct fuse_release_args *ra = container_of(args, typeof(*ra), args); |
| 98 |
|
| 99 |
iput(ra->inode); |
| 100 |
kfree(ra); |
| 101 |
} |
| 102 |
|
| 103 |
static void fuse_file_put(struct fuse_file *ff, bool sync) |
| 104 |
{ |
| 105 |
if (refcount_dec_and_test(&ff->count)) { |
| 106 |
struct fuse_release_args *ra = &ff->args->release_args; |
| 107 |
struct fuse_args *args = (ra ? &ra->args : NULL); |
| 108 |
|
| 109 |
if (ra && ra->inode) |
| 110 |
fuse_file_io_release(ff, ra->inode); |
| 111 |
|
| 112 |
if (!args) { |
| 113 |
/* Do nothing when server does not implement 'opendir' */ |
| 114 |
} else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) { |
| 115 |
fuse_release_end(ff->fm, args, 0); |
| 116 |
} else if (sync) { |
| 117 |
fuse_simple_request(ff->fm, args); |
| 118 |
fuse_release_end(ff->fm, args, 0); |
| 119 |
} else { |
| 120 |
args->end = fuse_release_end; |
| 121 |
if (fuse_simple_background(ff->fm, args, |
| 122 |
GFP_KERNEL | __GFP_NOFAIL)) |
| 123 |
fuse_release_end(ff->fm, args, -ENOTCONN); |
| 124 |
} |
| 125 |
kfree(ff); |
| 126 |
} |
| 127 |
} |
| 128 |
|
| 129 |
struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, |
| 130 |
unsigned int open_flags, bool isdir) |
| 131 |
{ |
| 132 |
struct fuse_conn *fc = fm->fc; |
| 133 |
struct fuse_file *ff; |
| 134 |
int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; |
| 135 |
bool open = isdir ? !fc->no_opendir : !fc->no_open; |
| 136 |
bool release = !isdir || open; |
| 137 |
|
| 138 |
/* |
| 139 |
* ff->args->release_args still needs to be allocated (so we can hold an |
| 140 |
* inode reference while there are pending inflight file operations when |
| 141 |
* ->release() is called, see fuse_prepare_release()) even if |
| 142 |
* fc->no_open is set else it becomes possible for reclaim to deadlock |
| 143 |
* if while servicing the readahead request the server triggers reclaim |
| 144 |
* and reclaim evicts the inode of the file being read ahead. |
| 145 |
*/ |
| 146 |
ff = fuse_file_alloc(fm, release); |
| 147 |
if (!ff) |
| 148 |
return ERR_PTR(-ENOMEM); |
| 149 |
|
| 150 |
ff->fh = 0; |
| 151 |
/* Default for no-open */ |
| 152 |
ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0); |
| 153 |
if (open) { |
| 154 |
/* Store outarg for fuse_finish_open() */ |
| 155 |
struct fuse_open_out *outargp = &ff->args->open_outarg; |
| 156 |
int err; |
| 157 |
|
| 158 |
err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); |
| 159 |
if (!err) { |
| 160 |
ff->fh = outargp->fh; |
| 161 |
ff->open_flags = outargp->open_flags; |
| 162 |
} else if (err != -ENOSYS) { |
| 163 |
fuse_file_free(ff); |
| 164 |
return ERR_PTR(err); |
| 165 |
} else { |
| 166 |
if (isdir) { |
| 167 |
/* No release needed */ |
| 168 |
kfree(ff->args); |
| 169 |
ff->args = NULL; |
| 170 |
fc->no_opendir = 1; |
| 171 |
} else { |
| 172 |
fc->no_open = 1; |
| 173 |
} |
| 174 |
} |
| 175 |
} |
| 176 |
|
| 177 |
if (isdir) |
| 178 |
ff->open_flags &= ~FOPEN_DIRECT_IO; |
| 179 |
|
| 180 |
ff->nodeid = nodeid; |
| 181 |
|
| 182 |
return ff; |
| 183 |
} |
| 184 |
|
| 185 |
int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, |
| 186 |
bool isdir) |
| 187 |
{ |
| 188 |
struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir); |
| 189 |
|
| 190 |
if (!IS_ERR(ff)) |
| 191 |
file->private_data = ff; |
| 192 |
|
| 193 |
return PTR_ERR_OR_ZERO(ff); |
| 194 |
} |
| 195 |
EXPORT_SYMBOL_GPL(fuse_do_open); |
| 196 |
|
| 197 |
static void fuse_link_write_file(struct file *file) |
| 198 |
{ |
| 199 |
struct inode *inode = file_inode(file); |
| 200 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 201 |
struct fuse_file *ff = file->private_data; |
| 202 |
/* |
| 203 |
* file may be written through mmap, so chain it onto the |
| 204 |
* inodes's write_file list |
| 205 |
*/ |
| 206 |
spin_lock(&fi->lock); |
| 207 |
if (list_empty(&ff->write_entry)) |
| 208 |
list_add(&ff->write_entry, &fi->write_files); |
| 209 |
spin_unlock(&fi->lock); |
| 210 |
} |
| 211 |
|
| 212 |
int fuse_finish_open(struct inode *inode, struct file *file) |
| 213 |
{ |
| 214 |
struct fuse_file *ff = file->private_data; |
| 215 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 216 |
int err; |
| 217 |
|
| 218 |
err = fuse_file_io_open(file, inode); |
| 219 |
if (err) |
| 220 |
return err; |
| 221 |
|
| 222 |
if (ff->open_flags & FOPEN_STREAM) |
| 223 |
stream_open(inode, file); |
| 224 |
else if (ff->open_flags & FOPEN_NONSEEKABLE) |
| 225 |
nonseekable_open(inode, file); |
| 226 |
|
| 227 |
if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache) |
| 228 |
fuse_link_write_file(file); |
| 229 |
|
| 230 |
return 0; |
| 231 |
} |
| 232 |
|
| 233 |
static void fuse_truncate_update_attr(struct inode *inode, struct file *file) |
| 234 |
{ |
| 235 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 236 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 237 |
|
| 238 |
spin_lock(&fi->lock); |
| 239 |
fi->attr_version = atomic64_inc_return(&fc->attr_version); |
| 240 |
i_size_write(inode, 0); |
| 241 |
spin_unlock(&fi->lock); |
| 242 |
file_update_time(file); |
| 243 |
fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); |
| 244 |
} |
| 245 |
|
| 246 |
static int fuse_open(struct inode *inode, struct file *file) |
| 247 |
{ |
| 248 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 249 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 250 |
struct fuse_conn *fc = fm->fc; |
| 251 |
struct fuse_file *ff; |
| 252 |
int err; |
| 253 |
bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc; |
| 254 |
bool is_wb_truncate = is_truncate && fc->writeback_cache; |
| 255 |
bool dax_truncate = is_truncate && FUSE_IS_DAX(inode); |
| 256 |
|
| 257 |
if (fuse_is_bad(inode)) |
| 258 |
return -EIO; |
| 259 |
|
| 260 |
err = generic_file_open(inode, file); |
| 261 |
if (err) |
| 262 |
return err; |
| 263 |
|
| 264 |
if (is_wb_truncate || dax_truncate) |
| 265 |
inode_lock(inode); |
| 266 |
|
| 267 |
if (dax_truncate) { |
| 268 |
filemap_invalidate_lock(inode->i_mapping); |
| 269 |
err = fuse_dax_break_layouts(inode, 0, -1); |
| 270 |
if (err) |
| 271 |
goto out_inode_unlock; |
| 272 |
} |
| 273 |
|
| 274 |
if (is_wb_truncate || dax_truncate) |
| 275 |
fuse_set_nowrite(inode); |
| 276 |
|
| 277 |
err = fuse_do_open(fm, get_node_id(inode), file, false); |
| 278 |
if (!err) { |
| 279 |
ff = file->private_data; |
| 280 |
err = fuse_finish_open(inode, file); |
| 281 |
if (err) |
| 282 |
fuse_sync_release(fi, ff, file->f_flags); |
| 283 |
else if (is_truncate) |
| 284 |
fuse_truncate_update_attr(inode, file); |
| 285 |
} |
| 286 |
|
| 287 |
if (is_wb_truncate || dax_truncate) |
| 288 |
fuse_release_nowrite(inode); |
| 289 |
if (!err) { |
| 290 |
if (is_truncate) |
| 291 |
truncate_pagecache(inode, 0); |
| 292 |
else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) |
| 293 |
invalidate_inode_pages2(inode->i_mapping); |
| 294 |
} |
| 295 |
if (dax_truncate) |
| 296 |
filemap_invalidate_unlock(inode->i_mapping); |
| 297 |
out_inode_unlock: |
| 298 |
if (is_wb_truncate || dax_truncate) |
| 299 |
inode_unlock(inode); |
| 300 |
|
| 301 |
return err; |
| 302 |
} |
| 303 |
|
| 304 |
static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, |
| 305 |
unsigned int flags, int opcode, bool sync) |
| 306 |
{ |
| 307 |
struct fuse_conn *fc = ff->fm->fc; |
| 308 |
struct fuse_release_args *ra = &ff->args->release_args; |
| 309 |
|
| 310 |
if (fuse_file_passthrough(ff)) |
| 311 |
fuse_passthrough_release(ff, fuse_inode_backing(fi)); |
| 312 |
|
| 313 |
/* Inode is NULL on error path of fuse_create_open() */ |
| 314 |
if (likely(fi)) { |
| 315 |
spin_lock(&fi->lock); |
| 316 |
list_del(&ff->write_entry); |
| 317 |
spin_unlock(&fi->lock); |
| 318 |
} |
| 319 |
spin_lock(&fc->lock); |
| 320 |
if (!RB_EMPTY_NODE(&ff->polled_node)) |
| 321 |
rb_erase(&ff->polled_node, &fc->polled_files); |
| 322 |
spin_unlock(&fc->lock); |
| 323 |
|
| 324 |
wake_up_interruptible_all(&ff->poll_wait); |
| 325 |
|
| 326 |
if (!ra) |
| 327 |
return; |
| 328 |
|
| 329 |
/* ff->args was used for open outarg */ |
| 330 |
memset(ff->args, 0, sizeof(*ff->args)); |
| 331 |
ra->inarg.fh = ff->fh; |
| 332 |
ra->inarg.flags = flags; |
| 333 |
ra->args.in_numargs = 1; |
| 334 |
ra->args.in_args[0].size = sizeof(struct fuse_release_in); |
| 335 |
ra->args.in_args[0].value = &ra->inarg; |
| 336 |
ra->args.opcode = opcode; |
| 337 |
ra->args.nodeid = ff->nodeid; |
| 338 |
ra->args.force = true; |
| 339 |
ra->args.nocreds = true; |
| 340 |
|
| 341 |
/* |
| 342 |
* Hold inode until release is finished. |
| 343 |
* From fuse_sync_release() the refcount is 1 and everything's |
| 344 |
* synchronous, so we are fine with not doing igrab() here. |
| 345 |
*/ |
| 346 |
ra->inode = sync ? NULL : igrab(&fi->inode); |
| 347 |
} |
| 348 |
|
| 349 |
void fuse_file_release(struct inode *inode, struct fuse_file *ff, |
| 350 |
unsigned int open_flags, fl_owner_t id, bool isdir) |
| 351 |
{ |
| 352 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 353 |
struct fuse_release_args *ra = &ff->args->release_args; |
| 354 |
int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; |
| 355 |
|
| 356 |
fuse_prepare_release(fi, ff, open_flags, opcode, false); |
| 357 |
|
| 358 |
if (ra && ff->flock) { |
| 359 |
ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; |
| 360 |
ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); |
| 361 |
} |
| 362 |
|
| 363 |
/* |
| 364 |
* Normally this will send the RELEASE request, however if |
| 365 |
* some asynchronous READ or WRITE requests are outstanding, |
| 366 |
* the sending will be delayed. |
| 367 |
* |
| 368 |
* Make the release synchronous if this is a fuseblk mount, |
| 369 |
* synchronous RELEASE is allowed (and desirable) in this case |
| 370 |
* because the server can be trusted not to screw up. |
| 371 |
* |
| 372 |
* Always use the asynchronous file put because the current thread |
| 373 |
* might be the fuse server. This can happen if a process starts some |
| 374 |
* aio and closes the fd before the aio completes. Since aio takes its |
| 375 |
* own ref to the file, the IO completion has to drop the ref, which is |
| 376 |
* how the fuse server can end up closing its clients' files. |
| 377 |
*/ |
| 378 |
fuse_file_put(ff, false); |
| 379 |
} |
| 380 |
|
| 381 |
void fuse_release_common(struct file *file, bool isdir) |
| 382 |
{ |
| 383 |
fuse_file_release(file_inode(file), file->private_data, file->f_flags, |
| 384 |
(fl_owner_t) file, isdir); |
| 385 |
} |
| 386 |
|
| 387 |
static int fuse_release(struct inode *inode, struct file *file) |
| 388 |
{ |
| 389 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 390 |
|
| 391 |
/* |
| 392 |
* Dirty pages might remain despite write_inode_now() call from |
| 393 |
* fuse_flush() due to writes racing with the close. |
| 394 |
*/ |
| 395 |
if (fc->writeback_cache) |
| 396 |
write_inode_now(inode, 1); |
| 397 |
|
| 398 |
fuse_release_common(file, false); |
| 399 |
|
| 400 |
/* return value is ignored by VFS */ |
| 401 |
return 0; |
| 402 |
} |
| 403 |
|
| 404 |
void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, |
| 405 |
unsigned int flags) |
| 406 |
{ |
| 407 |
WARN_ON(refcount_read(&ff->count) > 1); |
| 408 |
fuse_prepare_release(fi, ff, flags, FUSE_RELEASE, true); |
| 409 |
fuse_file_put(ff, true); |
| 410 |
} |
| 411 |
EXPORT_SYMBOL_GPL(fuse_sync_release); |
| 412 |
|
| 413 |
/* |
| 414 |
* Scramble the ID space with XTEA, so that the value of the files_struct |
| 415 |
* pointer is not exposed to userspace. |
| 416 |
*/ |
| 417 |
u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) |
| 418 |
{ |
| 419 |
u32 *k = fc->scramble_key; |
| 420 |
u64 v = (unsigned long) id; |
| 421 |
u32 v0 = v; |
| 422 |
u32 v1 = v >> 32; |
| 423 |
u32 sum = 0; |
| 424 |
int i; |
| 425 |
|
| 426 |
for (i = 0; i < 32; i++) { |
| 427 |
v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]); |
| 428 |
sum += 0x9E3779B9; |
| 429 |
v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]); |
| 430 |
} |
| 431 |
|
| 432 |
return (u64) v0 + ((u64) v1 << 32); |
| 433 |
} |
| 434 |
|
| 435 |
struct fuse_writepage_args { |
| 436 |
struct fuse_io_args ia; |
| 437 |
struct list_head queue_entry; |
| 438 |
struct inode *inode; |
| 439 |
struct fuse_sync_bucket *bucket; |
| 440 |
}; |
| 441 |
|
| 442 |
/* |
| 443 |
* Wait for all pending writepages on the inode to finish. |
| 444 |
* |
| 445 |
* This is currently done by blocking further writes with FUSE_NOWRITE |
| 446 |
* and waiting for all sent writes to complete. |
| 447 |
* |
| 448 |
* This must be called under i_mutex, otherwise the FUSE_NOWRITE usage |
| 449 |
* could conflict with truncation. |
| 450 |
*/ |
| 451 |
static void fuse_sync_writes(struct inode *inode) |
| 452 |
{ |
| 453 |
fuse_set_nowrite(inode); |
| 454 |
fuse_release_nowrite(inode); |
| 455 |
} |
| 456 |
|
| 457 |
static int fuse_flush(struct file *file, fl_owner_t id) |
| 458 |
{ |
| 459 |
struct inode *inode = file_inode(file); |
| 460 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 461 |
struct fuse_file *ff = file->private_data; |
| 462 |
struct fuse_flush_in inarg; |
| 463 |
FUSE_ARGS(args); |
| 464 |
int err; |
| 465 |
|
| 466 |
if (fuse_is_bad(inode)) |
| 467 |
return -EIO; |
| 468 |
|
| 469 |
if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache) |
| 470 |
return 0; |
| 471 |
|
| 472 |
err = write_inode_now(inode, 1); |
| 473 |
if (err) |
| 474 |
return err; |
| 475 |
|
| 476 |
err = filemap_check_errors(file->f_mapping); |
| 477 |
if (err) |
| 478 |
return err; |
| 479 |
|
| 480 |
err = 0; |
| 481 |
if (fm->fc->no_flush) |
| 482 |
goto inval_attr_out; |
| 483 |
|
| 484 |
memset(&inarg, 0, sizeof(inarg)); |
| 485 |
inarg.fh = ff->fh; |
| 486 |
inarg.lock_owner = fuse_lock_owner_id(fm->fc, id); |
| 487 |
args.opcode = FUSE_FLUSH; |
| 488 |
args.nodeid = get_node_id(inode); |
| 489 |
args.in_numargs = 1; |
| 490 |
args.in_args[0].size = sizeof(inarg); |
| 491 |
args.in_args[0].value = &inarg; |
| 492 |
args.force = true; |
| 493 |
|
| 494 |
err = fuse_simple_request(fm, &args); |
| 495 |
if (err == -ENOSYS) { |
| 496 |
fm->fc->no_flush = 1; |
| 497 |
err = 0; |
| 498 |
} |
| 499 |
|
| 500 |
inval_attr_out: |
| 501 |
/* |
| 502 |
* In memory i_blocks is not maintained by fuse, if writeback cache is |
| 503 |
* enabled, i_blocks from cached attr may not be accurate. |
| 504 |
*/ |
| 505 |
if (!err && fm->fc->writeback_cache) |
| 506 |
fuse_invalidate_attr_mask(inode, STATX_BLOCKS); |
| 507 |
return err; |
| 508 |
} |
| 509 |
|
| 510 |
int fuse_fsync_common(struct file *file, loff_t start, loff_t end, |
| 511 |
int datasync, int opcode) |
| 512 |
{ |
| 513 |
struct inode *inode = file->f_mapping->host; |
| 514 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 515 |
struct fuse_file *ff = file->private_data; |
| 516 |
FUSE_ARGS(args); |
| 517 |
struct fuse_fsync_in inarg; |
| 518 |
|
| 519 |
memset(&inarg, 0, sizeof(inarg)); |
| 520 |
inarg.fh = ff->fh; |
| 521 |
inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0; |
| 522 |
args.opcode = opcode; |
| 523 |
args.nodeid = get_node_id(inode); |
| 524 |
args.in_numargs = 1; |
| 525 |
args.in_args[0].size = sizeof(inarg); |
| 526 |
args.in_args[0].value = &inarg; |
| 527 |
return fuse_simple_request(fm, &args); |
| 528 |
} |
| 529 |
|
| 530 |
static int fuse_fsync(struct file *file, loff_t start, loff_t end, |
| 531 |
int datasync) |
| 532 |
{ |
| 533 |
struct inode *inode = file->f_mapping->host; |
| 534 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 535 |
int err; |
| 536 |
|
| 537 |
if (fuse_is_bad(inode)) |
| 538 |
return -EIO; |
| 539 |
|
| 540 |
inode_lock(inode); |
| 541 |
|
| 542 |
/* |
| 543 |
* Start writeback against all dirty pages of the inode, then |
| 544 |
* wait for all outstanding writes, before sending the FSYNC |
| 545 |
* request. |
| 546 |
*/ |
| 547 |
err = file_write_and_wait_range(file, start, end); |
| 548 |
if (err) |
| 549 |
goto out; |
| 550 |
|
| 551 |
fuse_sync_writes(inode); |
| 552 |
|
| 553 |
/* |
| 554 |
* Due to implementation of fuse writeback |
| 555 |
* file_write_and_wait_range() does not catch errors. |
| 556 |
* We have to do this directly after fuse_sync_writes() |
| 557 |
*/ |
| 558 |
err = file_check_and_advance_wb_err(file); |
| 559 |
if (err) |
| 560 |
goto out; |
| 561 |
|
| 562 |
err = sync_inode_metadata(inode, 1); |
| 563 |
if (err) |
| 564 |
goto out; |
| 565 |
|
| 566 |
if (fc->no_fsync) |
| 567 |
goto out; |
| 568 |
|
| 569 |
err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC); |
| 570 |
if (err == -ENOSYS) { |
| 571 |
fc->no_fsync = 1; |
| 572 |
err = 0; |
| 573 |
} |
| 574 |
out: |
| 575 |
inode_unlock(inode); |
| 576 |
|
| 577 |
return err; |
| 578 |
} |
| 579 |
|
| 580 |
void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, |
| 581 |
size_t count, int opcode) |
| 582 |
{ |
| 583 |
struct fuse_file *ff = file->private_data; |
| 584 |
struct fuse_args *args = &ia->ap.args; |
| 585 |
|
| 586 |
ia->read.in.fh = ff->fh; |
| 587 |
ia->read.in.offset = pos; |
| 588 |
ia->read.in.size = count; |
| 589 |
ia->read.in.flags = file->f_flags; |
| 590 |
args->opcode = opcode; |
| 591 |
args->nodeid = ff->nodeid; |
| 592 |
args->in_numargs = 1; |
| 593 |
args->in_args[0].size = sizeof(ia->read.in); |
| 594 |
args->in_args[0].value = &ia->read.in; |
| 595 |
args->out_argvar = true; |
| 596 |
args->out_numargs = 1; |
| 597 |
args->out_args[0].size = count; |
| 598 |
} |
| 599 |
|
| 600 |
static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, |
| 601 |
bool should_dirty) |
| 602 |
{ |
| 603 |
unsigned int i; |
| 604 |
|
| 605 |
for (i = 0; i < ap->num_folios; i++) { |
| 606 |
if (should_dirty) |
| 607 |
folio_mark_dirty_lock(ap->folios[i]); |
| 608 |
if (ap->args.is_pinned) |
| 609 |
unpin_folio(ap->folios[i]); |
| 610 |
} |
| 611 |
|
| 612 |
if (nres > 0 && ap->args.invalidate_vmap) |
| 613 |
invalidate_kernel_vmap_range(ap->args.vmap_base, nres); |
| 614 |
} |
| 615 |
|
| 616 |
static void fuse_io_release(struct kref *kref) |
| 617 |
{ |
| 618 |
kfree(container_of(kref, struct fuse_io_priv, refcnt)); |
| 619 |
} |
| 620 |
|
| 621 |
static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) |
| 622 |
{ |
| 623 |
if (io->err) |
| 624 |
return io->err; |
| 625 |
|
| 626 |
if (io->bytes >= 0 && io->write) |
| 627 |
return -EIO; |
| 628 |
|
| 629 |
return io->bytes < 0 ? io->size : io->bytes; |
| 630 |
} |
| 631 |
|
| 632 |
/* |
| 633 |
* In case of short read, the caller sets 'pos' to the position of |
| 634 |
* actual end of fuse request in IO request. Otherwise, if bytes_requested |
| 635 |
* == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1. |
| 636 |
* |
| 637 |
* An example: |
| 638 |
* User requested DIO read of 64K. It was split into two 32K fuse requests, |
| 639 |
* both submitted asynchronously. The first of them was ACKed by userspace as |
| 640 |
* fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The |
| 641 |
* second request was ACKed as short, e.g. only 1K was read, resulting in |
| 642 |
* pos == 33K. |
| 643 |
* |
| 644 |
* Thus, when all fuse requests are completed, the minimal non-negative 'pos' |
| 645 |
* will be equal to the length of the longest contiguous fragment of |
| 646 |
* transferred data starting from the beginning of IO request. |
| 647 |
*/ |
| 648 |
static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) |
| 649 |
{ |
| 650 |
int left; |
| 651 |
|
| 652 |
spin_lock(&io->lock); |
| 653 |
if (err) |
| 654 |
io->err = io->err ? : err; |
| 655 |
else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes)) |
| 656 |
io->bytes = pos; |
| 657 |
|
| 658 |
left = --io->reqs; |
| 659 |
if (!left && io->blocking) |
| 660 |
complete(io->done); |
| 661 |
spin_unlock(&io->lock); |
| 662 |
|
| 663 |
if (!left && !io->blocking) { |
| 664 |
ssize_t res = fuse_get_res_by_io(io); |
| 665 |
|
| 666 |
if (res >= 0) { |
| 667 |
struct inode *inode = file_inode(io->iocb->ki_filp); |
| 668 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 669 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 670 |
|
| 671 |
spin_lock(&fi->lock); |
| 672 |
fi->attr_version = atomic64_inc_return(&fc->attr_version); |
| 673 |
spin_unlock(&fi->lock); |
| 674 |
} |
| 675 |
|
| 676 |
io->iocb->ki_complete(io->iocb, res); |
| 677 |
} |
| 678 |
|
| 679 |
kref_put(&io->refcnt, fuse_io_release); |
| 680 |
} |
| 681 |
|
| 682 |
static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, |
| 683 |
unsigned int nfolios) |
| 684 |
{ |
| 685 |
struct fuse_io_args *ia; |
| 686 |
|
| 687 |
ia = kzalloc_obj(*ia); |
| 688 |
if (ia) { |
| 689 |
ia->io = io; |
| 690 |
ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL, |
| 691 |
&ia->ap.descs); |
| 692 |
if (!ia->ap.folios) { |
| 693 |
kfree(ia); |
| 694 |
ia = NULL; |
| 695 |
} |
| 696 |
} |
| 697 |
return ia; |
| 698 |
} |
| 699 |
|
| 700 |
static void fuse_io_free(struct fuse_io_args *ia) |
| 701 |
{ |
| 702 |
kfree(ia->ap.folios); |
| 703 |
kfree(ia); |
| 704 |
} |
| 705 |
|
| 706 |
static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, |
| 707 |
int err) |
| 708 |
{ |
| 709 |
struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); |
| 710 |
struct fuse_io_priv *io = ia->io; |
| 711 |
ssize_t pos = -1; |
| 712 |
size_t nres; |
| 713 |
|
| 714 |
if (err) { |
| 715 |
/* Nothing */ |
| 716 |
} else if (io->write) { |
| 717 |
if (ia->write.out.size > ia->write.in.size) { |
| 718 |
err = -EIO; |
| 719 |
} else { |
| 720 |
nres = ia->write.out.size; |
| 721 |
if (ia->write.in.size != ia->write.out.size) |
| 722 |
pos = ia->write.in.offset - io->offset + |
| 723 |
ia->write.out.size; |
| 724 |
} |
| 725 |
} else { |
| 726 |
u32 outsize = args->out_args[0].size; |
| 727 |
|
| 728 |
nres = outsize; |
| 729 |
if (ia->read.in.size != outsize) |
| 730 |
pos = ia->read.in.offset - io->offset + outsize; |
| 731 |
} |
| 732 |
|
| 733 |
fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty); |
| 734 |
|
| 735 |
fuse_aio_complete(io, err, pos); |
| 736 |
fuse_io_free(ia); |
| 737 |
} |
| 738 |
|
| 739 |
static ssize_t fuse_async_req_send(struct fuse_mount *fm, |
| 740 |
struct fuse_io_args *ia, size_t num_bytes) |
| 741 |
{ |
| 742 |
ssize_t err; |
| 743 |
struct fuse_io_priv *io = ia->io; |
| 744 |
|
| 745 |
spin_lock(&io->lock); |
| 746 |
kref_get(&io->refcnt); |
| 747 |
io->size += num_bytes; |
| 748 |
io->reqs++; |
| 749 |
spin_unlock(&io->lock); |
| 750 |
|
| 751 |
ia->ap.args.end = fuse_aio_complete_req; |
| 752 |
ia->ap.args.may_block = io->should_dirty; |
| 753 |
err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL); |
| 754 |
if (err) |
| 755 |
fuse_aio_complete_req(fm, &ia->ap.args, err); |
| 756 |
|
| 757 |
return num_bytes; |
| 758 |
} |
| 759 |
|
| 760 |
static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count, |
| 761 |
fl_owner_t owner) |
| 762 |
{ |
| 763 |
struct file *file = ia->io->iocb->ki_filp; |
| 764 |
struct fuse_file *ff = file->private_data; |
| 765 |
struct fuse_mount *fm = ff->fm; |
| 766 |
|
| 767 |
fuse_read_args_fill(ia, file, pos, count, FUSE_READ); |
| 768 |
if (owner != NULL) { |
| 769 |
ia->read.in.read_flags |= FUSE_READ_LOCKOWNER; |
| 770 |
ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner); |
| 771 |
} |
| 772 |
|
| 773 |
if (ia->io->async) |
| 774 |
return fuse_async_req_send(fm, ia, count); |
| 775 |
|
| 776 |
return fuse_simple_request(fm, &ia->ap.args); |
| 777 |
} |
| 778 |
|
| 779 |
static void fuse_read_update_size(struct inode *inode, loff_t size, |
| 780 |
u64 attr_ver) |
| 781 |
{ |
| 782 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 783 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 784 |
|
| 785 |
spin_lock(&fi->lock); |
| 786 |
if (attr_ver >= fi->attr_version && size < inode->i_size && |
| 787 |
!test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { |
| 788 |
fi->attr_version = atomic64_inc_return(&fc->attr_version); |
| 789 |
i_size_write(inode, size); |
| 790 |
} |
| 791 |
spin_unlock(&fi->lock); |
| 792 |
} |
| 793 |
|
| 794 |
static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, |
| 795 |
struct fuse_args_pages *ap) |
| 796 |
{ |
| 797 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 798 |
|
| 799 |
/* |
| 800 |
* If writeback_cache is enabled, a short read means there's a hole in |
| 801 |
* the file. Some data after the hole is in page cache, but has not |
| 802 |
* reached the client fs yet. So the hole is not present there. |
| 803 |
*/ |
| 804 |
if (!fc->writeback_cache) { |
| 805 |
loff_t pos = folio_pos(ap->folios[0]) + num_read; |
| 806 |
fuse_read_update_size(inode, pos, attr_ver); |
| 807 |
} |
| 808 |
} |
| 809 |
|
| 810 |
static int fuse_do_readfolio(struct file *file, struct folio *folio, |
| 811 |
size_t off, size_t len) |
| 812 |
{ |
| 813 |
struct inode *inode = folio->mapping->host; |
| 814 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 815 |
loff_t pos = folio_pos(folio) + off; |
| 816 |
struct fuse_folio_desc desc = { |
| 817 |
.offset = off, |
| 818 |
.length = len, |
| 819 |
}; |
| 820 |
struct fuse_io_args ia = { |
| 821 |
.ap.args.page_zeroing = true, |
| 822 |
.ap.args.out_pages = true, |
| 823 |
.ap.num_folios = 1, |
| 824 |
.ap.folios = &folio, |
| 825 |
.ap.descs = &desc, |
| 826 |
}; |
| 827 |
ssize_t res; |
| 828 |
u64 attr_ver; |
| 829 |
|
| 830 |
attr_ver = fuse_get_attr_version(fm->fc); |
| 831 |
|
| 832 |
/* Don't overflow end offset */ |
| 833 |
if (pos + (desc.length - 1) == LLONG_MAX) |
| 834 |
desc.length--; |
| 835 |
|
| 836 |
fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); |
| 837 |
res = fuse_simple_request(fm, &ia.ap.args); |
| 838 |
if (res < 0) |
| 839 |
return res; |
| 840 |
/* |
| 841 |
* Short read means EOF. If file size is larger, truncate it |
| 842 |
*/ |
| 843 |
if (res < desc.length) |
| 844 |
fuse_short_read(inode, attr_ver, res, &ia.ap); |
| 845 |
|
| 846 |
return 0; |
| 847 |
} |
| 848 |
|
| 849 |
static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, |
| 850 |
unsigned int flags, struct iomap *iomap, |
| 851 |
struct iomap *srcmap) |
| 852 |
{ |
| 853 |
iomap->type = IOMAP_MAPPED; |
| 854 |
iomap->length = length; |
| 855 |
iomap->offset = offset; |
| 856 |
return 0; |
| 857 |
} |
| 858 |
|
| 859 |
static const struct iomap_ops fuse_iomap_ops = { |
| 860 |
.iomap_begin = fuse_iomap_begin, |
| 861 |
}; |
| 862 |
|
| 863 |
struct fuse_fill_read_data { |
| 864 |
struct file *file; |
| 865 |
|
| 866 |
/* Fields below are used if sending the read request asynchronously */ |
| 867 |
struct fuse_conn *fc; |
| 868 |
struct fuse_io_args *ia; |
| 869 |
unsigned int nr_bytes; |
| 870 |
}; |
| 871 |
|
| 872 |
/* forward declarations */ |
| 873 |
static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, |
| 874 |
unsigned len, struct fuse_args_pages *ap, |
| 875 |
unsigned cur_bytes, bool write); |
| 876 |
static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, |
| 877 |
unsigned int count, bool async); |
| 878 |
|
| 879 |
static int fuse_handle_readahead(struct folio *folio, |
| 880 |
struct readahead_control *rac, |
| 881 |
struct fuse_fill_read_data *data, loff_t pos, |
| 882 |
size_t len) |
| 883 |
{ |
| 884 |
struct fuse_io_args *ia = data->ia; |
| 885 |
size_t off = offset_in_folio(folio, pos); |
| 886 |
struct fuse_conn *fc = data->fc; |
| 887 |
struct fuse_args_pages *ap; |
| 888 |
unsigned int nr_pages; |
| 889 |
|
| 890 |
if (ia && fuse_folios_need_send(fc, pos, len, &ia->ap, data->nr_bytes, |
| 891 |
false)) { |
| 892 |
fuse_send_readpages(ia, data->file, data->nr_bytes, |
| 893 |
fc->async_read); |
| 894 |
data->nr_bytes = 0; |
| 895 |
data->ia = NULL; |
| 896 |
ia = NULL; |
| 897 |
} |
| 898 |
if (!ia) { |
| 899 |
if (fc->num_background >= fc->congestion_threshold && |
| 900 |
rac->ra->async_size >= readahead_count(rac)) |
| 901 |
/* |
| 902 |
* Congested and only async pages left, so skip the |
| 903 |
* rest. |
| 904 |
*/ |
| 905 |
return -EAGAIN; |
| 906 |
|
| 907 |
nr_pages = min(fc->max_pages, readahead_count(rac)); |
| 908 |
data->ia = fuse_io_alloc(NULL, nr_pages); |
| 909 |
if (!data->ia) |
| 910 |
return -ENOMEM; |
| 911 |
ia = data->ia; |
| 912 |
} |
| 913 |
folio_get(folio); |
| 914 |
ap = &ia->ap; |
| 915 |
ap->folios[ap->num_folios] = folio; |
| 916 |
ap->descs[ap->num_folios].offset = off; |
| 917 |
ap->descs[ap->num_folios].length = len; |
| 918 |
data->nr_bytes += len; |
| 919 |
ap->num_folios++; |
| 920 |
|
| 921 |
return 0; |
| 922 |
} |
| 923 |
|
| 924 |
static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter, |
| 925 |
struct iomap_read_folio_ctx *ctx, |
| 926 |
size_t len) |
| 927 |
{ |
| 928 |
struct fuse_fill_read_data *data = ctx->read_ctx; |
| 929 |
struct folio *folio = ctx->cur_folio; |
| 930 |
loff_t pos = iter->pos; |
| 931 |
size_t off = offset_in_folio(folio, pos); |
| 932 |
struct file *file = data->file; |
| 933 |
int ret; |
| 934 |
|
| 935 |
if (ctx->rac) { |
| 936 |
ret = fuse_handle_readahead(folio, ctx->rac, data, pos, len); |
| 937 |
} else { |
| 938 |
/* |
| 939 |
* for non-readahead read requests, do reads synchronously |
| 940 |
* since it's not guaranteed that the server can handle |
| 941 |
* out-of-order reads |
| 942 |
*/ |
| 943 |
ret = fuse_do_readfolio(file, folio, off, len); |
| 944 |
if (!ret) |
| 945 |
iomap_finish_folio_read(folio, off, len, ret); |
| 946 |
} |
| 947 |
return ret; |
| 948 |
} |
| 949 |
|
| 950 |
static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx) |
| 951 |
{ |
| 952 |
struct fuse_fill_read_data *data = ctx->read_ctx; |
| 953 |
|
| 954 |
if (data->ia) |
| 955 |
fuse_send_readpages(data->ia, data->file, data->nr_bytes, |
| 956 |
data->fc->async_read); |
| 957 |
} |
| 958 |
|
| 959 |
static const struct iomap_read_ops fuse_iomap_read_ops = { |
| 960 |
.read_folio_range = fuse_iomap_read_folio_range_async, |
| 961 |
.submit_read = fuse_iomap_read_submit, |
| 962 |
}; |
| 963 |
|
| 964 |
static int fuse_read_folio(struct file *file, struct folio *folio) |
| 965 |
{ |
| 966 |
struct inode *inode = folio->mapping->host; |
| 967 |
struct fuse_fill_read_data data = { |
| 968 |
.file = file, |
| 969 |
}; |
| 970 |
struct iomap_read_folio_ctx ctx = { |
| 971 |
.cur_folio = folio, |
| 972 |
.ops = &fuse_iomap_read_ops, |
| 973 |
.read_ctx = &data, |
| 974 |
|
| 975 |
}; |
| 976 |
|
| 977 |
if (fuse_is_bad(inode)) { |
| 978 |
folio_unlock(folio); |
| 979 |
return -EIO; |
| 980 |
} |
| 981 |
|
| 982 |
iomap_read_folio(&fuse_iomap_ops, &ctx, NULL); |
| 983 |
fuse_invalidate_atime(inode); |
| 984 |
return 0; |
| 985 |
} |
| 986 |
|
| 987 |
static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, |
| 988 |
struct folio *folio, loff_t pos, |
| 989 |
size_t len) |
| 990 |
{ |
| 991 |
struct file *file = iter->private; |
| 992 |
size_t off = offset_in_folio(folio, pos); |
| 993 |
|
| 994 |
return fuse_do_readfolio(file, folio, off, len); |
| 995 |
} |
| 996 |
|
| 997 |
static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, |
| 998 |
int err) |
| 999 |
{ |
| 1000 |
int i; |
| 1001 |
struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); |
| 1002 |
struct fuse_args_pages *ap = &ia->ap; |
| 1003 |
size_t count = ia->read.in.size; |
| 1004 |
size_t num_read = args->out_args[0].size; |
| 1005 |
struct address_space *mapping; |
| 1006 |
struct inode *inode; |
| 1007 |
|
| 1008 |
WARN_ON_ONCE(!ap->num_folios); |
| 1009 |
mapping = ap->folios[0]->mapping; |
| 1010 |
inode = mapping->host; |
| 1011 |
|
| 1012 |
/* |
| 1013 |
* Short read means EOF. If file size is larger, truncate it |
| 1014 |
*/ |
| 1015 |
if (!err && num_read < count) |
| 1016 |
fuse_short_read(inode, ia->read.attr_ver, num_read, ap); |
| 1017 |
|
| 1018 |
fuse_invalidate_atime(inode); |
| 1019 |
|
| 1020 |
for (i = 0; i < ap->num_folios; i++) { |
| 1021 |
iomap_finish_folio_read(ap->folios[i], ap->descs[i].offset, |
| 1022 |
ap->descs[i].length, err); |
| 1023 |
folio_put(ap->folios[i]); |
| 1024 |
} |
| 1025 |
if (ia->ff) |
| 1026 |
fuse_file_put(ia->ff, false); |
| 1027 |
|
| 1028 |
fuse_io_free(ia); |
| 1029 |
} |
| 1030 |
|
| 1031 |
static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, |
| 1032 |
unsigned int count, bool async) |
| 1033 |
{ |
| 1034 |
struct fuse_file *ff = file->private_data; |
| 1035 |
struct fuse_mount *fm = ff->fm; |
| 1036 |
struct fuse_args_pages *ap = &ia->ap; |
| 1037 |
loff_t pos = folio_pos(ap->folios[0]); |
| 1038 |
ssize_t res; |
| 1039 |
int err; |
| 1040 |
|
| 1041 |
ap->args.out_pages = true; |
| 1042 |
ap->args.page_zeroing = true; |
| 1043 |
ap->args.page_replace = true; |
| 1044 |
|
| 1045 |
/* Don't overflow end offset */ |
| 1046 |
if (pos + (count - 1) == LLONG_MAX) { |
| 1047 |
count--; |
| 1048 |
ap->descs[ap->num_folios - 1].length--; |
| 1049 |
} |
| 1050 |
WARN_ON((loff_t) (pos + count) < 0); |
| 1051 |
|
| 1052 |
fuse_read_args_fill(ia, file, pos, count, FUSE_READ); |
| 1053 |
ia->read.attr_ver = fuse_get_attr_version(fm->fc); |
| 1054 |
if (async) { |
| 1055 |
ia->ff = fuse_file_get(ff); |
| 1056 |
ap->args.end = fuse_readpages_end; |
| 1057 |
err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); |
| 1058 |
if (!err) |
| 1059 |
return; |
| 1060 |
} else { |
| 1061 |
res = fuse_simple_request(fm, &ap->args); |
| 1062 |
err = res < 0 ? res : 0; |
| 1063 |
} |
| 1064 |
fuse_readpages_end(fm, &ap->args, err); |
| 1065 |
} |
| 1066 |
|
| 1067 |
static void fuse_readahead(struct readahead_control *rac) |
| 1068 |
{ |
| 1069 |
struct inode *inode = rac->mapping->host; |
| 1070 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1071 |
struct fuse_fill_read_data data = { |
| 1072 |
.file = rac->file, |
| 1073 |
.fc = fc, |
| 1074 |
}; |
| 1075 |
struct iomap_read_folio_ctx ctx = { |
| 1076 |
.ops = &fuse_iomap_read_ops, |
| 1077 |
.rac = rac, |
| 1078 |
.read_ctx = &data |
| 1079 |
}; |
| 1080 |
|
| 1081 |
if (fuse_is_bad(inode)) |
| 1082 |
return; |
| 1083 |
|
| 1084 |
iomap_readahead(&fuse_iomap_ops, &ctx, NULL); |
| 1085 |
} |
| 1086 |
|
| 1087 |
static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) |
| 1088 |
{ |
| 1089 |
struct inode *inode = iocb->ki_filp->f_mapping->host; |
| 1090 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1091 |
|
| 1092 |
/* |
| 1093 |
* In auto invalidate mode, always update attributes on read. |
| 1094 |
* Otherwise, only update if we attempt to read past EOF (to ensure |
| 1095 |
* i_size is up to date). |
| 1096 |
*/ |
| 1097 |
if (fc->auto_inval_data || |
| 1098 |
(iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { |
| 1099 |
int err; |
| 1100 |
err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE); |
| 1101 |
if (err) |
| 1102 |
return err; |
| 1103 |
} |
| 1104 |
|
| 1105 |
return generic_file_read_iter(iocb, to); |
| 1106 |
} |
| 1107 |
|
| 1108 |
static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, |
| 1109 |
loff_t pos, size_t count) |
| 1110 |
{ |
| 1111 |
struct fuse_args *args = &ia->ap.args; |
| 1112 |
|
| 1113 |
ia->write.in.fh = ff->fh; |
| 1114 |
ia->write.in.offset = pos; |
| 1115 |
ia->write.in.size = count; |
| 1116 |
args->opcode = FUSE_WRITE; |
| 1117 |
args->nodeid = ff->nodeid; |
| 1118 |
args->in_numargs = 2; |
| 1119 |
if (ff->fm->fc->minor < 9) |
| 1120 |
args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; |
| 1121 |
else |
| 1122 |
args->in_args[0].size = sizeof(ia->write.in); |
| 1123 |
args->in_args[0].value = &ia->write.in; |
| 1124 |
args->in_args[1].size = count; |
| 1125 |
args->out_numargs = 1; |
| 1126 |
args->out_args[0].size = sizeof(ia->write.out); |
| 1127 |
args->out_args[0].value = &ia->write.out; |
| 1128 |
} |
| 1129 |
|
| 1130 |
static unsigned int fuse_write_flags(struct kiocb *iocb) |
| 1131 |
{ |
| 1132 |
unsigned int flags = iocb->ki_filp->f_flags; |
| 1133 |
|
| 1134 |
if (iocb_is_dsync(iocb)) |
| 1135 |
flags |= O_DSYNC; |
| 1136 |
if (iocb->ki_flags & IOCB_SYNC) |
| 1137 |
flags |= O_SYNC; |
| 1138 |
|
| 1139 |
return flags; |
| 1140 |
} |
| 1141 |
|
| 1142 |
static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos, |
| 1143 |
size_t count, fl_owner_t owner) |
| 1144 |
{ |
| 1145 |
struct kiocb *iocb = ia->io->iocb; |
| 1146 |
struct file *file = iocb->ki_filp; |
| 1147 |
struct fuse_file *ff = file->private_data; |
| 1148 |
struct fuse_mount *fm = ff->fm; |
| 1149 |
struct fuse_write_in *inarg = &ia->write.in; |
| 1150 |
ssize_t err; |
| 1151 |
|
| 1152 |
fuse_write_args_fill(ia, ff, pos, count); |
| 1153 |
inarg->flags = fuse_write_flags(iocb); |
| 1154 |
if (owner != NULL) { |
| 1155 |
inarg->write_flags |= FUSE_WRITE_LOCKOWNER; |
| 1156 |
inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner); |
| 1157 |
} |
| 1158 |
|
| 1159 |
if (ia->io->async) |
| 1160 |
return fuse_async_req_send(fm, ia, count); |
| 1161 |
|
| 1162 |
err = fuse_simple_request(fm, &ia->ap.args); |
| 1163 |
if (!err && ia->write.out.size > count) |
| 1164 |
err = -EIO; |
| 1165 |
|
| 1166 |
return err ?: ia->write.out.size; |
| 1167 |
} |
| 1168 |
|
| 1169 |
bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) |
| 1170 |
{ |
| 1171 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1172 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1173 |
bool ret = false; |
| 1174 |
|
| 1175 |
spin_lock(&fi->lock); |
| 1176 |
fi->attr_version = atomic64_inc_return(&fc->attr_version); |
| 1177 |
if (written > 0 && pos > inode->i_size) { |
| 1178 |
i_size_write(inode, pos); |
| 1179 |
ret = true; |
| 1180 |
} |
| 1181 |
spin_unlock(&fi->lock); |
| 1182 |
|
| 1183 |
fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); |
| 1184 |
|
| 1185 |
return ret; |
| 1186 |
} |
| 1187 |
|
| 1188 |
static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, |
| 1189 |
struct kiocb *iocb, struct inode *inode, |
| 1190 |
loff_t pos, size_t count) |
| 1191 |
{ |
| 1192 |
struct fuse_args_pages *ap = &ia->ap; |
| 1193 |
struct file *file = iocb->ki_filp; |
| 1194 |
struct fuse_file *ff = file->private_data; |
| 1195 |
struct fuse_mount *fm = ff->fm; |
| 1196 |
unsigned int offset, i; |
| 1197 |
bool short_write; |
| 1198 |
int err; |
| 1199 |
|
| 1200 |
for (i = 0; i < ap->num_folios; i++) |
| 1201 |
folio_wait_writeback(ap->folios[i]); |
| 1202 |
|
| 1203 |
fuse_write_args_fill(ia, ff, pos, count); |
| 1204 |
ia->write.in.flags = fuse_write_flags(iocb); |
| 1205 |
if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) |
| 1206 |
ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; |
| 1207 |
|
| 1208 |
err = fuse_simple_request(fm, &ap->args); |
| 1209 |
if (!err && ia->write.out.size > count) |
| 1210 |
err = -EIO; |
| 1211 |
|
| 1212 |
short_write = ia->write.out.size < count; |
| 1213 |
offset = ap->descs[0].offset; |
| 1214 |
count = ia->write.out.size; |
| 1215 |
for (i = 0; i < ap->num_folios; i++) { |
| 1216 |
struct folio *folio = ap->folios[i]; |
| 1217 |
|
| 1218 |
if (err) { |
| 1219 |
folio_clear_uptodate(folio); |
| 1220 |
} else { |
| 1221 |
if (count >= folio_size(folio) - offset) |
| 1222 |
count -= folio_size(folio) - offset; |
| 1223 |
else { |
| 1224 |
if (short_write) |
| 1225 |
folio_clear_uptodate(folio); |
| 1226 |
count = 0; |
| 1227 |
} |
| 1228 |
offset = 0; |
| 1229 |
} |
| 1230 |
if (ia->write.folio_locked && (i == ap->num_folios - 1)) |
| 1231 |
folio_unlock(folio); |
| 1232 |
folio_put(folio); |
| 1233 |
} |
| 1234 |
|
| 1235 |
return err; |
| 1236 |
} |
| 1237 |
|
| 1238 |
static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, |
| 1239 |
struct address_space *mapping, |
| 1240 |
struct iov_iter *ii, loff_t pos, |
| 1241 |
unsigned int max_folios) |
| 1242 |
{ |
| 1243 |
struct fuse_args_pages *ap = &ia->ap; |
| 1244 |
struct fuse_conn *fc = get_fuse_conn(mapping->host); |
| 1245 |
unsigned offset = pos & (PAGE_SIZE - 1); |
| 1246 |
size_t count = 0; |
| 1247 |
unsigned int num; |
| 1248 |
int err = 0; |
| 1249 |
|
| 1250 |
num = min(iov_iter_count(ii), fc->max_write); |
| 1251 |
|
| 1252 |
ap->args.in_pages = true; |
| 1253 |
|
| 1254 |
while (num && ap->num_folios < max_folios) { |
| 1255 |
size_t tmp; |
| 1256 |
struct folio *folio; |
| 1257 |
pgoff_t index = pos >> PAGE_SHIFT; |
| 1258 |
unsigned int bytes; |
| 1259 |
unsigned int folio_offset; |
| 1260 |
|
| 1261 |
again: |
| 1262 |
folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, |
| 1263 |
mapping_gfp_mask(mapping)); |
| 1264 |
if (IS_ERR(folio)) { |
| 1265 |
err = PTR_ERR(folio); |
| 1266 |
break; |
| 1267 |
} |
| 1268 |
|
| 1269 |
if (mapping_writably_mapped(mapping)) |
| 1270 |
flush_dcache_folio(folio); |
| 1271 |
|
| 1272 |
folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; |
| 1273 |
bytes = min(folio_size(folio) - folio_offset, num); |
| 1274 |
|
| 1275 |
tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii); |
| 1276 |
flush_dcache_folio(folio); |
| 1277 |
|
| 1278 |
if (!tmp) { |
| 1279 |
folio_unlock(folio); |
| 1280 |
folio_put(folio); |
| 1281 |
|
| 1282 |
/* |
| 1283 |
* Ensure forward progress by faulting in |
| 1284 |
* while not holding the folio lock: |
| 1285 |
*/ |
| 1286 |
if (fault_in_iov_iter_readable(ii, bytes)) { |
| 1287 |
err = -EFAULT; |
| 1288 |
break; |
| 1289 |
} |
| 1290 |
|
| 1291 |
goto again; |
| 1292 |
} |
| 1293 |
|
| 1294 |
ap->folios[ap->num_folios] = folio; |
| 1295 |
ap->descs[ap->num_folios].offset = folio_offset; |
| 1296 |
ap->descs[ap->num_folios].length = tmp; |
| 1297 |
ap->num_folios++; |
| 1298 |
|
| 1299 |
count += tmp; |
| 1300 |
pos += tmp; |
| 1301 |
num -= tmp; |
| 1302 |
offset += tmp; |
| 1303 |
if (offset == folio_size(folio)) |
| 1304 |
offset = 0; |
| 1305 |
|
| 1306 |
/* If we copied full folio, mark it uptodate */ |
| 1307 |
if (tmp == folio_size(folio)) |
| 1308 |
folio_mark_uptodate(folio); |
| 1309 |
|
| 1310 |
if (folio_test_uptodate(folio)) { |
| 1311 |
folio_unlock(folio); |
| 1312 |
} else { |
| 1313 |
ia->write.folio_locked = true; |
| 1314 |
break; |
| 1315 |
} |
| 1316 |
if (!fc->big_writes || offset != 0) |
| 1317 |
break; |
| 1318 |
} |
| 1319 |
|
| 1320 |
return count > 0 ? count : err; |
| 1321 |
} |
| 1322 |
|
| 1323 |
static inline unsigned int fuse_wr_pages(loff_t pos, size_t len, |
| 1324 |
unsigned int max_pages) |
| 1325 |
{ |
| 1326 |
unsigned int pages = ((pos + len - 1) >> PAGE_SHIFT) - |
| 1327 |
(pos >> PAGE_SHIFT) + 1; |
| 1328 |
|
| 1329 |
return min(pages, max_pages); |
| 1330 |
} |
| 1331 |
|
| 1332 |
static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) |
| 1333 |
{ |
| 1334 |
struct address_space *mapping = iocb->ki_filp->f_mapping; |
| 1335 |
struct inode *inode = mapping->host; |
| 1336 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1337 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1338 |
loff_t pos = iocb->ki_pos; |
| 1339 |
int err = 0; |
| 1340 |
ssize_t res = 0; |
| 1341 |
|
| 1342 |
if (inode->i_size < pos + iov_iter_count(ii)) |
| 1343 |
set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); |
| 1344 |
|
| 1345 |
do { |
| 1346 |
ssize_t count; |
| 1347 |
struct fuse_io_args ia = {}; |
| 1348 |
struct fuse_args_pages *ap = &ia.ap; |
| 1349 |
unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), |
| 1350 |
fc->max_pages); |
| 1351 |
|
| 1352 |
ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); |
| 1353 |
if (!ap->folios) { |
| 1354 |
err = -ENOMEM; |
| 1355 |
break; |
| 1356 |
} |
| 1357 |
|
| 1358 |
count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages); |
| 1359 |
if (count <= 0) { |
| 1360 |
err = count; |
| 1361 |
} else { |
| 1362 |
err = fuse_send_write_pages(&ia, iocb, inode, |
| 1363 |
pos, count); |
| 1364 |
if (!err) { |
| 1365 |
size_t num_written = ia.write.out.size; |
| 1366 |
|
| 1367 |
res += num_written; |
| 1368 |
pos += num_written; |
| 1369 |
|
| 1370 |
/* break out of the loop on short write */ |
| 1371 |
if (num_written != count) |
| 1372 |
err = -EIO; |
| 1373 |
} |
| 1374 |
} |
| 1375 |
kfree(ap->folios); |
| 1376 |
} while (!err && iov_iter_count(ii)); |
| 1377 |
|
| 1378 |
fuse_write_update_attr(inode, pos, res); |
| 1379 |
clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); |
| 1380 |
|
| 1381 |
if (!res) |
| 1382 |
return err; |
| 1383 |
iocb->ki_pos += res; |
| 1384 |
return res; |
| 1385 |
} |
| 1386 |
|
| 1387 |
static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter) |
| 1388 |
{ |
| 1389 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 1390 |
|
| 1391 |
return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode); |
| 1392 |
} |
| 1393 |
|
| 1394 |
/* |
| 1395 |
* @return true if an exclusive lock for direct IO writes is needed |
| 1396 |
*/ |
| 1397 |
static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from) |
| 1398 |
{ |
| 1399 |
struct file *file = iocb->ki_filp; |
| 1400 |
struct fuse_file *ff = file->private_data; |
| 1401 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 1402 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1403 |
|
| 1404 |
/* Server side has to advise that it supports parallel dio writes. */ |
| 1405 |
if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) |
| 1406 |
return true; |
| 1407 |
|
| 1408 |
/* |
| 1409 |
* Append will need to know the eventual EOF - always needs an |
| 1410 |
* exclusive lock. |
| 1411 |
*/ |
| 1412 |
if (iocb->ki_flags & IOCB_APPEND) |
| 1413 |
return true; |
| 1414 |
|
| 1415 |
/* shared locks are not allowed with parallel page cache IO */ |
| 1416 |
if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) |
| 1417 |
return true; |
| 1418 |
|
| 1419 |
/* Parallel dio beyond EOF is not supported, at least for now. */ |
| 1420 |
if (fuse_io_past_eof(iocb, from)) |
| 1421 |
return true; |
| 1422 |
|
| 1423 |
return false; |
| 1424 |
} |
| 1425 |
|
| 1426 |
static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, |
| 1427 |
bool *exclusive) |
| 1428 |
{ |
| 1429 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 1430 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1431 |
|
| 1432 |
*exclusive = fuse_dio_wr_exclusive_lock(iocb, from); |
| 1433 |
if (*exclusive) { |
| 1434 |
inode_lock(inode); |
| 1435 |
} else { |
| 1436 |
inode_lock_shared(inode); |
| 1437 |
/* |
| 1438 |
* New parallal dio allowed only if inode is not in caching |
| 1439 |
* mode and denies new opens in caching mode. This check |
| 1440 |
* should be performed only after taking shared inode lock. |
| 1441 |
* Previous past eof check was without inode lock and might |
| 1442 |
* have raced, so check it again. |
| 1443 |
*/ |
| 1444 |
if (fuse_io_past_eof(iocb, from) || |
| 1445 |
fuse_inode_uncached_io_start(fi, NULL) != 0) { |
| 1446 |
inode_unlock_shared(inode); |
| 1447 |
inode_lock(inode); |
| 1448 |
*exclusive = true; |
| 1449 |
} |
| 1450 |
} |
| 1451 |
} |
| 1452 |
|
| 1453 |
static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) |
| 1454 |
{ |
| 1455 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 1456 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1457 |
|
| 1458 |
if (exclusive) { |
| 1459 |
inode_unlock(inode); |
| 1460 |
} else { |
| 1461 |
/* Allow opens in caching mode after last parallel dio end */ |
| 1462 |
fuse_inode_uncached_io_end(fi); |
| 1463 |
inode_unlock_shared(inode); |
| 1464 |
} |
| 1465 |
} |
| 1466 |
|
| 1467 |
static const struct iomap_write_ops fuse_iomap_write_ops = { |
| 1468 |
.read_folio_range = fuse_iomap_read_folio_range, |
| 1469 |
}; |
| 1470 |
|
| 1471 |
static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) |
| 1472 |
{ |
| 1473 |
struct file *file = iocb->ki_filp; |
| 1474 |
struct mnt_idmap *idmap = file_mnt_idmap(file); |
| 1475 |
struct address_space *mapping = file->f_mapping; |
| 1476 |
ssize_t written = 0; |
| 1477 |
struct inode *inode = mapping->host; |
| 1478 |
ssize_t err, count; |
| 1479 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1480 |
bool writeback = false; |
| 1481 |
|
| 1482 |
if (fc->writeback_cache) { |
| 1483 |
/* Update size (EOF optimization) and mode (SUID clearing) */ |
| 1484 |
err = fuse_update_attributes(mapping->host, file, |
| 1485 |
STATX_SIZE | STATX_MODE); |
| 1486 |
if (err) |
| 1487 |
return err; |
| 1488 |
|
| 1489 |
if (!fc->handle_killpriv_v2 || |
| 1490 |
!setattr_should_drop_suidgid(idmap, file_inode(file))) |
| 1491 |
writeback = true; |
| 1492 |
} |
| 1493 |
|
| 1494 |
inode_lock(inode); |
| 1495 |
|
| 1496 |
err = count = generic_write_checks(iocb, from); |
| 1497 |
if (err <= 0) |
| 1498 |
goto out; |
| 1499 |
|
| 1500 |
task_io_account_write(count); |
| 1501 |
|
| 1502 |
err = kiocb_modified(iocb); |
| 1503 |
if (err) |
| 1504 |
goto out; |
| 1505 |
|
| 1506 |
if (iocb->ki_flags & IOCB_DIRECT) { |
| 1507 |
written = generic_file_direct_write(iocb, from); |
| 1508 |
if (written < 0 || !iov_iter_count(from)) |
| 1509 |
goto out; |
| 1510 |
written = direct_write_fallback(iocb, from, written, |
| 1511 |
fuse_perform_write(iocb, from)); |
| 1512 |
} else if (writeback) { |
| 1513 |
/* |
| 1514 |
* Use iomap so that we can do granular uptodate reads |
| 1515 |
* and granular dirty tracking for large folios. |
| 1516 |
*/ |
| 1517 |
written = iomap_file_buffered_write(iocb, from, |
| 1518 |
&fuse_iomap_ops, |
| 1519 |
&fuse_iomap_write_ops, |
| 1520 |
file); |
| 1521 |
} else { |
| 1522 |
written = fuse_perform_write(iocb, from); |
| 1523 |
} |
| 1524 |
out: |
| 1525 |
inode_unlock(inode); |
| 1526 |
if (written > 0) |
| 1527 |
written = generic_write_sync(iocb, written); |
| 1528 |
|
| 1529 |
return written ? written : err; |
| 1530 |
} |
| 1531 |
|
| 1532 |
static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii) |
| 1533 |
{ |
| 1534 |
return (unsigned long)iter_iov(ii)->iov_base + ii->iov_offset; |
| 1535 |
} |
| 1536 |
|
| 1537 |
static inline size_t fuse_get_frag_size(const struct iov_iter *ii, |
| 1538 |
size_t max_size) |
| 1539 |
{ |
| 1540 |
return min(iov_iter_single_seg_count(ii), max_size); |
| 1541 |
} |
| 1542 |
|
| 1543 |
static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, |
| 1544 |
size_t *nbytesp, int write, |
| 1545 |
unsigned int max_pages, |
| 1546 |
bool use_pages_for_kvec_io) |
| 1547 |
{ |
| 1548 |
bool flush_or_invalidate = false; |
| 1549 |
unsigned int nr_pages = 0; |
| 1550 |
size_t nbytes = 0; /* # bytes already packed in req */ |
| 1551 |
ssize_t ret = 0; |
| 1552 |
|
| 1553 |
/* Special case for kernel I/O: can copy directly into the buffer. |
| 1554 |
* However if the implementation of fuse_conn requires pages instead of |
| 1555 |
* pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead. |
| 1556 |
*/ |
| 1557 |
if (iov_iter_is_kvec(ii)) { |
| 1558 |
void *user_addr = (void *)fuse_get_user_addr(ii); |
| 1559 |
|
| 1560 |
if (!use_pages_for_kvec_io) { |
| 1561 |
size_t frag_size = fuse_get_frag_size(ii, *nbytesp); |
| 1562 |
|
| 1563 |
if (write) |
| 1564 |
ap->args.in_args[1].value = user_addr; |
| 1565 |
else |
| 1566 |
ap->args.out_args[0].value = user_addr; |
| 1567 |
|
| 1568 |
iov_iter_advance(ii, frag_size); |
| 1569 |
*nbytesp = frag_size; |
| 1570 |
return 0; |
| 1571 |
} |
| 1572 |
|
| 1573 |
if (is_vmalloc_addr(user_addr)) { |
| 1574 |
ap->args.vmap_base = user_addr; |
| 1575 |
flush_or_invalidate = true; |
| 1576 |
} |
| 1577 |
} |
| 1578 |
|
| 1579 |
/* |
| 1580 |
* Until there is support for iov_iter_extract_folios(), we have to |
| 1581 |
* manually extract pages using iov_iter_extract_pages() and then |
| 1582 |
* copy that to a folios array. |
| 1583 |
*/ |
| 1584 |
struct page **pages = kzalloc(max_pages * sizeof(struct page *), |
| 1585 |
GFP_KERNEL); |
| 1586 |
if (!pages) { |
| 1587 |
ret = -ENOMEM; |
| 1588 |
goto out; |
| 1589 |
} |
| 1590 |
|
| 1591 |
while (nbytes < *nbytesp && nr_pages < max_pages) { |
| 1592 |
unsigned nfolios, i; |
| 1593 |
size_t start; |
| 1594 |
|
| 1595 |
ret = iov_iter_extract_pages(ii, &pages, |
| 1596 |
*nbytesp - nbytes, |
| 1597 |
max_pages - nr_pages, |
| 1598 |
0, &start); |
| 1599 |
if (ret < 0) |
| 1600 |
break; |
| 1601 |
|
| 1602 |
nbytes += ret; |
| 1603 |
|
| 1604 |
nfolios = DIV_ROUND_UP(ret + start, PAGE_SIZE); |
| 1605 |
|
| 1606 |
for (i = 0; i < nfolios; i++) { |
| 1607 |
struct folio *folio = page_folio(pages[i]); |
| 1608 |
unsigned int offset = start + |
| 1609 |
(folio_page_idx(folio, pages[i]) << PAGE_SHIFT); |
| 1610 |
unsigned int len = umin(ret, PAGE_SIZE - start); |
| 1611 |
|
| 1612 |
ap->descs[ap->num_folios].offset = offset; |
| 1613 |
ap->descs[ap->num_folios].length = len; |
| 1614 |
ap->folios[ap->num_folios] = folio; |
| 1615 |
start = 0; |
| 1616 |
ret -= len; |
| 1617 |
ap->num_folios++; |
| 1618 |
} |
| 1619 |
|
| 1620 |
nr_pages += nfolios; |
| 1621 |
} |
| 1622 |
kfree(pages); |
| 1623 |
|
| 1624 |
if (write && flush_or_invalidate) |
| 1625 |
flush_kernel_vmap_range(ap->args.vmap_base, nbytes); |
| 1626 |
|
| 1627 |
ap->args.invalidate_vmap = !write && flush_or_invalidate; |
| 1628 |
ap->args.is_pinned = iov_iter_extract_will_pin(ii); |
| 1629 |
ap->args.user_pages = true; |
| 1630 |
if (write) |
| 1631 |
ap->args.in_pages = true; |
| 1632 |
else |
| 1633 |
ap->args.out_pages = true; |
| 1634 |
|
| 1635 |
out: |
| 1636 |
*nbytesp = nbytes; |
| 1637 |
|
| 1638 |
return ret < 0 ? ret : 0; |
| 1639 |
} |
| 1640 |
|
| 1641 |
ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, |
| 1642 |
loff_t *ppos, int flags) |
| 1643 |
{ |
| 1644 |
int write = flags & FUSE_DIO_WRITE; |
| 1645 |
int cuse = flags & FUSE_DIO_CUSE; |
| 1646 |
struct file *file = io->iocb->ki_filp; |
| 1647 |
struct address_space *mapping = file->f_mapping; |
| 1648 |
struct inode *inode = mapping->host; |
| 1649 |
struct fuse_file *ff = file->private_data; |
| 1650 |
struct fuse_conn *fc = ff->fm->fc; |
| 1651 |
size_t nmax = write ? fc->max_write : fc->max_read; |
| 1652 |
loff_t pos = *ppos; |
| 1653 |
size_t count = iov_iter_count(iter); |
| 1654 |
pgoff_t idx_from = pos >> PAGE_SHIFT; |
| 1655 |
pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT; |
| 1656 |
ssize_t res = 0; |
| 1657 |
int err = 0; |
| 1658 |
struct fuse_io_args *ia; |
| 1659 |
unsigned int max_pages; |
| 1660 |
bool fopen_direct_io = ff->open_flags & FOPEN_DIRECT_IO; |
| 1661 |
|
| 1662 |
max_pages = iov_iter_npages(iter, fc->max_pages); |
| 1663 |
ia = fuse_io_alloc(io, max_pages); |
| 1664 |
if (!ia) |
| 1665 |
return -ENOMEM; |
| 1666 |
|
| 1667 |
if (fopen_direct_io) { |
| 1668 |
res = filemap_write_and_wait_range(mapping, pos, pos + count - 1); |
| 1669 |
if (res) { |
| 1670 |
fuse_io_free(ia); |
| 1671 |
return res; |
| 1672 |
} |
| 1673 |
} |
| 1674 |
if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) { |
| 1675 |
if (!write) |
| 1676 |
inode_lock(inode); |
| 1677 |
fuse_sync_writes(inode); |
| 1678 |
if (!write) |
| 1679 |
inode_unlock(inode); |
| 1680 |
} |
| 1681 |
|
| 1682 |
if (fopen_direct_io && write) { |
| 1683 |
res = invalidate_inode_pages2_range(mapping, idx_from, idx_to); |
| 1684 |
if (res) { |
| 1685 |
fuse_io_free(ia); |
| 1686 |
return res; |
| 1687 |
} |
| 1688 |
} |
| 1689 |
|
| 1690 |
io->should_dirty = !write && user_backed_iter(iter); |
| 1691 |
while (count) { |
| 1692 |
ssize_t nres; |
| 1693 |
fl_owner_t owner = current->files; |
| 1694 |
size_t nbytes = min(count, nmax); |
| 1695 |
|
| 1696 |
err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, |
| 1697 |
max_pages, fc->use_pages_for_kvec_io); |
| 1698 |
if (err && !nbytes) |
| 1699 |
break; |
| 1700 |
|
| 1701 |
if (write) { |
| 1702 |
if (!capable(CAP_FSETID)) |
| 1703 |
ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; |
| 1704 |
|
| 1705 |
nres = fuse_send_write(ia, pos, nbytes, owner); |
| 1706 |
} else { |
| 1707 |
nres = fuse_send_read(ia, pos, nbytes, owner); |
| 1708 |
} |
| 1709 |
|
| 1710 |
if (!io->async || nres < 0) { |
| 1711 |
fuse_release_user_pages(&ia->ap, nres, io->should_dirty); |
| 1712 |
fuse_io_free(ia); |
| 1713 |
} |
| 1714 |
ia = NULL; |
| 1715 |
if (nres < 0) { |
| 1716 |
iov_iter_revert(iter, nbytes); |
| 1717 |
err = nres; |
| 1718 |
break; |
| 1719 |
} |
| 1720 |
WARN_ON(nres > nbytes); |
| 1721 |
|
| 1722 |
count -= nres; |
| 1723 |
res += nres; |
| 1724 |
pos += nres; |
| 1725 |
if (nres != nbytes) { |
| 1726 |
iov_iter_revert(iter, nbytes - nres); |
| 1727 |
break; |
| 1728 |
} |
| 1729 |
if (count) { |
| 1730 |
max_pages = iov_iter_npages(iter, fc->max_pages); |
| 1731 |
ia = fuse_io_alloc(io, max_pages); |
| 1732 |
if (!ia) |
| 1733 |
break; |
| 1734 |
} |
| 1735 |
} |
| 1736 |
if (ia) |
| 1737 |
fuse_io_free(ia); |
| 1738 |
if (res > 0) |
| 1739 |
*ppos = pos; |
| 1740 |
|
| 1741 |
if (res > 0 && write && fopen_direct_io) { |
| 1742 |
/* |
| 1743 |
* As in generic_file_direct_write(), invalidate after the |
| 1744 |
* write, to invalidate read-ahead cache that may have competed |
| 1745 |
* with the write. |
| 1746 |
*/ |
| 1747 |
invalidate_inode_pages2_range(mapping, idx_from, idx_to); |
| 1748 |
} |
| 1749 |
|
| 1750 |
return res > 0 ? res : err; |
| 1751 |
} |
| 1752 |
EXPORT_SYMBOL_GPL(fuse_direct_io); |
| 1753 |
|
| 1754 |
static ssize_t __fuse_direct_read(struct fuse_io_priv *io, |
| 1755 |
struct iov_iter *iter, |
| 1756 |
loff_t *ppos) |
| 1757 |
{ |
| 1758 |
ssize_t res; |
| 1759 |
struct inode *inode = file_inode(io->iocb->ki_filp); |
| 1760 |
|
| 1761 |
res = fuse_direct_io(io, iter, ppos, 0); |
| 1762 |
|
| 1763 |
fuse_invalidate_atime(inode); |
| 1764 |
|
| 1765 |
return res; |
| 1766 |
} |
| 1767 |
|
| 1768 |
static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter); |
| 1769 |
|
| 1770 |
static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) |
| 1771 |
{ |
| 1772 |
ssize_t res; |
| 1773 |
|
| 1774 |
if (!is_sync_kiocb(iocb)) { |
| 1775 |
res = fuse_direct_IO(iocb, to); |
| 1776 |
} else { |
| 1777 |
struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); |
| 1778 |
|
| 1779 |
res = __fuse_direct_read(&io, to, &iocb->ki_pos); |
| 1780 |
} |
| 1781 |
|
| 1782 |
return res; |
| 1783 |
} |
| 1784 |
|
| 1785 |
static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) |
| 1786 |
{ |
| 1787 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 1788 |
ssize_t res; |
| 1789 |
bool exclusive; |
| 1790 |
|
| 1791 |
fuse_dio_lock(iocb, from, &exclusive); |
| 1792 |
res = generic_write_checks(iocb, from); |
| 1793 |
if (res > 0) { |
| 1794 |
task_io_account_write(res); |
| 1795 |
if (!is_sync_kiocb(iocb)) { |
| 1796 |
res = fuse_direct_IO(iocb, from); |
| 1797 |
} else { |
| 1798 |
struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); |
| 1799 |
|
| 1800 |
res = fuse_direct_io(&io, from, &iocb->ki_pos, |
| 1801 |
FUSE_DIO_WRITE); |
| 1802 |
fuse_write_update_attr(inode, iocb->ki_pos, res); |
| 1803 |
} |
| 1804 |
} |
| 1805 |
fuse_dio_unlock(iocb, exclusive); |
| 1806 |
|
| 1807 |
return res; |
| 1808 |
} |
| 1809 |
|
| 1810 |
static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) |
| 1811 |
{ |
| 1812 |
struct file *file = iocb->ki_filp; |
| 1813 |
struct fuse_file *ff = file->private_data; |
| 1814 |
struct inode *inode = file_inode(file); |
| 1815 |
|
| 1816 |
if (fuse_is_bad(inode)) |
| 1817 |
return -EIO; |
| 1818 |
|
| 1819 |
if (FUSE_IS_DAX(inode)) |
| 1820 |
return fuse_dax_read_iter(iocb, to); |
| 1821 |
|
| 1822 |
/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ |
| 1823 |
if (ff->open_flags & FOPEN_DIRECT_IO) |
| 1824 |
return fuse_direct_read_iter(iocb, to); |
| 1825 |
else if (fuse_file_passthrough(ff)) |
| 1826 |
return fuse_passthrough_read_iter(iocb, to); |
| 1827 |
else |
| 1828 |
return fuse_cache_read_iter(iocb, to); |
| 1829 |
} |
| 1830 |
|
| 1831 |
static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) |
| 1832 |
{ |
| 1833 |
struct file *file = iocb->ki_filp; |
| 1834 |
struct fuse_file *ff = file->private_data; |
| 1835 |
struct inode *inode = file_inode(file); |
| 1836 |
|
| 1837 |
if (fuse_is_bad(inode)) |
| 1838 |
return -EIO; |
| 1839 |
|
| 1840 |
if (FUSE_IS_DAX(inode)) |
| 1841 |
return fuse_dax_write_iter(iocb, from); |
| 1842 |
|
| 1843 |
/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ |
| 1844 |
if (ff->open_flags & FOPEN_DIRECT_IO) |
| 1845 |
return fuse_direct_write_iter(iocb, from); |
| 1846 |
else if (fuse_file_passthrough(ff)) |
| 1847 |
return fuse_passthrough_write_iter(iocb, from); |
| 1848 |
else |
| 1849 |
return fuse_cache_write_iter(iocb, from); |
| 1850 |
} |
| 1851 |
|
| 1852 |
static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, |
| 1853 |
struct pipe_inode_info *pipe, size_t len, |
| 1854 |
unsigned int flags) |
| 1855 |
{ |
| 1856 |
struct fuse_file *ff = in->private_data; |
| 1857 |
|
| 1858 |
/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ |
| 1859 |
if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) |
| 1860 |
return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); |
| 1861 |
else |
| 1862 |
return filemap_splice_read(in, ppos, pipe, len, flags); |
| 1863 |
} |
| 1864 |
|
| 1865 |
static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, |
| 1866 |
loff_t *ppos, size_t len, unsigned int flags) |
| 1867 |
{ |
| 1868 |
struct fuse_file *ff = out->private_data; |
| 1869 |
|
| 1870 |
/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ |
| 1871 |
if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) |
| 1872 |
return fuse_passthrough_splice_write(pipe, out, ppos, len, flags); |
| 1873 |
else |
| 1874 |
return iter_file_splice_write(pipe, out, ppos, len, flags); |
| 1875 |
} |
| 1876 |
|
| 1877 |
static void fuse_writepage_free(struct fuse_writepage_args *wpa) |
| 1878 |
{ |
| 1879 |
struct fuse_args_pages *ap = &wpa->ia.ap; |
| 1880 |
|
| 1881 |
if (wpa->bucket) |
| 1882 |
fuse_sync_bucket_dec(wpa->bucket); |
| 1883 |
|
| 1884 |
fuse_file_put(wpa->ia.ff, false); |
| 1885 |
|
| 1886 |
kfree(ap->folios); |
| 1887 |
kfree(wpa); |
| 1888 |
} |
| 1889 |
|
| 1890 |
static void fuse_writepage_finish(struct fuse_writepage_args *wpa) |
| 1891 |
{ |
| 1892 |
struct fuse_args_pages *ap = &wpa->ia.ap; |
| 1893 |
struct inode *inode = wpa->inode; |
| 1894 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1895 |
int i; |
| 1896 |
|
| 1897 |
for (i = 0; i < ap->num_folios; i++) |
| 1898 |
/* |
| 1899 |
* Benchmarks showed that ending writeback within the |
| 1900 |
* scope of the fi->lock alleviates xarray lock |
| 1901 |
* contention and noticeably improves performance. |
| 1902 |
*/ |
| 1903 |
iomap_finish_folio_write(inode, ap->folios[i], |
| 1904 |
ap->descs[i].length); |
| 1905 |
|
| 1906 |
wake_up(&fi->page_waitq); |
| 1907 |
} |
| 1908 |
|
| 1909 |
/* Called under fi->lock, may release and reacquire it */ |
| 1910 |
static void fuse_send_writepage(struct fuse_mount *fm, |
| 1911 |
struct fuse_writepage_args *wpa, loff_t size) |
| 1912 |
__releases(fi->lock) |
| 1913 |
__acquires(fi->lock) |
| 1914 |
{ |
| 1915 |
struct fuse_inode *fi = get_fuse_inode(wpa->inode); |
| 1916 |
struct fuse_args_pages *ap = &wpa->ia.ap; |
| 1917 |
struct fuse_write_in *inarg = &wpa->ia.write.in; |
| 1918 |
struct fuse_args *args = &ap->args; |
| 1919 |
__u64 data_size = 0; |
| 1920 |
int err, i; |
| 1921 |
|
| 1922 |
for (i = 0; i < ap->num_folios; i++) |
| 1923 |
data_size += ap->descs[i].length; |
| 1924 |
|
| 1925 |
fi->writectr++; |
| 1926 |
if (inarg->offset + data_size <= size) { |
| 1927 |
inarg->size = data_size; |
| 1928 |
} else if (inarg->offset < size) { |
| 1929 |
inarg->size = size - inarg->offset; |
| 1930 |
} else { |
| 1931 |
/* Got truncated off completely */ |
| 1932 |
goto out_free; |
| 1933 |
} |
| 1934 |
|
| 1935 |
args->in_args[1].size = inarg->size; |
| 1936 |
args->force = true; |
| 1937 |
args->nocreds = true; |
| 1938 |
|
| 1939 |
err = fuse_simple_background(fm, args, GFP_ATOMIC); |
| 1940 |
if (err == -ENOMEM) { |
| 1941 |
spin_unlock(&fi->lock); |
| 1942 |
err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL); |
| 1943 |
spin_lock(&fi->lock); |
| 1944 |
} |
| 1945 |
|
| 1946 |
/* Fails on broken connection only */ |
| 1947 |
if (unlikely(err)) |
| 1948 |
goto out_free; |
| 1949 |
|
| 1950 |
return; |
| 1951 |
|
| 1952 |
out_free: |
| 1953 |
fi->writectr--; |
| 1954 |
fuse_writepage_finish(wpa); |
| 1955 |
spin_unlock(&fi->lock); |
| 1956 |
fuse_writepage_free(wpa); |
| 1957 |
spin_lock(&fi->lock); |
| 1958 |
} |
| 1959 |
|
| 1960 |
/* |
| 1961 |
* If fi->writectr is positive (no truncate or fsync going on) send |
| 1962 |
* all queued writepage requests. |
| 1963 |
* |
| 1964 |
* Called with fi->lock |
| 1965 |
*/ |
| 1966 |
void fuse_flush_writepages(struct inode *inode) |
| 1967 |
__releases(fi->lock) |
| 1968 |
__acquires(fi->lock) |
| 1969 |
{ |
| 1970 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 1971 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1972 |
loff_t crop = i_size_read(inode); |
| 1973 |
struct fuse_writepage_args *wpa; |
| 1974 |
|
| 1975 |
while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { |
| 1976 |
wpa = list_entry(fi->queued_writes.next, |
| 1977 |
struct fuse_writepage_args, queue_entry); |
| 1978 |
list_del_init(&wpa->queue_entry); |
| 1979 |
fuse_send_writepage(fm, wpa, crop); |
| 1980 |
} |
| 1981 |
} |
| 1982 |
|
| 1983 |
static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, |
| 1984 |
int error) |
| 1985 |
{ |
| 1986 |
struct fuse_writepage_args *wpa = |
| 1987 |
container_of(args, typeof(*wpa), ia.ap.args); |
| 1988 |
struct inode *inode = wpa->inode; |
| 1989 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1990 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1991 |
|
| 1992 |
mapping_set_error(inode->i_mapping, error); |
| 1993 |
/* |
| 1994 |
* A writeback finished and this might have updated mtime/ctime on |
| 1995 |
* server making local mtime/ctime stale. Hence invalidate attrs. |
| 1996 |
* Do this only if writeback_cache is not enabled. If writeback_cache |
| 1997 |
* is enabled, we trust local ctime/mtime. |
| 1998 |
*/ |
| 1999 |
if (!fc->writeback_cache) |
| 2000 |
fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); |
| 2001 |
spin_lock(&fi->lock); |
| 2002 |
fi->writectr--; |
| 2003 |
fuse_writepage_finish(wpa); |
| 2004 |
spin_unlock(&fi->lock); |
| 2005 |
fuse_writepage_free(wpa); |
| 2006 |
} |
| 2007 |
|
| 2008 |
static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi) |
| 2009 |
{ |
| 2010 |
struct fuse_file *ff; |
| 2011 |
|
| 2012 |
spin_lock(&fi->lock); |
| 2013 |
ff = list_first_entry_or_null(&fi->write_files, struct fuse_file, |
| 2014 |
write_entry); |
| 2015 |
if (ff) |
| 2016 |
fuse_file_get(ff); |
| 2017 |
spin_unlock(&fi->lock); |
| 2018 |
|
| 2019 |
return ff; |
| 2020 |
} |
| 2021 |
|
| 2022 |
static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi) |
| 2023 |
{ |
| 2024 |
struct fuse_file *ff = __fuse_write_file_get(fi); |
| 2025 |
WARN_ON(!ff); |
| 2026 |
return ff; |
| 2027 |
} |
| 2028 |
|
| 2029 |
int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) |
| 2030 |
{ |
| 2031 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 2032 |
struct fuse_file *ff; |
| 2033 |
int err; |
| 2034 |
|
| 2035 |
ff = __fuse_write_file_get(fi); |
| 2036 |
err = fuse_flush_times(inode, ff); |
| 2037 |
if (ff) |
| 2038 |
fuse_file_put(ff, false); |
| 2039 |
|
| 2040 |
return err; |
| 2041 |
} |
| 2042 |
|
| 2043 |
static struct fuse_writepage_args *fuse_writepage_args_alloc(void) |
| 2044 |
{ |
| 2045 |
struct fuse_writepage_args *wpa; |
| 2046 |
struct fuse_args_pages *ap; |
| 2047 |
|
| 2048 |
wpa = kzalloc_obj(*wpa, GFP_NOFS); |
| 2049 |
if (wpa) { |
| 2050 |
ap = &wpa->ia.ap; |
| 2051 |
ap->num_folios = 0; |
| 2052 |
ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); |
| 2053 |
if (!ap->folios) { |
| 2054 |
kfree(wpa); |
| 2055 |
wpa = NULL; |
| 2056 |
} |
| 2057 |
} |
| 2058 |
return wpa; |
| 2059 |
|
| 2060 |
} |
| 2061 |
|
| 2062 |
static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, |
| 2063 |
struct fuse_writepage_args *wpa) |
| 2064 |
{ |
| 2065 |
if (!fc->sync_fs) |
| 2066 |
return; |
| 2067 |
|
| 2068 |
rcu_read_lock(); |
| 2069 |
/* Prevent resurrection of dead bucket in unlikely race with syncfs */ |
| 2070 |
do { |
| 2071 |
wpa->bucket = rcu_dereference(fc->curr_bucket); |
| 2072 |
} while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count))); |
| 2073 |
rcu_read_unlock(); |
| 2074 |
} |
| 2075 |
|
| 2076 |
static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, |
| 2077 |
uint32_t folio_index, loff_t offset, unsigned len) |
| 2078 |
{ |
| 2079 |
struct fuse_args_pages *ap = &wpa->ia.ap; |
| 2080 |
|
| 2081 |
ap->folios[folio_index] = folio; |
| 2082 |
ap->descs[folio_index].offset = offset; |
| 2083 |
ap->descs[folio_index].length = len; |
| 2084 |
} |
| 2085 |
|
| 2086 |
static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, |
| 2087 |
size_t offset, |
| 2088 |
struct fuse_file *ff) |
| 2089 |
{ |
| 2090 |
struct inode *inode = folio->mapping->host; |
| 2091 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 2092 |
struct fuse_writepage_args *wpa; |
| 2093 |
struct fuse_args_pages *ap; |
| 2094 |
|
| 2095 |
wpa = fuse_writepage_args_alloc(); |
| 2096 |
if (!wpa) |
| 2097 |
return NULL; |
| 2098 |
|
| 2099 |
fuse_writepage_add_to_bucket(fc, wpa); |
| 2100 |
fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio) + offset, 0); |
| 2101 |
wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE; |
| 2102 |
wpa->inode = inode; |
| 2103 |
wpa->ia.ff = ff; |
| 2104 |
|
| 2105 |
ap = &wpa->ia.ap; |
| 2106 |
ap->args.in_pages = true; |
| 2107 |
ap->args.end = fuse_writepage_end; |
| 2108 |
|
| 2109 |
return wpa; |
| 2110 |
} |
| 2111 |
|
| 2112 |
struct fuse_fill_wb_data { |
| 2113 |
struct fuse_writepage_args *wpa; |
| 2114 |
struct fuse_file *ff; |
| 2115 |
unsigned int max_folios; |
| 2116 |
/* |
| 2117 |
* nr_bytes won't overflow since fuse_folios_need_send() caps |
| 2118 |
* wb requests to never exceed fc->max_pages (which has an upper bound |
| 2119 |
* of U16_MAX). |
| 2120 |
*/ |
| 2121 |
unsigned int nr_bytes; |
| 2122 |
}; |
| 2123 |
|
| 2124 |
static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, |
| 2125 |
unsigned int max_pages) |
| 2126 |
{ |
| 2127 |
struct fuse_args_pages *ap = &data->wpa->ia.ap; |
| 2128 |
struct folio **folios; |
| 2129 |
struct fuse_folio_desc *descs; |
| 2130 |
unsigned int nfolios = min_t(unsigned int, |
| 2131 |
max_t(unsigned int, data->max_folios * 2, |
| 2132 |
FUSE_DEFAULT_MAX_PAGES_PER_REQ), |
| 2133 |
max_pages); |
| 2134 |
WARN_ON(nfolios <= data->max_folios); |
| 2135 |
|
| 2136 |
folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); |
| 2137 |
if (!folios) |
| 2138 |
return false; |
| 2139 |
|
| 2140 |
memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios); |
| 2141 |
memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios); |
| 2142 |
kfree(ap->folios); |
| 2143 |
ap->folios = folios; |
| 2144 |
ap->descs = descs; |
| 2145 |
data->max_folios = nfolios; |
| 2146 |
|
| 2147 |
return true; |
| 2148 |
} |
| 2149 |
|
| 2150 |
static void fuse_writepages_send(struct inode *inode, |
| 2151 |
struct fuse_fill_wb_data *data) |
| 2152 |
{ |
| 2153 |
struct fuse_writepage_args *wpa = data->wpa; |
| 2154 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 2155 |
|
| 2156 |
spin_lock(&fi->lock); |
| 2157 |
list_add_tail(&wpa->queue_entry, &fi->queued_writes); |
| 2158 |
fuse_flush_writepages(inode); |
| 2159 |
spin_unlock(&fi->lock); |
| 2160 |
} |
| 2161 |
|
| 2162 |
static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, |
| 2163 |
unsigned len, struct fuse_args_pages *ap, |
| 2164 |
unsigned cur_bytes, bool write) |
| 2165 |
{ |
| 2166 |
struct folio *prev_folio; |
| 2167 |
struct fuse_folio_desc prev_desc; |
| 2168 |
unsigned bytes = cur_bytes + len; |
| 2169 |
loff_t prev_pos; |
| 2170 |
size_t max_bytes = write ? fc->max_write : fc->max_read; |
| 2171 |
|
| 2172 |
WARN_ON(!ap->num_folios); |
| 2173 |
|
| 2174 |
/* Reached max pages */ |
| 2175 |
if ((bytes + PAGE_SIZE - 1) >> PAGE_SHIFT > fc->max_pages) |
| 2176 |
return true; |
| 2177 |
|
| 2178 |
if (bytes > max_bytes) |
| 2179 |
return true; |
| 2180 |
|
| 2181 |
/* Discontinuity */ |
| 2182 |
prev_folio = ap->folios[ap->num_folios - 1]; |
| 2183 |
prev_desc = ap->descs[ap->num_folios - 1]; |
| 2184 |
prev_pos = folio_pos(prev_folio) + prev_desc.offset + prev_desc.length; |
| 2185 |
if (prev_pos != pos) |
| 2186 |
return true; |
| 2187 |
|
| 2188 |
return false; |
| 2189 |
} |
| 2190 |
|
| 2191 |
static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, |
| 2192 |
struct folio *folio, u64 pos, |
| 2193 |
unsigned len, u64 end_pos) |
| 2194 |
{ |
| 2195 |
struct fuse_fill_wb_data *data = wpc->wb_ctx; |
| 2196 |
struct fuse_writepage_args *wpa = data->wpa; |
| 2197 |
struct fuse_args_pages *ap = &wpa->ia.ap; |
| 2198 |
struct inode *inode = wpc->inode; |
| 2199 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 2200 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 2201 |
loff_t offset = offset_in_folio(folio, pos); |
| 2202 |
|
| 2203 |
WARN_ON_ONCE(!data); |
| 2204 |
|
| 2205 |
if (!data->ff) { |
| 2206 |
data->ff = fuse_write_file_get(fi); |
| 2207 |
if (!data->ff) |
| 2208 |
return -EIO; |
| 2209 |
} |
| 2210 |
|
| 2211 |
if (wpa) { |
| 2212 |
bool send = fuse_folios_need_send(fc, pos, len, ap, |
| 2213 |
data->nr_bytes, true); |
| 2214 |
|
| 2215 |
if (!send) { |
| 2216 |
/* |
| 2217 |
* Need to grow the pages array? If so, did the |
| 2218 |
* expansion fail? |
| 2219 |
*/ |
| 2220 |
send = (ap->num_folios == data->max_folios) && |
| 2221 |
!fuse_pages_realloc(data, fc->max_pages); |
| 2222 |
} |
| 2223 |
|
| 2224 |
if (send) { |
| 2225 |
fuse_writepages_send(inode, data); |
| 2226 |
data->wpa = NULL; |
| 2227 |
data->nr_bytes = 0; |
| 2228 |
} |
| 2229 |
} |
| 2230 |
|
| 2231 |
if (data->wpa == NULL) { |
| 2232 |
wpa = fuse_writepage_args_setup(folio, offset, data->ff); |
| 2233 |
if (!wpa) |
| 2234 |
return -ENOMEM; |
| 2235 |
fuse_file_get(wpa->ia.ff); |
| 2236 |
data->max_folios = 1; |
| 2237 |
ap = &wpa->ia.ap; |
| 2238 |
} |
| 2239 |
|
| 2240 |
fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, |
| 2241 |
offset, len); |
| 2242 |
data->nr_bytes += len; |
| 2243 |
|
| 2244 |
ap->num_folios++; |
| 2245 |
if (!data->wpa) |
| 2246 |
data->wpa = wpa; |
| 2247 |
|
| 2248 |
return len; |
| 2249 |
} |
| 2250 |
|
| 2251 |
static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, |
| 2252 |
int error) |
| 2253 |
{ |
| 2254 |
struct fuse_fill_wb_data *data = wpc->wb_ctx; |
| 2255 |
|
| 2256 |
WARN_ON_ONCE(!data); |
| 2257 |
|
| 2258 |
if (data->wpa) { |
| 2259 |
WARN_ON(!data->wpa->ia.ap.num_folios); |
| 2260 |
fuse_writepages_send(wpc->inode, data); |
| 2261 |
} |
| 2262 |
|
| 2263 |
if (data->ff) |
| 2264 |
fuse_file_put(data->ff, false); |
| 2265 |
|
| 2266 |
return error; |
| 2267 |
} |
| 2268 |
|
| 2269 |
static const struct iomap_writeback_ops fuse_writeback_ops = { |
| 2270 |
.writeback_range = fuse_iomap_writeback_range, |
| 2271 |
.writeback_submit = fuse_iomap_writeback_submit, |
| 2272 |
}; |
| 2273 |
|
| 2274 |
static int fuse_writepages(struct address_space *mapping, |
| 2275 |
struct writeback_control *wbc) |
| 2276 |
{ |
| 2277 |
struct inode *inode = mapping->host; |
| 2278 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 2279 |
struct fuse_fill_wb_data data = {}; |
| 2280 |
struct iomap_writepage_ctx wpc = { |
| 2281 |
.inode = inode, |
| 2282 |
.iomap.type = IOMAP_MAPPED, |
| 2283 |
.wbc = wbc, |
| 2284 |
.ops = &fuse_writeback_ops, |
| 2285 |
.wb_ctx = &data, |
| 2286 |
}; |
| 2287 |
|
| 2288 |
if (fuse_is_bad(inode)) |
| 2289 |
return -EIO; |
| 2290 |
|
| 2291 |
if (wbc->sync_mode == WB_SYNC_NONE && |
| 2292 |
fc->num_background >= fc->congestion_threshold) |
| 2293 |
return 0; |
| 2294 |
|
| 2295 |
return iomap_writepages(&wpc); |
| 2296 |
} |
| 2297 |
|
| 2298 |
static int fuse_launder_folio(struct folio *folio) |
| 2299 |
{ |
| 2300 |
int err = 0; |
| 2301 |
struct fuse_fill_wb_data data = {}; |
| 2302 |
struct iomap_writepage_ctx wpc = { |
| 2303 |
.inode = folio->mapping->host, |
| 2304 |
.iomap.type = IOMAP_MAPPED, |
| 2305 |
.ops = &fuse_writeback_ops, |
| 2306 |
.wb_ctx = &data, |
| 2307 |
}; |
| 2308 |
|
| 2309 |
if (folio_clear_dirty_for_io(folio)) { |
| 2310 |
err = iomap_writeback_folio(&wpc, folio); |
| 2311 |
err = fuse_iomap_writeback_submit(&wpc, err); |
| 2312 |
if (!err) |
| 2313 |
folio_wait_writeback(folio); |
| 2314 |
} |
| 2315 |
return err; |
| 2316 |
} |
| 2317 |
|
| 2318 |
/* |
| 2319 |
* Write back dirty data/metadata now (there may not be any suitable |
| 2320 |
* open files later for data) |
| 2321 |
*/ |
| 2322 |
static void fuse_vma_close(struct vm_area_struct *vma) |
| 2323 |
{ |
| 2324 |
int err; |
| 2325 |
|
| 2326 |
err = write_inode_now(vma->vm_file->f_mapping->host, 1); |
| 2327 |
mapping_set_error(vma->vm_file->f_mapping, err); |
| 2328 |
} |
| 2329 |
|
| 2330 |
/* |
| 2331 |
* Wait for writeback against this page to complete before allowing it |
| 2332 |
* to be marked dirty again, and hence written back again, possibly |
| 2333 |
* before the previous writepage completed. |
| 2334 |
* |
| 2335 |
* Block here, instead of in ->writepage(), so that the userspace fs |
| 2336 |
* can only block processes actually operating on the filesystem. |
| 2337 |
* |
| 2338 |
* Otherwise unprivileged userspace fs would be able to block |
| 2339 |
* unrelated: |
| 2340 |
* |
| 2341 |
* - page migration |
| 2342 |
* - sync(2) |
| 2343 |
* - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER |
| 2344 |
*/ |
| 2345 |
static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) |
| 2346 |
{ |
| 2347 |
struct folio *folio = page_folio(vmf->page); |
| 2348 |
struct inode *inode = file_inode(vmf->vma->vm_file); |
| 2349 |
|
| 2350 |
file_update_time(vmf->vma->vm_file); |
| 2351 |
folio_lock(folio); |
| 2352 |
if (folio->mapping != inode->i_mapping) { |
| 2353 |
folio_unlock(folio); |
| 2354 |
return VM_FAULT_NOPAGE; |
| 2355 |
} |
| 2356 |
|
| 2357 |
folio_wait_writeback(folio); |
| 2358 |
return VM_FAULT_LOCKED; |
| 2359 |
} |
| 2360 |
|
| 2361 |
static const struct vm_operations_struct fuse_file_vm_ops = { |
| 2362 |
.close = fuse_vma_close, |
| 2363 |
.fault = filemap_fault, |
| 2364 |
.map_pages = filemap_map_pages, |
| 2365 |
.page_mkwrite = fuse_page_mkwrite, |
| 2366 |
}; |
| 2367 |
|
| 2368 |
static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) |
| 2369 |
{ |
| 2370 |
struct fuse_file *ff = file->private_data; |
| 2371 |
struct fuse_conn *fc = ff->fm->fc; |
| 2372 |
struct inode *inode = file_inode(file); |
| 2373 |
int rc; |
| 2374 |
|
| 2375 |
/* DAX mmap is superior to direct_io mmap */ |
| 2376 |
if (FUSE_IS_DAX(inode)) |
| 2377 |
return fuse_dax_mmap(file, vma); |
| 2378 |
|
| 2379 |
/* |
| 2380 |
* If inode is in passthrough io mode, because it has some file open |
| 2381 |
* in passthrough mode, either mmap to backing file or fail mmap, |
| 2382 |
* because mixing cached mmap and passthrough io mode is not allowed. |
| 2383 |
*/ |
| 2384 |
if (fuse_file_passthrough(ff)) |
| 2385 |
return fuse_passthrough_mmap(file, vma); |
| 2386 |
else if (fuse_inode_backing(get_fuse_inode(inode))) |
| 2387 |
return -ENODEV; |
| 2388 |
|
| 2389 |
/* |
| 2390 |
* FOPEN_DIRECT_IO handling is special compared to O_DIRECT, |
| 2391 |
* as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP. |
| 2392 |
*/ |
| 2393 |
if (ff->open_flags & FOPEN_DIRECT_IO) { |
| 2394 |
/* |
| 2395 |
* Can't provide the coherency needed for MAP_SHARED |
| 2396 |
* if FUSE_DIRECT_IO_ALLOW_MMAP isn't set. |
| 2397 |
*/ |
| 2398 |
if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_allow_mmap) |
| 2399 |
return -ENODEV; |
| 2400 |
|
| 2401 |
invalidate_inode_pages2(file->f_mapping); |
| 2402 |
|
| 2403 |
if (!(vma->vm_flags & VM_MAYSHARE)) { |
| 2404 |
/* MAP_PRIVATE */ |
| 2405 |
return generic_file_mmap(file, vma); |
| 2406 |
} |
| 2407 |
|
| 2408 |
/* |
| 2409 |
* First mmap of direct_io file enters caching inode io mode. |
| 2410 |
* Also waits for parallel dio writers to go into serial mode |
| 2411 |
* (exclusive instead of shared lock). |
| 2412 |
* After first mmap, the inode stays in caching io mode until |
| 2413 |
* the direct_io file release. |
| 2414 |
*/ |
| 2415 |
rc = fuse_file_cached_io_open(inode, ff); |
| 2416 |
if (rc) |
| 2417 |
return rc; |
| 2418 |
} |
| 2419 |
|
| 2420 |
if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) |
| 2421 |
fuse_link_write_file(file); |
| 2422 |
|
| 2423 |
file_accessed(file); |
| 2424 |
vma->vm_ops = &fuse_file_vm_ops; |
| 2425 |
return 0; |
| 2426 |
} |
| 2427 |
|
| 2428 |
static int convert_fuse_file_lock(struct fuse_conn *fc, |
| 2429 |
const struct fuse_file_lock *ffl, |
| 2430 |
struct file_lock *fl) |
| 2431 |
{ |
| 2432 |
switch (ffl->type) { |
| 2433 |
case F_UNLCK: |
| 2434 |
break; |
| 2435 |
|
| 2436 |
case F_RDLCK: |
| 2437 |
case F_WRLCK: |
| 2438 |
if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX || |
| 2439 |
ffl->end < ffl->start) |
| 2440 |
return -EIO; |
| 2441 |
|
| 2442 |
fl->fl_start = ffl->start; |
| 2443 |
fl->fl_end = ffl->end; |
| 2444 |
|
| 2445 |
/* |
| 2446 |
* Convert pid into init's pid namespace. The locks API will |
| 2447 |
* translate it into the caller's pid namespace. |
| 2448 |
*/ |
| 2449 |
rcu_read_lock(); |
| 2450 |
fl->c.flc_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns); |
| 2451 |
rcu_read_unlock(); |
| 2452 |
break; |
| 2453 |
|
| 2454 |
default: |
| 2455 |
return -EIO; |
| 2456 |
} |
| 2457 |
fl->c.flc_type = ffl->type; |
| 2458 |
return 0; |
| 2459 |
} |
| 2460 |
|
| 2461 |
static void fuse_lk_fill(struct fuse_args *args, struct file *file, |
| 2462 |
const struct file_lock *fl, int opcode, pid_t pid, |
| 2463 |
int flock, struct fuse_lk_in *inarg) |
| 2464 |
{ |
| 2465 |
struct inode *inode = file_inode(file); |
| 2466 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 2467 |
struct fuse_file *ff = file->private_data; |
| 2468 |
|
| 2469 |
memset(inarg, 0, sizeof(*inarg)); |
| 2470 |
inarg->fh = ff->fh; |
| 2471 |
inarg->owner = fuse_lock_owner_id(fc, fl->c.flc_owner); |
| 2472 |
inarg->lk.start = fl->fl_start; |
| 2473 |
inarg->lk.end = fl->fl_end; |
| 2474 |
inarg->lk.type = fl->c.flc_type; |
| 2475 |
inarg->lk.pid = pid; |
| 2476 |
if (flock) |
| 2477 |
inarg->lk_flags |= FUSE_LK_FLOCK; |
| 2478 |
args->opcode = opcode; |
| 2479 |
args->nodeid = get_node_id(inode); |
| 2480 |
args->in_numargs = 1; |
| 2481 |
args->in_args[0].size = sizeof(*inarg); |
| 2482 |
args->in_args[0].value = inarg; |
| 2483 |
} |
| 2484 |
|
| 2485 |
static int fuse_getlk(struct file *file, struct file_lock *fl) |
| 2486 |
{ |
| 2487 |
struct inode *inode = file_inode(file); |
| 2488 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 2489 |
FUSE_ARGS(args); |
| 2490 |
struct fuse_lk_in inarg; |
| 2491 |
struct fuse_lk_out outarg; |
| 2492 |
int err; |
| 2493 |
|
| 2494 |
fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg); |
| 2495 |
args.out_numargs = 1; |
| 2496 |
args.out_args[0].size = sizeof(outarg); |
| 2497 |
args.out_args[0].value = &outarg; |
| 2498 |
err = fuse_simple_request(fm, &args); |
| 2499 |
if (!err) |
| 2500 |
err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl); |
| 2501 |
|
| 2502 |
return err; |
| 2503 |
} |
| 2504 |
|
| 2505 |
static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) |
| 2506 |
{ |
| 2507 |
struct inode *inode = file_inode(file); |
| 2508 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 2509 |
FUSE_ARGS(args); |
| 2510 |
struct fuse_lk_in inarg; |
| 2511 |
int opcode = (fl->c.flc_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK; |
| 2512 |
struct pid *pid = fl->c.flc_type != F_UNLCK ? task_tgid(current) : NULL; |
| 2513 |
pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns); |
| 2514 |
int err; |
| 2515 |
|
| 2516 |
if (fl->fl_lmops && fl->fl_lmops->lm_grant) { |
| 2517 |
/* NLM needs asynchronous locks, which we don't support yet */ |
| 2518 |
return -ENOLCK; |
| 2519 |
} |
| 2520 |
|
| 2521 |
fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg); |
| 2522 |
err = fuse_simple_request(fm, &args); |
| 2523 |
|
| 2524 |
/* locking is restartable */ |
| 2525 |
if (err == -EINTR) |
| 2526 |
err = -ERESTARTSYS; |
| 2527 |
|
| 2528 |
return err; |
| 2529 |
} |
| 2530 |
|
| 2531 |
static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl) |
| 2532 |
{ |
| 2533 |
struct inode *inode = file_inode(file); |
| 2534 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 2535 |
int err; |
| 2536 |
|
| 2537 |
if (cmd == F_CANCELLK) { |
| 2538 |
err = 0; |
| 2539 |
} else if (cmd == F_GETLK) { |
| 2540 |
if (fc->no_lock) { |
| 2541 |
posix_test_lock(file, fl); |
| 2542 |
err = 0; |
| 2543 |
} else |
| 2544 |
err = fuse_getlk(file, fl); |
| 2545 |
} else { |
| 2546 |
if (fc->no_lock) |
| 2547 |
err = posix_lock_file(file, fl, NULL); |
| 2548 |
else |
| 2549 |
err = fuse_setlk(file, fl, 0); |
| 2550 |
} |
| 2551 |
return err; |
| 2552 |
} |
| 2553 |
|
| 2554 |
static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) |
| 2555 |
{ |
| 2556 |
struct inode *inode = file_inode(file); |
| 2557 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 2558 |
int err; |
| 2559 |
|
| 2560 |
if (fc->no_flock) { |
| 2561 |
err = locks_lock_file_wait(file, fl); |
| 2562 |
} else { |
| 2563 |
struct fuse_file *ff = file->private_data; |
| 2564 |
|
| 2565 |
/* emulate flock with POSIX locks */ |
| 2566 |
ff->flock = true; |
| 2567 |
err = fuse_setlk(file, fl, 1); |
| 2568 |
} |
| 2569 |
|
| 2570 |
return err; |
| 2571 |
} |
| 2572 |
|
| 2573 |
static sector_t fuse_bmap(struct address_space *mapping, sector_t block) |
| 2574 |
{ |
| 2575 |
struct inode *inode = mapping->host; |
| 2576 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 2577 |
FUSE_ARGS(args); |
| 2578 |
struct fuse_bmap_in inarg; |
| 2579 |
struct fuse_bmap_out outarg; |
| 2580 |
int err; |
| 2581 |
|
| 2582 |
if (!inode->i_sb->s_bdev || fm->fc->no_bmap) |
| 2583 |
return 0; |
| 2584 |
|
| 2585 |
memset(&inarg, 0, sizeof(inarg)); |
| 2586 |
inarg.block = block; |
| 2587 |
inarg.blocksize = inode->i_sb->s_blocksize; |
| 2588 |
args.opcode = FUSE_BMAP; |
| 2589 |
args.nodeid = get_node_id(inode); |
| 2590 |
args.in_numargs = 1; |
| 2591 |
args.in_args[0].size = sizeof(inarg); |
| 2592 |
args.in_args[0].value = &inarg; |
| 2593 |
args.out_numargs = 1; |
| 2594 |
args.out_args[0].size = sizeof(outarg); |
| 2595 |
args.out_args[0].value = &outarg; |
| 2596 |
err = fuse_simple_request(fm, &args); |
| 2597 |
if (err == -ENOSYS) |
| 2598 |
fm->fc->no_bmap = 1; |
| 2599 |
|
| 2600 |
return err ? 0 : outarg.block; |
| 2601 |
} |
| 2602 |
|
| 2603 |
static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) |
| 2604 |
{ |
| 2605 |
struct inode *inode = file->f_mapping->host; |
| 2606 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 2607 |
struct fuse_file *ff = file->private_data; |
| 2608 |
FUSE_ARGS(args); |
| 2609 |
struct fuse_lseek_in inarg = { |
| 2610 |
.fh = ff->fh, |
| 2611 |
.offset = offset, |
| 2612 |
.whence = whence |
| 2613 |
}; |
| 2614 |
struct fuse_lseek_out outarg; |
| 2615 |
int err; |
| 2616 |
|
| 2617 |
if (fm->fc->no_lseek) |
| 2618 |
goto fallback; |
| 2619 |
|
| 2620 |
args.opcode = FUSE_LSEEK; |
| 2621 |
args.nodeid = ff->nodeid; |
| 2622 |
args.in_numargs = 1; |
| 2623 |
args.in_args[0].size = sizeof(inarg); |
| 2624 |
args.in_args[0].value = &inarg; |
| 2625 |
args.out_numargs = 1; |
| 2626 |
args.out_args[0].size = sizeof(outarg); |
| 2627 |
args.out_args[0].value = &outarg; |
| 2628 |
err = fuse_simple_request(fm, &args); |
| 2629 |
if (err) { |
| 2630 |
if (err == -ENOSYS) { |
| 2631 |
fm->fc->no_lseek = 1; |
| 2632 |
goto fallback; |
| 2633 |
} |
| 2634 |
return err; |
| 2635 |
} |
| 2636 |
|
| 2637 |
return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes); |
| 2638 |
|
| 2639 |
fallback: |
| 2640 |
err = fuse_update_attributes(inode, file, STATX_SIZE); |
| 2641 |
if (!err) |
| 2642 |
return generic_file_llseek(file, offset, whence); |
| 2643 |
else |
| 2644 |
return err; |
| 2645 |
} |
| 2646 |
|
| 2647 |
static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) |
| 2648 |
{ |
| 2649 |
loff_t retval; |
| 2650 |
struct inode *inode = file_inode(file); |
| 2651 |
|
| 2652 |
switch (whence) { |
| 2653 |
case SEEK_SET: |
| 2654 |
case SEEK_CUR: |
| 2655 |
/* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */ |
| 2656 |
retval = generic_file_llseek(file, offset, whence); |
| 2657 |
break; |
| 2658 |
case SEEK_END: |
| 2659 |
inode_lock(inode); |
| 2660 |
retval = fuse_update_attributes(inode, file, STATX_SIZE); |
| 2661 |
if (!retval) |
| 2662 |
retval = generic_file_llseek(file, offset, whence); |
| 2663 |
inode_unlock(inode); |
| 2664 |
break; |
| 2665 |
case SEEK_HOLE: |
| 2666 |
case SEEK_DATA: |
| 2667 |
inode_lock(inode); |
| 2668 |
retval = fuse_lseek(file, offset, whence); |
| 2669 |
inode_unlock(inode); |
| 2670 |
break; |
| 2671 |
default: |
| 2672 |
retval = -EINVAL; |
| 2673 |
} |
| 2674 |
|
| 2675 |
return retval; |
| 2676 |
} |
| 2677 |
|
| 2678 |
/* |
| 2679 |
* All files which have been polled are linked to RB tree |
| 2680 |
* fuse_conn->polled_files which is indexed by kh. Walk the tree and |
| 2681 |
* find the matching one. |
| 2682 |
*/ |
| 2683 |
static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, |
| 2684 |
struct rb_node **parent_out) |
| 2685 |
{ |
| 2686 |
struct rb_node **link = &fc->polled_files.rb_node; |
| 2687 |
struct rb_node *last = NULL; |
| 2688 |
|
| 2689 |
while (*link) { |
| 2690 |
struct fuse_file *ff; |
| 2691 |
|
| 2692 |
last = *link; |
| 2693 |
ff = rb_entry(last, struct fuse_file, polled_node); |
| 2694 |
|
| 2695 |
if (kh < ff->kh) |
| 2696 |
link = &last->rb_left; |
| 2697 |
else if (kh > ff->kh) |
| 2698 |
link = &last->rb_right; |
| 2699 |
else |
| 2700 |
return link; |
| 2701 |
} |
| 2702 |
|
| 2703 |
if (parent_out) |
| 2704 |
*parent_out = last; |
| 2705 |
return link; |
| 2706 |
} |
| 2707 |
|
| 2708 |
/* |
| 2709 |
* The file is about to be polled. Make sure it's on the polled_files |
| 2710 |
* RB tree. Note that files once added to the polled_files tree are |
| 2711 |
* not removed before the file is released. This is because a file |
| 2712 |
* polled once is likely to be polled again. |
| 2713 |
*/ |
| 2714 |
static void fuse_register_polled_file(struct fuse_conn *fc, |
| 2715 |
struct fuse_file *ff) |
| 2716 |
{ |
| 2717 |
spin_lock(&fc->lock); |
| 2718 |
if (RB_EMPTY_NODE(&ff->polled_node)) { |
| 2719 |
struct rb_node **link, *parent; |
| 2720 |
|
| 2721 |
link = fuse_find_polled_node(fc, ff->kh, &parent); |
| 2722 |
BUG_ON(*link); |
| 2723 |
rb_link_node(&ff->polled_node, parent, link); |
| 2724 |
rb_insert_color(&ff->polled_node, &fc->polled_files); |
| 2725 |
} |
| 2726 |
spin_unlock(&fc->lock); |
| 2727 |
} |
| 2728 |
|
| 2729 |
__poll_t fuse_file_poll(struct file *file, poll_table *wait) |
| 2730 |
{ |
| 2731 |
struct fuse_file *ff = file->private_data; |
| 2732 |
struct fuse_mount *fm = ff->fm; |
| 2733 |
struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; |
| 2734 |
struct fuse_poll_out outarg; |
| 2735 |
FUSE_ARGS(args); |
| 2736 |
int err; |
| 2737 |
|
| 2738 |
if (fm->fc->no_poll) |
| 2739 |
return DEFAULT_POLLMASK; |
| 2740 |
|
| 2741 |
poll_wait(file, &ff->poll_wait, wait); |
| 2742 |
inarg.events = mangle_poll(poll_requested_events(wait)); |
| 2743 |
|
| 2744 |
/* |
| 2745 |
* Ask for notification iff there's someone waiting for it. |
| 2746 |
* The client may ignore the flag and always notify. |
| 2747 |
*/ |
| 2748 |
if (waitqueue_active(&ff->poll_wait)) { |
| 2749 |
inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; |
| 2750 |
fuse_register_polled_file(fm->fc, ff); |
| 2751 |
} |
| 2752 |
|
| 2753 |
args.opcode = FUSE_POLL; |
| 2754 |
args.nodeid = ff->nodeid; |
| 2755 |
args.in_numargs = 1; |
| 2756 |
args.in_args[0].size = sizeof(inarg); |
| 2757 |
args.in_args[0].value = &inarg; |
| 2758 |
args.out_numargs = 1; |
| 2759 |
args.out_args[0].size = sizeof(outarg); |
| 2760 |
args.out_args[0].value = &outarg; |
| 2761 |
err = fuse_simple_request(fm, &args); |
| 2762 |
|
| 2763 |
if (!err) |
| 2764 |
return demangle_poll(outarg.revents); |
| 2765 |
if (err == -ENOSYS) { |
| 2766 |
fm->fc->no_poll = 1; |
| 2767 |
return DEFAULT_POLLMASK; |
| 2768 |
} |
| 2769 |
return EPOLLERR; |
| 2770 |
} |
| 2771 |
EXPORT_SYMBOL_GPL(fuse_file_poll); |
| 2772 |
|
| 2773 |
/* |
| 2774 |
* This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and |
| 2775 |
* wakes up the poll waiters. |
| 2776 |
*/ |
| 2777 |
int fuse_notify_poll_wakeup(struct fuse_conn *fc, |
| 2778 |
struct fuse_notify_poll_wakeup_out *outarg) |
| 2779 |
{ |
| 2780 |
u64 kh = outarg->kh; |
| 2781 |
struct rb_node **link; |
| 2782 |
|
| 2783 |
spin_lock(&fc->lock); |
| 2784 |
|
| 2785 |
link = fuse_find_polled_node(fc, kh, NULL); |
| 2786 |
if (*link) { |
| 2787 |
struct fuse_file *ff; |
| 2788 |
|
| 2789 |
ff = rb_entry(*link, struct fuse_file, polled_node); |
| 2790 |
wake_up_interruptible_sync(&ff->poll_wait); |
| 2791 |
} |
| 2792 |
|
| 2793 |
spin_unlock(&fc->lock); |
| 2794 |
return 0; |
| 2795 |
} |
| 2796 |
|
| 2797 |
static void fuse_do_truncate(struct file *file) |
| 2798 |
{ |
| 2799 |
struct inode *inode = file->f_mapping->host; |
| 2800 |
struct iattr attr; |
| 2801 |
|
| 2802 |
attr.ia_valid = ATTR_SIZE; |
| 2803 |
attr.ia_size = i_size_read(inode); |
| 2804 |
|
| 2805 |
attr.ia_file = file; |
| 2806 |
attr.ia_valid |= ATTR_FILE; |
| 2807 |
|
| 2808 |
fuse_do_setattr(file_mnt_idmap(file), file_dentry(file), &attr, file); |
| 2809 |
} |
| 2810 |
|
| 2811 |
static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) |
| 2812 |
{ |
| 2813 |
return round_up(off, fc->max_pages << PAGE_SHIFT); |
| 2814 |
} |
| 2815 |
|
| 2816 |
static ssize_t |
| 2817 |
fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) |
| 2818 |
{ |
| 2819 |
DECLARE_COMPLETION_ONSTACK(wait); |
| 2820 |
ssize_t ret = 0; |
| 2821 |
struct file *file = iocb->ki_filp; |
| 2822 |
struct fuse_file *ff = file->private_data; |
| 2823 |
loff_t pos = 0; |
| 2824 |
struct inode *inode; |
| 2825 |
loff_t i_size; |
| 2826 |
size_t count = iov_iter_count(iter), shortened = 0; |
| 2827 |
loff_t offset = iocb->ki_pos; |
| 2828 |
struct fuse_io_priv *io; |
| 2829 |
|
| 2830 |
pos = offset; |
| 2831 |
inode = file->f_mapping->host; |
| 2832 |
i_size = i_size_read(inode); |
| 2833 |
|
| 2834 |
if ((iov_iter_rw(iter) == READ) && (offset >= i_size)) |
| 2835 |
return 0; |
| 2836 |
|
| 2837 |
io = kmalloc_obj(struct fuse_io_priv); |
| 2838 |
if (!io) |
| 2839 |
return -ENOMEM; |
| 2840 |
spin_lock_init(&io->lock); |
| 2841 |
kref_init(&io->refcnt); |
| 2842 |
io->reqs = 1; |
| 2843 |
io->bytes = -1; |
| 2844 |
io->size = 0; |
| 2845 |
io->offset = offset; |
| 2846 |
io->write = (iov_iter_rw(iter) == WRITE); |
| 2847 |
io->err = 0; |
| 2848 |
/* |
| 2849 |
* By default, we want to optimize all I/Os with async request |
| 2850 |
* submission to the client filesystem if supported. |
| 2851 |
*/ |
| 2852 |
io->async = ff->fm->fc->async_dio; |
| 2853 |
io->iocb = iocb; |
| 2854 |
io->blocking = is_sync_kiocb(iocb); |
| 2855 |
|
| 2856 |
/* optimization for short read */ |
| 2857 |
if (io->async && !io->write && offset + count > i_size) { |
| 2858 |
iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset)); |
| 2859 |
shortened = count - iov_iter_count(iter); |
| 2860 |
count -= shortened; |
| 2861 |
} |
| 2862 |
|
| 2863 |
/* |
| 2864 |
* We cannot asynchronously extend the size of a file. |
| 2865 |
* In such case the aio will behave exactly like sync io. |
| 2866 |
*/ |
| 2867 |
if ((offset + count > i_size) && io->write) |
| 2868 |
io->blocking = true; |
| 2869 |
|
| 2870 |
if (io->async && io->blocking) { |
| 2871 |
/* |
| 2872 |
* Additional reference to keep io around after |
| 2873 |
* calling fuse_aio_complete() |
| 2874 |
*/ |
| 2875 |
kref_get(&io->refcnt); |
| 2876 |
io->done = &wait; |
| 2877 |
} |
| 2878 |
|
| 2879 |
if (iov_iter_rw(iter) == WRITE) { |
| 2880 |
ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); |
| 2881 |
fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); |
| 2882 |
} else { |
| 2883 |
ret = __fuse_direct_read(io, iter, &pos); |
| 2884 |
} |
| 2885 |
iov_iter_reexpand(iter, iov_iter_count(iter) + shortened); |
| 2886 |
|
| 2887 |
if (io->async) { |
| 2888 |
bool blocking = io->blocking; |
| 2889 |
|
| 2890 |
fuse_aio_complete(io, ret < 0 ? ret : 0, -1); |
| 2891 |
|
| 2892 |
/* we have a non-extending, async request, so return */ |
| 2893 |
if (!blocking) |
| 2894 |
return -EIOCBQUEUED; |
| 2895 |
|
| 2896 |
wait_for_completion(&wait); |
| 2897 |
ret = fuse_get_res_by_io(io); |
| 2898 |
} |
| 2899 |
|
| 2900 |
kref_put(&io->refcnt, fuse_io_release); |
| 2901 |
|
| 2902 |
if (iov_iter_rw(iter) == WRITE) { |
| 2903 |
fuse_write_update_attr(inode, pos, ret); |
| 2904 |
/* For extending writes we already hold exclusive lock */ |
| 2905 |
if (ret < 0 && offset + count > i_size) |
| 2906 |
fuse_do_truncate(file); |
| 2907 |
} |
| 2908 |
|
| 2909 |
return ret; |
| 2910 |
} |
| 2911 |
|
| 2912 |
static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end) |
| 2913 |
{ |
| 2914 |
int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX); |
| 2915 |
|
| 2916 |
if (!err) |
| 2917 |
fuse_sync_writes(inode); |
| 2918 |
|
| 2919 |
return err; |
| 2920 |
} |
| 2921 |
|
| 2922 |
static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, |
| 2923 |
loff_t length) |
| 2924 |
{ |
| 2925 |
struct fuse_file *ff = file->private_data; |
| 2926 |
struct inode *inode = file_inode(file); |
| 2927 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 2928 |
struct fuse_mount *fm = ff->fm; |
| 2929 |
FUSE_ARGS(args); |
| 2930 |
struct fuse_fallocate_in inarg = { |
| 2931 |
.fh = ff->fh, |
| 2932 |
.offset = offset, |
| 2933 |
.length = length, |
| 2934 |
.mode = mode |
| 2935 |
}; |
| 2936 |
int err; |
| 2937 |
bool block_faults = FUSE_IS_DAX(inode) && |
| 2938 |
(!(mode & FALLOC_FL_KEEP_SIZE) || |
| 2939 |
(mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))); |
| 2940 |
|
| 2941 |
if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | |
| 2942 |
FALLOC_FL_ZERO_RANGE)) |
| 2943 |
return -EOPNOTSUPP; |
| 2944 |
|
| 2945 |
if (fm->fc->no_fallocate) |
| 2946 |
return -EOPNOTSUPP; |
| 2947 |
|
| 2948 |
inode_lock(inode); |
| 2949 |
if (block_faults) { |
| 2950 |
filemap_invalidate_lock(inode->i_mapping); |
| 2951 |
err = fuse_dax_break_layouts(inode, 0, -1); |
| 2952 |
if (err) |
| 2953 |
goto out; |
| 2954 |
} |
| 2955 |
|
| 2956 |
if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) { |
| 2957 |
loff_t endbyte = offset + length - 1; |
| 2958 |
|
| 2959 |
err = fuse_writeback_range(inode, offset, endbyte); |
| 2960 |
if (err) |
| 2961 |
goto out; |
| 2962 |
} |
| 2963 |
|
| 2964 |
if (!(mode & FALLOC_FL_KEEP_SIZE) && |
| 2965 |
offset + length > i_size_read(inode)) { |
| 2966 |
err = inode_newsize_ok(inode, offset + length); |
| 2967 |
if (err) |
| 2968 |
goto out; |
| 2969 |
} |
| 2970 |
|
| 2971 |
err = file_modified(file); |
| 2972 |
if (err) |
| 2973 |
goto out; |
| 2974 |
|
| 2975 |
if (!(mode & FALLOC_FL_KEEP_SIZE)) |
| 2976 |
set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); |
| 2977 |
|
| 2978 |
args.opcode = FUSE_FALLOCATE; |
| 2979 |
args.nodeid = ff->nodeid; |
| 2980 |
args.in_numargs = 1; |
| 2981 |
args.in_args[0].size = sizeof(inarg); |
| 2982 |
args.in_args[0].value = &inarg; |
| 2983 |
err = fuse_simple_request(fm, &args); |
| 2984 |
if (err == -ENOSYS) { |
| 2985 |
fm->fc->no_fallocate = 1; |
| 2986 |
err = -EOPNOTSUPP; |
| 2987 |
} |
| 2988 |
if (err) |
| 2989 |
goto out; |
| 2990 |
|
| 2991 |
/* we could have extended the file */ |
| 2992 |
if (!(mode & FALLOC_FL_KEEP_SIZE)) { |
| 2993 |
if (fuse_write_update_attr(inode, offset + length, length)) |
| 2994 |
file_update_time(file); |
| 2995 |
} |
| 2996 |
|
| 2997 |
if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) |
| 2998 |
truncate_pagecache_range(inode, offset, offset + length - 1); |
| 2999 |
|
| 3000 |
fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); |
| 3001 |
|
| 3002 |
out: |
| 3003 |
if (!(mode & FALLOC_FL_KEEP_SIZE)) |
| 3004 |
clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); |
| 3005 |
|
| 3006 |
if (block_faults) |
| 3007 |
filemap_invalidate_unlock(inode->i_mapping); |
| 3008 |
|
| 3009 |
inode_unlock(inode); |
| 3010 |
|
| 3011 |
fuse_flush_time_update(inode); |
| 3012 |
|
| 3013 |
return err; |
| 3014 |
} |
| 3015 |
|
| 3016 |
static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, |
| 3017 |
struct file *file_out, loff_t pos_out, |
| 3018 |
size_t len, unsigned int flags) |
| 3019 |
{ |
| 3020 |
struct fuse_file *ff_in = file_in->private_data; |
| 3021 |
struct fuse_file *ff_out = file_out->private_data; |
| 3022 |
struct inode *inode_in = file_inode(file_in); |
| 3023 |
struct inode *inode_out = file_inode(file_out); |
| 3024 |
struct fuse_inode *fi_out = get_fuse_inode(inode_out); |
| 3025 |
struct fuse_mount *fm = ff_in->fm; |
| 3026 |
struct fuse_conn *fc = fm->fc; |
| 3027 |
FUSE_ARGS(args); |
| 3028 |
struct fuse_copy_file_range_in inarg = { |
| 3029 |
.fh_in = ff_in->fh, |
| 3030 |
.off_in = pos_in, |
| 3031 |
.nodeid_out = ff_out->nodeid, |
| 3032 |
.fh_out = ff_out->fh, |
| 3033 |
.off_out = pos_out, |
| 3034 |
.len = len, |
| 3035 |
.flags = flags |
| 3036 |
}; |
| 3037 |
struct fuse_write_out outarg; |
| 3038 |
struct fuse_copy_file_range_out outarg_64; |
| 3039 |
u64 bytes_copied; |
| 3040 |
ssize_t err; |
| 3041 |
/* mark unstable when write-back is not used, and file_out gets |
| 3042 |
* extended */ |
| 3043 |
bool is_unstable = (!fc->writeback_cache) && |
| 3044 |
((pos_out + len) > inode_out->i_size); |
| 3045 |
|
| 3046 |
if (fc->no_copy_file_range) |
| 3047 |
return -EOPNOTSUPP; |
| 3048 |
|
| 3049 |
if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) |
| 3050 |
return -EXDEV; |
| 3051 |
|
| 3052 |
inode_lock(inode_in); |
| 3053 |
err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1); |
| 3054 |
inode_unlock(inode_in); |
| 3055 |
if (err) |
| 3056 |
return err; |
| 3057 |
|
| 3058 |
inode_lock(inode_out); |
| 3059 |
|
| 3060 |
err = file_modified(file_out); |
| 3061 |
if (err) |
| 3062 |
goto out; |
| 3063 |
|
| 3064 |
/* |
| 3065 |
* Write out dirty pages in the destination file before sending the COPY |
| 3066 |
* request to userspace. After the request is completed, truncate off |
| 3067 |
* pages (including partial ones) from the cache that have been copied, |
| 3068 |
* since these contain stale data at that point. |
| 3069 |
* |
| 3070 |
* This should be mostly correct, but if the COPY writes to partial |
| 3071 |
* pages (at the start or end) and the parts not covered by the COPY are |
| 3072 |
* written through a memory map after calling fuse_writeback_range(), |
| 3073 |
* then these partial page modifications will be lost on truncation. |
| 3074 |
* |
| 3075 |
* It is unlikely that someone would rely on such mixed style |
| 3076 |
* modifications. Yet this does give less guarantees than if the |
| 3077 |
* copying was performed with write(2). |
| 3078 |
* |
| 3079 |
* To fix this a mapping->invalidate_lock could be used to prevent new |
| 3080 |
* faults while the copy is ongoing. |
| 3081 |
*/ |
| 3082 |
err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1); |
| 3083 |
if (err) |
| 3084 |
goto out; |
| 3085 |
|
| 3086 |
if (is_unstable) |
| 3087 |
set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); |
| 3088 |
|
| 3089 |
args.opcode = FUSE_COPY_FILE_RANGE_64; |
| 3090 |
args.nodeid = ff_in->nodeid; |
| 3091 |
args.in_numargs = 1; |
| 3092 |
args.in_args[0].size = sizeof(inarg); |
| 3093 |
args.in_args[0].value = &inarg; |
| 3094 |
args.out_numargs = 1; |
| 3095 |
args.out_args[0].size = sizeof(outarg_64); |
| 3096 |
args.out_args[0].value = &outarg_64; |
| 3097 |
if (fc->no_copy_file_range_64) { |
| 3098 |
fallback: |
| 3099 |
/* Fall back to old op that can't handle large copy length */ |
| 3100 |
args.opcode = FUSE_COPY_FILE_RANGE; |
| 3101 |
args.out_args[0].size = sizeof(outarg); |
| 3102 |
args.out_args[0].value = &outarg; |
| 3103 |
inarg.len = len = min_t(size_t, len, UINT_MAX & PAGE_MASK); |
| 3104 |
} |
| 3105 |
err = fuse_simple_request(fm, &args); |
| 3106 |
if (err == -ENOSYS) { |
| 3107 |
if (fc->no_copy_file_range_64) { |
| 3108 |
fc->no_copy_file_range = 1; |
| 3109 |
err = -EOPNOTSUPP; |
| 3110 |
} else { |
| 3111 |
fc->no_copy_file_range_64 = 1; |
| 3112 |
goto fallback; |
| 3113 |
} |
| 3114 |
} |
| 3115 |
if (err) |
| 3116 |
goto out; |
| 3117 |
|
| 3118 |
bytes_copied = fc->no_copy_file_range_64 ? |
| 3119 |
outarg.size : outarg_64.bytes_copied; |
| 3120 |
|
| 3121 |
if (bytes_copied > len) { |
| 3122 |
err = -EIO; |
| 3123 |
goto out; |
| 3124 |
} |
| 3125 |
|
| 3126 |
truncate_inode_pages_range(inode_out->i_mapping, |
| 3127 |
ALIGN_DOWN(pos_out, PAGE_SIZE), |
| 3128 |
ALIGN(pos_out + bytes_copied, PAGE_SIZE) - 1); |
| 3129 |
|
| 3130 |
file_update_time(file_out); |
| 3131 |
fuse_write_update_attr(inode_out, pos_out + bytes_copied, bytes_copied); |
| 3132 |
|
| 3133 |
err = bytes_copied; |
| 3134 |
out: |
| 3135 |
if (is_unstable) |
| 3136 |
clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); |
| 3137 |
|
| 3138 |
inode_unlock(inode_out); |
| 3139 |
file_accessed(file_in); |
| 3140 |
|
| 3141 |
fuse_flush_time_update(inode_out); |
| 3142 |
|
| 3143 |
return err; |
| 3144 |
} |
| 3145 |
|
| 3146 |
static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, |
| 3147 |
struct file *dst_file, loff_t dst_off, |
| 3148 |
size_t len, unsigned int flags) |
| 3149 |
{ |
| 3150 |
ssize_t ret; |
| 3151 |
|
| 3152 |
ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off, |
| 3153 |
len, flags); |
| 3154 |
|
| 3155 |
if (ret == -EOPNOTSUPP || ret == -EXDEV) |
| 3156 |
ret = splice_copy_file_range(src_file, src_off, dst_file, |
| 3157 |
dst_off, len); |
| 3158 |
return ret; |
| 3159 |
} |
| 3160 |
|
| 3161 |
static const struct file_operations fuse_file_operations = { |
| 3162 |
.llseek = fuse_file_llseek, |
| 3163 |
.read_iter = fuse_file_read_iter, |
| 3164 |
.write_iter = fuse_file_write_iter, |
| 3165 |
.mmap = fuse_file_mmap, |
| 3166 |
.open = fuse_open, |
| 3167 |
.flush = fuse_flush, |
| 3168 |
.release = fuse_release, |
| 3169 |
.fsync = fuse_fsync, |
| 3170 |
.lock = fuse_file_lock, |
| 3171 |
.get_unmapped_area = thp_get_unmapped_area, |
| 3172 |
.flock = fuse_file_flock, |
| 3173 |
.splice_read = fuse_splice_read, |
| 3174 |
.splice_write = fuse_splice_write, |
| 3175 |
.unlocked_ioctl = fuse_file_ioctl, |
| 3176 |
.compat_ioctl = fuse_file_compat_ioctl, |
| 3177 |
.poll = fuse_file_poll, |
| 3178 |
.fallocate = fuse_file_fallocate, |
| 3179 |
.copy_file_range = fuse_copy_file_range, |
| 3180 |
.setlease = generic_setlease, |
| 3181 |
}; |
| 3182 |
|
| 3183 |
static const struct address_space_operations fuse_file_aops = { |
| 3184 |
.read_folio = fuse_read_folio, |
| 3185 |
.readahead = fuse_readahead, |
| 3186 |
.writepages = fuse_writepages, |
| 3187 |
.launder_folio = fuse_launder_folio, |
| 3188 |
.dirty_folio = iomap_dirty_folio, |
| 3189 |
.release_folio = iomap_release_folio, |
| 3190 |
.invalidate_folio = iomap_invalidate_folio, |
| 3191 |
.is_partially_uptodate = iomap_is_partially_uptodate, |
| 3192 |
.migrate_folio = filemap_migrate_folio, |
| 3193 |
.bmap = fuse_bmap, |
| 3194 |
.direct_IO = fuse_direct_IO, |
| 3195 |
}; |
| 3196 |
|
| 3197 |
void fuse_init_file_inode(struct inode *inode, unsigned int flags) |
| 3198 |
{ |
| 3199 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 3200 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 3201 |
|
| 3202 |
inode->i_fop = &fuse_file_operations; |
| 3203 |
inode->i_data.a_ops = &fuse_file_aops; |
| 3204 |
if (fc->writeback_cache) |
| 3205 |
mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data); |
| 3206 |
|
| 3207 |
INIT_LIST_HEAD(&fi->write_files); |
| 3208 |
INIT_LIST_HEAD(&fi->queued_writes); |
| 3209 |
fi->writectr = 0; |
| 3210 |
fi->iocachectr = 0; |
| 3211 |
init_waitqueue_head(&fi->page_waitq); |
| 3212 |
init_waitqueue_head(&fi->direct_io_waitq); |
| 3213 |
|
| 3214 |
if (IS_ENABLED(CONFIG_FUSE_DAX)) |
| 3215 |
fuse_dax_inode_init(inode, flags); |
| 3216 |
} |