| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* |
| 3 |
* dax: direct host memory access |
| 4 |
* Copyright (C) 2020 Red Hat, Inc. |
| 5 |
*/ |
| 6 |
|
| 7 |
#include "fuse_i.h" |
| 8 |
|
| 9 |
#include <linux/delay.h> |
| 10 |
#include <linux/dax.h> |
| 11 |
#include <linux/uio.h> |
| 12 |
#include <linux/pagemap.h> |
| 13 |
#include <linux/iomap.h> |
| 14 |
#include <linux/interval_tree.h> |
| 15 |
|
| 16 |
/* |
| 17 |
* Default memory range size. A power of 2 so it agrees with common FUSE_INIT |
| 18 |
* map_alignment values 4KB and 64KB. |
| 19 |
*/ |
| 20 |
#define FUSE_DAX_SHIFT 21 |
| 21 |
#define FUSE_DAX_SZ (1 << FUSE_DAX_SHIFT) |
| 22 |
#define FUSE_DAX_PAGES (FUSE_DAX_SZ / PAGE_SIZE) |
| 23 |
|
| 24 |
/* Number of ranges reclaimer will try to free in one invocation */ |
| 25 |
#define FUSE_DAX_RECLAIM_CHUNK (10) |
| 26 |
|
| 27 |
/* |
| 28 |
* Dax memory reclaim threshold in percetage of total ranges. When free |
| 29 |
* number of free ranges drops below this threshold, reclaim can trigger |
| 30 |
* Default is 20% |
| 31 |
*/ |
| 32 |
#define FUSE_DAX_RECLAIM_THRESHOLD (20) |
| 33 |
|
| 34 |
/** Translation information for file offsets to DAX window offsets */ |
| 35 |
struct fuse_dax_mapping { |
| 36 |
/* Pointer to inode where this memory range is mapped */ |
| 37 |
struct inode *inode; |
| 38 |
|
| 39 |
/* Will connect in fcd->free_ranges to keep track of free memory */ |
| 40 |
struct list_head list; |
| 41 |
|
| 42 |
/* For interval tree in file/inode */ |
| 43 |
struct interval_tree_node itn; |
| 44 |
|
| 45 |
/* Will connect in fc->busy_ranges to keep track busy memory */ |
| 46 |
struct list_head busy_list; |
| 47 |
|
| 48 |
/** Position in DAX window */ |
| 49 |
u64 window_offset; |
| 50 |
|
| 51 |
/** Length of mapping, in bytes */ |
| 52 |
loff_t length; |
| 53 |
|
| 54 |
/* Is this mapping read-only or read-write */ |
| 55 |
bool writable; |
| 56 |
|
| 57 |
/* reference count when the mapping is used by dax iomap. */ |
| 58 |
refcount_t refcnt; |
| 59 |
}; |
| 60 |
|
| 61 |
/* Per-inode dax map */ |
| 62 |
struct fuse_inode_dax { |
| 63 |
/* Semaphore to protect modifications to the dmap tree */ |
| 64 |
struct rw_semaphore sem; |
| 65 |
|
| 66 |
/* Sorted rb tree of struct fuse_dax_mapping elements */ |
| 67 |
struct rb_root_cached tree; |
| 68 |
unsigned long nr; |
| 69 |
}; |
| 70 |
|
| 71 |
struct fuse_conn_dax { |
| 72 |
/* DAX device */ |
| 73 |
struct dax_device *dev; |
| 74 |
|
| 75 |
/* Lock protecting accessess to members of this structure */ |
| 76 |
spinlock_t lock; |
| 77 |
|
| 78 |
/* List of memory ranges which are busy */ |
| 79 |
unsigned long nr_busy_ranges; |
| 80 |
struct list_head busy_ranges; |
| 81 |
|
| 82 |
/* Worker to free up memory ranges */ |
| 83 |
struct delayed_work free_work; |
| 84 |
|
| 85 |
/* Wait queue for a dax range to become free */ |
| 86 |
wait_queue_head_t range_waitq; |
| 87 |
|
| 88 |
/* DAX Window Free Ranges */ |
| 89 |
long nr_free_ranges; |
| 90 |
struct list_head free_ranges; |
| 91 |
|
| 92 |
unsigned long nr_ranges; |
| 93 |
}; |
| 94 |
|
| 95 |
static inline struct fuse_dax_mapping * |
| 96 |
node_to_dmap(struct interval_tree_node *node) |
| 97 |
{ |
| 98 |
if (!node) |
| 99 |
return NULL; |
| 100 |
|
| 101 |
return container_of(node, struct fuse_dax_mapping, itn); |
| 102 |
} |
| 103 |
|
| 104 |
static struct fuse_dax_mapping * |
| 105 |
alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode); |
| 106 |
|
| 107 |
static void |
| 108 |
__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms) |
| 109 |
{ |
| 110 |
unsigned long free_threshold; |
| 111 |
|
| 112 |
/* If number of free ranges are below threshold, start reclaim */ |
| 113 |
free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100, |
| 114 |
1); |
| 115 |
if (fcd->nr_free_ranges < free_threshold) |
| 116 |
queue_delayed_work(system_long_wq, &fcd->free_work, |
| 117 |
msecs_to_jiffies(delay_ms)); |
| 118 |
} |
| 119 |
|
| 120 |
static void kick_dmap_free_worker(struct fuse_conn_dax *fcd, |
| 121 |
unsigned long delay_ms) |
| 122 |
{ |
| 123 |
spin_lock(&fcd->lock); |
| 124 |
__kick_dmap_free_worker(fcd, delay_ms); |
| 125 |
spin_unlock(&fcd->lock); |
| 126 |
} |
| 127 |
|
| 128 |
static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd) |
| 129 |
{ |
| 130 |
struct fuse_dax_mapping *dmap; |
| 131 |
|
| 132 |
spin_lock(&fcd->lock); |
| 133 |
dmap = list_first_entry_or_null(&fcd->free_ranges, |
| 134 |
struct fuse_dax_mapping, list); |
| 135 |
if (dmap) { |
| 136 |
list_del_init(&dmap->list); |
| 137 |
WARN_ON(fcd->nr_free_ranges <= 0); |
| 138 |
fcd->nr_free_ranges--; |
| 139 |
} |
| 140 |
__kick_dmap_free_worker(fcd, 0); |
| 141 |
spin_unlock(&fcd->lock); |
| 142 |
|
| 143 |
return dmap; |
| 144 |
} |
| 145 |
|
| 146 |
/* This assumes fcd->lock is held */ |
| 147 |
static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd, |
| 148 |
struct fuse_dax_mapping *dmap) |
| 149 |
{ |
| 150 |
list_del_init(&dmap->busy_list); |
| 151 |
WARN_ON(fcd->nr_busy_ranges == 0); |
| 152 |
fcd->nr_busy_ranges--; |
| 153 |
} |
| 154 |
|
| 155 |
static void dmap_remove_busy_list(struct fuse_conn_dax *fcd, |
| 156 |
struct fuse_dax_mapping *dmap) |
| 157 |
{ |
| 158 |
spin_lock(&fcd->lock); |
| 159 |
__dmap_remove_busy_list(fcd, dmap); |
| 160 |
spin_unlock(&fcd->lock); |
| 161 |
} |
| 162 |
|
| 163 |
/* This assumes fcd->lock is held */ |
| 164 |
static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd, |
| 165 |
struct fuse_dax_mapping *dmap) |
| 166 |
{ |
| 167 |
list_add_tail(&dmap->list, &fcd->free_ranges); |
| 168 |
fcd->nr_free_ranges++; |
| 169 |
wake_up(&fcd->range_waitq); |
| 170 |
} |
| 171 |
|
| 172 |
static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd, |
| 173 |
struct fuse_dax_mapping *dmap) |
| 174 |
{ |
| 175 |
/* Return fuse_dax_mapping to free list */ |
| 176 |
spin_lock(&fcd->lock); |
| 177 |
__dmap_add_to_free_pool(fcd, dmap); |
| 178 |
spin_unlock(&fcd->lock); |
| 179 |
} |
| 180 |
|
| 181 |
static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx, |
| 182 |
struct fuse_dax_mapping *dmap, bool writable, |
| 183 |
bool upgrade) |
| 184 |
{ |
| 185 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 186 |
struct fuse_conn_dax *fcd = fm->fc->dax; |
| 187 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 188 |
struct fuse_setupmapping_in inarg; |
| 189 |
loff_t offset = start_idx << FUSE_DAX_SHIFT; |
| 190 |
FUSE_ARGS(args); |
| 191 |
ssize_t err; |
| 192 |
|
| 193 |
WARN_ON(fcd->nr_free_ranges < 0); |
| 194 |
|
| 195 |
/* Ask fuse daemon to setup mapping */ |
| 196 |
memset(&inarg, 0, sizeof(inarg)); |
| 197 |
inarg.foffset = offset; |
| 198 |
inarg.fh = -1; |
| 199 |
inarg.moffset = dmap->window_offset; |
| 200 |
inarg.len = FUSE_DAX_SZ; |
| 201 |
inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ; |
| 202 |
if (writable) |
| 203 |
inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE; |
| 204 |
args.opcode = FUSE_SETUPMAPPING; |
| 205 |
args.nodeid = fi->nodeid; |
| 206 |
args.in_numargs = 1; |
| 207 |
args.in_args[0].size = sizeof(inarg); |
| 208 |
args.in_args[0].value = &inarg; |
| 209 |
err = fuse_simple_request(fm, &args); |
| 210 |
if (err < 0) |
| 211 |
return err; |
| 212 |
dmap->writable = writable; |
| 213 |
if (!upgrade) { |
| 214 |
/* |
| 215 |
* We don't take a reference on inode. inode is valid right now |
| 216 |
* and when inode is going away, cleanup logic should first |
| 217 |
* cleanup dmap entries. |
| 218 |
*/ |
| 219 |
dmap->inode = inode; |
| 220 |
dmap->itn.start = dmap->itn.last = start_idx; |
| 221 |
/* Protected by fi->dax->sem */ |
| 222 |
interval_tree_insert(&dmap->itn, &fi->dax->tree); |
| 223 |
fi->dax->nr++; |
| 224 |
spin_lock(&fcd->lock); |
| 225 |
list_add_tail(&dmap->busy_list, &fcd->busy_ranges); |
| 226 |
fcd->nr_busy_ranges++; |
| 227 |
spin_unlock(&fcd->lock); |
| 228 |
} |
| 229 |
return 0; |
| 230 |
} |
| 231 |
|
| 232 |
static int fuse_send_removemapping(struct inode *inode, |
| 233 |
struct fuse_removemapping_in *inargp, |
| 234 |
struct fuse_removemapping_one *remove_one) |
| 235 |
{ |
| 236 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 237 |
struct fuse_mount *fm = get_fuse_mount(inode); |
| 238 |
FUSE_ARGS(args); |
| 239 |
|
| 240 |
args.opcode = FUSE_REMOVEMAPPING; |
| 241 |
args.nodeid = fi->nodeid; |
| 242 |
args.in_numargs = 3; |
| 243 |
fuse_set_zero_arg0(&args); |
| 244 |
args.in_args[1].size = sizeof(*inargp); |
| 245 |
args.in_args[1].value = inargp; |
| 246 |
args.in_args[2].size = inargp->count * sizeof(*remove_one); |
| 247 |
args.in_args[2].value = remove_one; |
| 248 |
return fuse_simple_request(fm, &args); |
| 249 |
} |
| 250 |
|
| 251 |
static int dmap_removemapping_list(struct inode *inode, unsigned int num, |
| 252 |
struct list_head *to_remove) |
| 253 |
{ |
| 254 |
struct fuse_removemapping_one *remove_one, *ptr; |
| 255 |
struct fuse_removemapping_in inarg; |
| 256 |
struct fuse_dax_mapping *dmap; |
| 257 |
int ret, i = 0, nr_alloc; |
| 258 |
|
| 259 |
nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY); |
| 260 |
remove_one = kmalloc_objs(*remove_one, nr_alloc, GFP_NOFS); |
| 261 |
if (!remove_one) |
| 262 |
return -ENOMEM; |
| 263 |
|
| 264 |
ptr = remove_one; |
| 265 |
list_for_each_entry(dmap, to_remove, list) { |
| 266 |
ptr->moffset = dmap->window_offset; |
| 267 |
ptr->len = dmap->length; |
| 268 |
ptr++; |
| 269 |
i++; |
| 270 |
num--; |
| 271 |
if (i >= nr_alloc || num == 0) { |
| 272 |
memset(&inarg, 0, sizeof(inarg)); |
| 273 |
inarg.count = i; |
| 274 |
ret = fuse_send_removemapping(inode, &inarg, |
| 275 |
remove_one); |
| 276 |
if (ret) |
| 277 |
goto out; |
| 278 |
ptr = remove_one; |
| 279 |
i = 0; |
| 280 |
} |
| 281 |
} |
| 282 |
out: |
| 283 |
kfree(remove_one); |
| 284 |
return ret; |
| 285 |
} |
| 286 |
|
| 287 |
/* |
| 288 |
* Cleanup dmap entry and add back to free list. This should be called with |
| 289 |
* fcd->lock held. |
| 290 |
*/ |
| 291 |
static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd, |
| 292 |
struct fuse_dax_mapping *dmap) |
| 293 |
{ |
| 294 |
pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n", |
| 295 |
dmap->itn.start, dmap->itn.last, dmap->window_offset, |
| 296 |
dmap->length); |
| 297 |
__dmap_remove_busy_list(fcd, dmap); |
| 298 |
dmap->inode = NULL; |
| 299 |
dmap->itn.start = dmap->itn.last = 0; |
| 300 |
__dmap_add_to_free_pool(fcd, dmap); |
| 301 |
} |
| 302 |
|
| 303 |
/* |
| 304 |
* Free inode dmap entries whose range falls inside [start, end]. |
| 305 |
* Does not take any locks. At this point of time it should only be |
| 306 |
* called from evict_inode() path where we know all dmap entries can be |
| 307 |
* reclaimed. |
| 308 |
*/ |
| 309 |
static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd, |
| 310 |
struct inode *inode, |
| 311 |
loff_t start, loff_t end) |
| 312 |
{ |
| 313 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 314 |
struct fuse_dax_mapping *dmap, *n; |
| 315 |
int err, num = 0; |
| 316 |
LIST_HEAD(to_remove); |
| 317 |
unsigned long start_idx = start >> FUSE_DAX_SHIFT; |
| 318 |
unsigned long end_idx = end >> FUSE_DAX_SHIFT; |
| 319 |
struct interval_tree_node *node; |
| 320 |
|
| 321 |
while (1) { |
| 322 |
node = interval_tree_iter_first(&fi->dax->tree, start_idx, |
| 323 |
end_idx); |
| 324 |
if (!node) |
| 325 |
break; |
| 326 |
dmap = node_to_dmap(node); |
| 327 |
/* inode is going away. There should not be any users of dmap */ |
| 328 |
WARN_ON(refcount_read(&dmap->refcnt) > 1); |
| 329 |
interval_tree_remove(&dmap->itn, &fi->dax->tree); |
| 330 |
num++; |
| 331 |
list_add(&dmap->list, &to_remove); |
| 332 |
} |
| 333 |
|
| 334 |
/* Nothing to remove */ |
| 335 |
if (list_empty(&to_remove)) |
| 336 |
return; |
| 337 |
|
| 338 |
WARN_ON(fi->dax->nr < num); |
| 339 |
fi->dax->nr -= num; |
| 340 |
err = dmap_removemapping_list(inode, num, &to_remove); |
| 341 |
if (err && err != -ENOTCONN) { |
| 342 |
pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n", |
| 343 |
start, end); |
| 344 |
} |
| 345 |
spin_lock(&fcd->lock); |
| 346 |
list_for_each_entry_safe(dmap, n, &to_remove, list) { |
| 347 |
list_del_init(&dmap->list); |
| 348 |
dmap_reinit_add_to_free_pool(fcd, dmap); |
| 349 |
} |
| 350 |
spin_unlock(&fcd->lock); |
| 351 |
} |
| 352 |
|
| 353 |
static int dmap_removemapping_one(struct inode *inode, |
| 354 |
struct fuse_dax_mapping *dmap) |
| 355 |
{ |
| 356 |
struct fuse_removemapping_one forget_one; |
| 357 |
struct fuse_removemapping_in inarg; |
| 358 |
|
| 359 |
memset(&inarg, 0, sizeof(inarg)); |
| 360 |
inarg.count = 1; |
| 361 |
memset(&forget_one, 0, sizeof(forget_one)); |
| 362 |
forget_one.moffset = dmap->window_offset; |
| 363 |
forget_one.len = dmap->length; |
| 364 |
|
| 365 |
return fuse_send_removemapping(inode, &inarg, &forget_one); |
| 366 |
} |
| 367 |
|
| 368 |
/* |
| 369 |
* It is called from evict_inode() and by that time inode is going away. So |
| 370 |
* this function does not take any locks like fi->dax->sem for traversing |
| 371 |
* that fuse inode interval tree. If that lock is taken then lock validator |
| 372 |
* complains of deadlock situation w.r.t fs_reclaim lock. |
| 373 |
*/ |
| 374 |
void fuse_dax_inode_cleanup(struct inode *inode) |
| 375 |
{ |
| 376 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 377 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 378 |
|
| 379 |
/* |
| 380 |
* fuse_evict_inode() has already called truncate_inode_pages_final() |
| 381 |
* before we arrive here. So we should not have to worry about any |
| 382 |
* pages/exception entries still associated with inode. |
| 383 |
*/ |
| 384 |
inode_reclaim_dmap_range(fc->dax, inode, 0, -1); |
| 385 |
WARN_ON(fi->dax->nr); |
| 386 |
} |
| 387 |
|
| 388 |
static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length) |
| 389 |
{ |
| 390 |
iomap->addr = IOMAP_NULL_ADDR; |
| 391 |
iomap->length = length; |
| 392 |
iomap->type = IOMAP_HOLE; |
| 393 |
} |
| 394 |
|
| 395 |
static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length, |
| 396 |
struct iomap *iomap, struct fuse_dax_mapping *dmap, |
| 397 |
unsigned int flags) |
| 398 |
{ |
| 399 |
loff_t offset, len; |
| 400 |
loff_t i_size = i_size_read(inode); |
| 401 |
|
| 402 |
offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT); |
| 403 |
len = min(length, dmap->length - offset); |
| 404 |
|
| 405 |
/* If length is beyond end of file, truncate further */ |
| 406 |
if (pos + len > i_size) |
| 407 |
len = i_size - pos; |
| 408 |
|
| 409 |
if (len > 0) { |
| 410 |
iomap->addr = dmap->window_offset + offset; |
| 411 |
iomap->length = len; |
| 412 |
if (flags & IOMAP_FAULT) |
| 413 |
iomap->length = ALIGN(len, PAGE_SIZE); |
| 414 |
iomap->type = IOMAP_MAPPED; |
| 415 |
/* |
| 416 |
* increace refcnt so that reclaim code knows this dmap is in |
| 417 |
* use. This assumes fi->dax->sem mutex is held either |
| 418 |
* shared/exclusive. |
| 419 |
*/ |
| 420 |
refcount_inc(&dmap->refcnt); |
| 421 |
|
| 422 |
/* iomap->private should be NULL */ |
| 423 |
WARN_ON_ONCE(iomap->private); |
| 424 |
iomap->private = dmap; |
| 425 |
} else { |
| 426 |
/* Mapping beyond end of file is hole */ |
| 427 |
fuse_fill_iomap_hole(iomap, length); |
| 428 |
} |
| 429 |
} |
| 430 |
|
| 431 |
static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos, |
| 432 |
loff_t length, unsigned int flags, |
| 433 |
struct iomap *iomap) |
| 434 |
{ |
| 435 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 436 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 437 |
struct fuse_conn_dax *fcd = fc->dax; |
| 438 |
struct fuse_dax_mapping *dmap, *alloc_dmap = NULL; |
| 439 |
int ret; |
| 440 |
bool writable = flags & IOMAP_WRITE; |
| 441 |
unsigned long start_idx = pos >> FUSE_DAX_SHIFT; |
| 442 |
struct interval_tree_node *node; |
| 443 |
|
| 444 |
/* |
| 445 |
* Can't do inline reclaim in fault path. We call |
| 446 |
* dax_layout_busy_page() before we free a range. And |
| 447 |
* fuse_wait_dax_page() drops mapping->invalidate_lock and requires it. |
| 448 |
* In fault path we enter with mapping->invalidate_lock held and can't |
| 449 |
* drop it. Also in fault path we hold mapping->invalidate_lock shared |
| 450 |
* and not exclusive, so that creates further issues with |
| 451 |
* fuse_wait_dax_page(). Hence return -EAGAIN and fuse_dax_fault() |
| 452 |
* will wait for a memory range to become free and retry. |
| 453 |
*/ |
| 454 |
if (flags & IOMAP_FAULT) { |
| 455 |
alloc_dmap = alloc_dax_mapping(fcd); |
| 456 |
if (!alloc_dmap) |
| 457 |
return -EAGAIN; |
| 458 |
} else { |
| 459 |
alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode); |
| 460 |
if (IS_ERR(alloc_dmap)) |
| 461 |
return PTR_ERR(alloc_dmap); |
| 462 |
} |
| 463 |
|
| 464 |
/* If we are here, we should have memory allocated */ |
| 465 |
if (WARN_ON(!alloc_dmap)) |
| 466 |
return -EIO; |
| 467 |
|
| 468 |
/* |
| 469 |
* Take write lock so that only one caller can try to setup mapping |
| 470 |
* and other waits. |
| 471 |
*/ |
| 472 |
down_write(&fi->dax->sem); |
| 473 |
/* |
| 474 |
* We dropped lock. Check again if somebody else setup |
| 475 |
* mapping already. |
| 476 |
*/ |
| 477 |
node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); |
| 478 |
if (node) { |
| 479 |
dmap = node_to_dmap(node); |
| 480 |
fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); |
| 481 |
dmap_add_to_free_pool(fcd, alloc_dmap); |
| 482 |
up_write(&fi->dax->sem); |
| 483 |
return 0; |
| 484 |
} |
| 485 |
|
| 486 |
/* Setup one mapping */ |
| 487 |
ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap, |
| 488 |
writable, false); |
| 489 |
if (ret < 0) { |
| 490 |
dmap_add_to_free_pool(fcd, alloc_dmap); |
| 491 |
up_write(&fi->dax->sem); |
| 492 |
return ret; |
| 493 |
} |
| 494 |
fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags); |
| 495 |
up_write(&fi->dax->sem); |
| 496 |
return 0; |
| 497 |
} |
| 498 |
|
| 499 |
static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos, |
| 500 |
loff_t length, unsigned int flags, |
| 501 |
struct iomap *iomap) |
| 502 |
{ |
| 503 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 504 |
struct fuse_dax_mapping *dmap; |
| 505 |
int ret; |
| 506 |
unsigned long idx = pos >> FUSE_DAX_SHIFT; |
| 507 |
struct interval_tree_node *node; |
| 508 |
|
| 509 |
/* |
| 510 |
* Take exclusive lock so that only one caller can try to setup |
| 511 |
* mapping and others wait. |
| 512 |
*/ |
| 513 |
down_write(&fi->dax->sem); |
| 514 |
node = interval_tree_iter_first(&fi->dax->tree, idx, idx); |
| 515 |
|
| 516 |
/* We are holding either inode lock or invalidate_lock, and that should |
| 517 |
* ensure that dmap can't be truncated. We are holding a reference |
| 518 |
* on dmap and that should make sure it can't be reclaimed. So dmap |
| 519 |
* should still be there in tree despite the fact we dropped and |
| 520 |
* re-acquired the fi->dax->sem lock. |
| 521 |
*/ |
| 522 |
ret = -EIO; |
| 523 |
if (WARN_ON(!node)) |
| 524 |
goto out_err; |
| 525 |
|
| 526 |
dmap = node_to_dmap(node); |
| 527 |
|
| 528 |
/* We took an extra reference on dmap to make sure its not reclaimd. |
| 529 |
* Now we hold fi->dax->sem lock and that reference is not needed |
| 530 |
* anymore. Drop it. |
| 531 |
*/ |
| 532 |
if (refcount_dec_and_test(&dmap->refcnt)) { |
| 533 |
/* refcount should not hit 0. This object only goes |
| 534 |
* away when fuse connection goes away |
| 535 |
*/ |
| 536 |
WARN_ON_ONCE(1); |
| 537 |
} |
| 538 |
|
| 539 |
/* Maybe another thread already upgraded mapping while we were not |
| 540 |
* holding lock. |
| 541 |
*/ |
| 542 |
if (dmap->writable) { |
| 543 |
ret = 0; |
| 544 |
goto out_fill_iomap; |
| 545 |
} |
| 546 |
|
| 547 |
ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true, |
| 548 |
true); |
| 549 |
if (ret < 0) |
| 550 |
goto out_err; |
| 551 |
out_fill_iomap: |
| 552 |
fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); |
| 553 |
out_err: |
| 554 |
up_write(&fi->dax->sem); |
| 555 |
return ret; |
| 556 |
} |
| 557 |
|
| 558 |
/* This is just for DAX and the mapping is ephemeral, do not use it for other |
| 559 |
* purposes since there is no block device with a permanent mapping. |
| 560 |
*/ |
| 561 |
static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length, |
| 562 |
unsigned int flags, struct iomap *iomap, |
| 563 |
struct iomap *srcmap) |
| 564 |
{ |
| 565 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 566 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 567 |
struct fuse_dax_mapping *dmap; |
| 568 |
bool writable = flags & IOMAP_WRITE; |
| 569 |
unsigned long start_idx = pos >> FUSE_DAX_SHIFT; |
| 570 |
struct interval_tree_node *node; |
| 571 |
|
| 572 |
/* We don't support FIEMAP */ |
| 573 |
if (WARN_ON(flags & IOMAP_REPORT)) |
| 574 |
return -EIO; |
| 575 |
|
| 576 |
iomap->offset = pos; |
| 577 |
iomap->flags = 0; |
| 578 |
iomap->bdev = NULL; |
| 579 |
iomap->dax_dev = fc->dax->dev; |
| 580 |
|
| 581 |
/* |
| 582 |
* Both read/write and mmap path can race here. So we need something |
| 583 |
* to make sure if we are setting up mapping, then other path waits |
| 584 |
* |
| 585 |
* For now, use a semaphore for this. It probably needs to be |
| 586 |
* optimized later. |
| 587 |
*/ |
| 588 |
down_read(&fi->dax->sem); |
| 589 |
node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); |
| 590 |
if (node) { |
| 591 |
dmap = node_to_dmap(node); |
| 592 |
if (writable && !dmap->writable) { |
| 593 |
/* Upgrade read-only mapping to read-write. This will |
| 594 |
* require exclusive fi->dax->sem lock as we don't want |
| 595 |
* two threads to be trying to this simultaneously |
| 596 |
* for same dmap. So drop shared lock and acquire |
| 597 |
* exclusive lock. |
| 598 |
* |
| 599 |
* Before dropping fi->dax->sem lock, take reference |
| 600 |
* on dmap so that its not freed by range reclaim. |
| 601 |
*/ |
| 602 |
refcount_inc(&dmap->refcnt); |
| 603 |
up_read(&fi->dax->sem); |
| 604 |
pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n", |
| 605 |
__func__, pos, length); |
| 606 |
return fuse_upgrade_dax_mapping(inode, pos, length, |
| 607 |
flags, iomap); |
| 608 |
} else { |
| 609 |
fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); |
| 610 |
up_read(&fi->dax->sem); |
| 611 |
return 0; |
| 612 |
} |
| 613 |
} else { |
| 614 |
up_read(&fi->dax->sem); |
| 615 |
pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n", |
| 616 |
__func__, pos, length); |
| 617 |
if (pos >= i_size_read(inode)) |
| 618 |
goto iomap_hole; |
| 619 |
|
| 620 |
return fuse_setup_new_dax_mapping(inode, pos, length, flags, |
| 621 |
iomap); |
| 622 |
} |
| 623 |
|
| 624 |
/* |
| 625 |
* If read beyond end of file happens, fs code seems to return |
| 626 |
* it as hole |
| 627 |
*/ |
| 628 |
iomap_hole: |
| 629 |
fuse_fill_iomap_hole(iomap, length); |
| 630 |
pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n", |
| 631 |
__func__, pos, length, iomap->length); |
| 632 |
return 0; |
| 633 |
} |
| 634 |
|
| 635 |
static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length, |
| 636 |
ssize_t written, unsigned int flags, |
| 637 |
struct iomap *iomap) |
| 638 |
{ |
| 639 |
struct fuse_dax_mapping *dmap = iomap->private; |
| 640 |
|
| 641 |
if (dmap) { |
| 642 |
if (refcount_dec_and_test(&dmap->refcnt)) { |
| 643 |
/* refcount should not hit 0. This object only goes |
| 644 |
* away when fuse connection goes away |
| 645 |
*/ |
| 646 |
WARN_ON_ONCE(1); |
| 647 |
} |
| 648 |
} |
| 649 |
|
| 650 |
/* DAX writes beyond end-of-file aren't handled using iomap, so the |
| 651 |
* file size is unchanged and there is nothing to do here. |
| 652 |
*/ |
| 653 |
return 0; |
| 654 |
} |
| 655 |
|
| 656 |
static const struct iomap_ops fuse_iomap_ops = { |
| 657 |
.iomap_begin = fuse_iomap_begin, |
| 658 |
.iomap_end = fuse_iomap_end, |
| 659 |
}; |
| 660 |
|
| 661 |
static void fuse_wait_dax_page(struct inode *inode) |
| 662 |
{ |
| 663 |
filemap_invalidate_unlock(inode->i_mapping); |
| 664 |
schedule(); |
| 665 |
filemap_invalidate_lock(inode->i_mapping); |
| 666 |
} |
| 667 |
|
| 668 |
/* Should be called with mapping->invalidate_lock held exclusively. */ |
| 669 |
int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start, |
| 670 |
u64 dmap_end) |
| 671 |
{ |
| 672 |
return dax_break_layout(inode, dmap_start, dmap_end, |
| 673 |
fuse_wait_dax_page); |
| 674 |
} |
| 675 |
|
| 676 |
ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) |
| 677 |
{ |
| 678 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 679 |
ssize_t ret; |
| 680 |
|
| 681 |
if (iocb->ki_flags & IOCB_NOWAIT) { |
| 682 |
if (!inode_trylock_shared(inode)) |
| 683 |
return -EAGAIN; |
| 684 |
} else { |
| 685 |
inode_lock_shared(inode); |
| 686 |
} |
| 687 |
|
| 688 |
ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops); |
| 689 |
inode_unlock_shared(inode); |
| 690 |
|
| 691 |
/* TODO file_accessed(iocb->f_filp) */ |
| 692 |
return ret; |
| 693 |
} |
| 694 |
|
| 695 |
static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from) |
| 696 |
{ |
| 697 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 698 |
|
| 699 |
return (iov_iter_rw(from) == WRITE && |
| 700 |
((iocb->ki_pos) >= i_size_read(inode) || |
| 701 |
(iocb->ki_pos + iov_iter_count(from) > i_size_read(inode)))); |
| 702 |
} |
| 703 |
|
| 704 |
static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from) |
| 705 |
{ |
| 706 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 707 |
struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); |
| 708 |
ssize_t ret; |
| 709 |
|
| 710 |
ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); |
| 711 |
|
| 712 |
fuse_write_update_attr(inode, iocb->ki_pos, ret); |
| 713 |
return ret; |
| 714 |
} |
| 715 |
|
| 716 |
ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) |
| 717 |
{ |
| 718 |
struct inode *inode = file_inode(iocb->ki_filp); |
| 719 |
ssize_t ret; |
| 720 |
|
| 721 |
if (iocb->ki_flags & IOCB_NOWAIT) { |
| 722 |
if (!inode_trylock(inode)) |
| 723 |
return -EAGAIN; |
| 724 |
} else { |
| 725 |
inode_lock(inode); |
| 726 |
} |
| 727 |
|
| 728 |
ret = generic_write_checks(iocb, from); |
| 729 |
if (ret <= 0) |
| 730 |
goto out; |
| 731 |
|
| 732 |
ret = file_remove_privs(iocb->ki_filp); |
| 733 |
if (ret) |
| 734 |
goto out; |
| 735 |
/* TODO file_update_time() but we don't want metadata I/O */ |
| 736 |
|
| 737 |
/* Do not use dax for file extending writes as write and on |
| 738 |
* disk i_size increase are not atomic otherwise. |
| 739 |
*/ |
| 740 |
if (file_extending_write(iocb, from)) |
| 741 |
ret = fuse_dax_direct_write(iocb, from); |
| 742 |
else |
| 743 |
ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops); |
| 744 |
|
| 745 |
out: |
| 746 |
inode_unlock(inode); |
| 747 |
|
| 748 |
if (ret > 0) |
| 749 |
ret = generic_write_sync(iocb, ret); |
| 750 |
return ret; |
| 751 |
} |
| 752 |
|
| 753 |
static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order, |
| 754 |
bool write) |
| 755 |
{ |
| 756 |
vm_fault_t ret; |
| 757 |
struct inode *inode = file_inode(vmf->vma->vm_file); |
| 758 |
struct super_block *sb = inode->i_sb; |
| 759 |
unsigned long pfn; |
| 760 |
int error = 0; |
| 761 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 762 |
struct fuse_conn_dax *fcd = fc->dax; |
| 763 |
bool retry = false; |
| 764 |
|
| 765 |
if (write) |
| 766 |
sb_start_pagefault(sb); |
| 767 |
retry: |
| 768 |
if (retry && !(fcd->nr_free_ranges > 0)) |
| 769 |
wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0)); |
| 770 |
|
| 771 |
/* |
| 772 |
* We need to serialize against not only truncate but also against |
| 773 |
* fuse dax memory range reclaim. While a range is being reclaimed, |
| 774 |
* we do not want any read/write/mmap to make progress and try |
| 775 |
* to populate page cache or access memory we are trying to free. |
| 776 |
*/ |
| 777 |
filemap_invalidate_lock_shared(inode->i_mapping); |
| 778 |
ret = dax_iomap_fault(vmf, order, &pfn, &error, &fuse_iomap_ops); |
| 779 |
if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) { |
| 780 |
error = 0; |
| 781 |
retry = true; |
| 782 |
filemap_invalidate_unlock_shared(inode->i_mapping); |
| 783 |
goto retry; |
| 784 |
} |
| 785 |
|
| 786 |
if (ret & VM_FAULT_NEEDDSYNC) |
| 787 |
ret = dax_finish_sync_fault(vmf, order, pfn); |
| 788 |
filemap_invalidate_unlock_shared(inode->i_mapping); |
| 789 |
|
| 790 |
if (write) |
| 791 |
sb_end_pagefault(sb); |
| 792 |
|
| 793 |
return ret; |
| 794 |
} |
| 795 |
|
| 796 |
static vm_fault_t fuse_dax_fault(struct vm_fault *vmf) |
| 797 |
{ |
| 798 |
return __fuse_dax_fault(vmf, 0, vmf->flags & FAULT_FLAG_WRITE); |
| 799 |
} |
| 800 |
|
| 801 |
static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf, unsigned int order) |
| 802 |
{ |
| 803 |
return __fuse_dax_fault(vmf, order, vmf->flags & FAULT_FLAG_WRITE); |
| 804 |
} |
| 805 |
|
| 806 |
static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf) |
| 807 |
{ |
| 808 |
return __fuse_dax_fault(vmf, 0, true); |
| 809 |
} |
| 810 |
|
| 811 |
static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf) |
| 812 |
{ |
| 813 |
return __fuse_dax_fault(vmf, 0, true); |
| 814 |
} |
| 815 |
|
| 816 |
static const struct vm_operations_struct fuse_dax_vm_ops = { |
| 817 |
.fault = fuse_dax_fault, |
| 818 |
.huge_fault = fuse_dax_huge_fault, |
| 819 |
.page_mkwrite = fuse_dax_page_mkwrite, |
| 820 |
.pfn_mkwrite = fuse_dax_pfn_mkwrite, |
| 821 |
}; |
| 822 |
|
| 823 |
int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma) |
| 824 |
{ |
| 825 |
file_accessed(file); |
| 826 |
vma->vm_ops = &fuse_dax_vm_ops; |
| 827 |
vm_flags_set(vma, VM_MIXEDMAP | VM_HUGEPAGE); |
| 828 |
return 0; |
| 829 |
} |
| 830 |
|
| 831 |
static int dmap_writeback_invalidate(struct inode *inode, |
| 832 |
struct fuse_dax_mapping *dmap) |
| 833 |
{ |
| 834 |
int ret; |
| 835 |
loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT; |
| 836 |
loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1); |
| 837 |
|
| 838 |
ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos); |
| 839 |
if (ret) { |
| 840 |
pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n", |
| 841 |
ret, start_pos, end_pos); |
| 842 |
return ret; |
| 843 |
} |
| 844 |
|
| 845 |
ret = invalidate_inode_pages2_range(inode->i_mapping, |
| 846 |
start_pos >> PAGE_SHIFT, |
| 847 |
end_pos >> PAGE_SHIFT); |
| 848 |
if (ret) |
| 849 |
pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n", |
| 850 |
ret); |
| 851 |
|
| 852 |
return ret; |
| 853 |
} |
| 854 |
|
| 855 |
static int reclaim_one_dmap_locked(struct inode *inode, |
| 856 |
struct fuse_dax_mapping *dmap) |
| 857 |
{ |
| 858 |
int ret; |
| 859 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 860 |
|
| 861 |
/* |
| 862 |
* igrab() was done to make sure inode won't go under us, and this |
| 863 |
* further avoids the race with evict(). |
| 864 |
*/ |
| 865 |
ret = dmap_writeback_invalidate(inode, dmap); |
| 866 |
if (ret) |
| 867 |
return ret; |
| 868 |
|
| 869 |
/* Remove dax mapping from inode interval tree now */ |
| 870 |
interval_tree_remove(&dmap->itn, &fi->dax->tree); |
| 871 |
fi->dax->nr--; |
| 872 |
|
| 873 |
/* It is possible that umount/shutdown has killed the fuse connection |
| 874 |
* and worker thread is trying to reclaim memory in parallel. Don't |
| 875 |
* warn in that case. |
| 876 |
*/ |
| 877 |
ret = dmap_removemapping_one(inode, dmap); |
| 878 |
if (ret && ret != -ENOTCONN) { |
| 879 |
pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n", |
| 880 |
dmap->window_offset, dmap->length, ret); |
| 881 |
} |
| 882 |
return 0; |
| 883 |
} |
| 884 |
|
| 885 |
/* Find first mapped dmap for an inode and return file offset. Caller needs |
| 886 |
* to hold fi->dax->sem lock either shared or exclusive. |
| 887 |
*/ |
| 888 |
static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode) |
| 889 |
{ |
| 890 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 891 |
struct fuse_dax_mapping *dmap; |
| 892 |
struct interval_tree_node *node; |
| 893 |
|
| 894 |
for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node; |
| 895 |
node = interval_tree_iter_next(node, 0, -1)) { |
| 896 |
dmap = node_to_dmap(node); |
| 897 |
/* still in use. */ |
| 898 |
if (refcount_read(&dmap->refcnt) > 1) |
| 899 |
continue; |
| 900 |
|
| 901 |
return dmap; |
| 902 |
} |
| 903 |
|
| 904 |
return NULL; |
| 905 |
} |
| 906 |
|
| 907 |
/* |
| 908 |
* Find first mapping in the tree and free it and return it. Do not add |
| 909 |
* it back to free pool. |
| 910 |
*/ |
| 911 |
static struct fuse_dax_mapping * |
| 912 |
inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode, |
| 913 |
bool *retry) |
| 914 |
{ |
| 915 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 916 |
struct fuse_dax_mapping *dmap; |
| 917 |
u64 dmap_start, dmap_end; |
| 918 |
unsigned long start_idx; |
| 919 |
int ret; |
| 920 |
struct interval_tree_node *node; |
| 921 |
|
| 922 |
filemap_invalidate_lock(inode->i_mapping); |
| 923 |
|
| 924 |
/* Lookup a dmap and corresponding file offset to reclaim. */ |
| 925 |
down_read(&fi->dax->sem); |
| 926 |
dmap = inode_lookup_first_dmap(inode); |
| 927 |
if (dmap) { |
| 928 |
start_idx = dmap->itn.start; |
| 929 |
dmap_start = start_idx << FUSE_DAX_SHIFT; |
| 930 |
dmap_end = dmap_start + FUSE_DAX_SZ - 1; |
| 931 |
} |
| 932 |
up_read(&fi->dax->sem); |
| 933 |
|
| 934 |
if (!dmap) |
| 935 |
goto out_mmap_sem; |
| 936 |
/* |
| 937 |
* Make sure there are no references to inode pages using |
| 938 |
* get_user_pages() |
| 939 |
*/ |
| 940 |
ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end); |
| 941 |
if (ret) { |
| 942 |
pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n", |
| 943 |
ret); |
| 944 |
dmap = ERR_PTR(ret); |
| 945 |
goto out_mmap_sem; |
| 946 |
} |
| 947 |
|
| 948 |
down_write(&fi->dax->sem); |
| 949 |
node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); |
| 950 |
/* Range already got reclaimed by somebody else */ |
| 951 |
if (!node) { |
| 952 |
if (retry) |
| 953 |
*retry = true; |
| 954 |
goto out_write_dmap_sem; |
| 955 |
} |
| 956 |
|
| 957 |
dmap = node_to_dmap(node); |
| 958 |
/* still in use. */ |
| 959 |
if (refcount_read(&dmap->refcnt) > 1) { |
| 960 |
dmap = NULL; |
| 961 |
if (retry) |
| 962 |
*retry = true; |
| 963 |
goto out_write_dmap_sem; |
| 964 |
} |
| 965 |
|
| 966 |
ret = reclaim_one_dmap_locked(inode, dmap); |
| 967 |
if (ret < 0) { |
| 968 |
dmap = ERR_PTR(ret); |
| 969 |
goto out_write_dmap_sem; |
| 970 |
} |
| 971 |
|
| 972 |
/* Clean up dmap. Do not add back to free list */ |
| 973 |
dmap_remove_busy_list(fcd, dmap); |
| 974 |
dmap->inode = NULL; |
| 975 |
dmap->itn.start = dmap->itn.last = 0; |
| 976 |
|
| 977 |
pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n", |
| 978 |
__func__, inode, dmap->window_offset, dmap->length); |
| 979 |
|
| 980 |
out_write_dmap_sem: |
| 981 |
up_write(&fi->dax->sem); |
| 982 |
out_mmap_sem: |
| 983 |
filemap_invalidate_unlock(inode->i_mapping); |
| 984 |
return dmap; |
| 985 |
} |
| 986 |
|
| 987 |
static struct fuse_dax_mapping * |
| 988 |
alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode) |
| 989 |
{ |
| 990 |
struct fuse_dax_mapping *dmap; |
| 991 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 992 |
|
| 993 |
while (1) { |
| 994 |
bool retry = false; |
| 995 |
|
| 996 |
dmap = alloc_dax_mapping(fcd); |
| 997 |
if (dmap) |
| 998 |
return dmap; |
| 999 |
|
| 1000 |
dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry); |
| 1001 |
/* |
| 1002 |
* Either we got a mapping or it is an error, return in both |
| 1003 |
* the cases. |
| 1004 |
*/ |
| 1005 |
if (dmap) |
| 1006 |
return dmap; |
| 1007 |
|
| 1008 |
/* If we could not reclaim a mapping because it |
| 1009 |
* had a reference or some other temporary failure, |
| 1010 |
* Try again. We want to give up inline reclaim only |
| 1011 |
* if there is no range assigned to this node. Otherwise |
| 1012 |
* if a deadlock is possible if we sleep with |
| 1013 |
* mapping->invalidate_lock held and worker to free memory |
| 1014 |
* can't make progress due to unavailability of |
| 1015 |
* mapping->invalidate_lock. So sleep only if fi->dax->nr=0 |
| 1016 |
*/ |
| 1017 |
if (retry) |
| 1018 |
continue; |
| 1019 |
/* |
| 1020 |
* There are no mappings which can be reclaimed. Wait for one. |
| 1021 |
* We are not holding fi->dax->sem. So it is possible |
| 1022 |
* that range gets added now. But as we are not holding |
| 1023 |
* mapping->invalidate_lock, worker should still be able to |
| 1024 |
* free up a range and wake us up. |
| 1025 |
*/ |
| 1026 |
if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) { |
| 1027 |
if (wait_event_killable_exclusive(fcd->range_waitq, |
| 1028 |
(fcd->nr_free_ranges > 0))) { |
| 1029 |
return ERR_PTR(-EINTR); |
| 1030 |
} |
| 1031 |
} |
| 1032 |
} |
| 1033 |
} |
| 1034 |
|
| 1035 |
static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd, |
| 1036 |
struct inode *inode, |
| 1037 |
unsigned long start_idx) |
| 1038 |
{ |
| 1039 |
int ret; |
| 1040 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1041 |
struct fuse_dax_mapping *dmap; |
| 1042 |
struct interval_tree_node *node; |
| 1043 |
|
| 1044 |
/* Find fuse dax mapping at file offset inode. */ |
| 1045 |
node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); |
| 1046 |
|
| 1047 |
/* Range already got cleaned up by somebody else */ |
| 1048 |
if (!node) |
| 1049 |
return 0; |
| 1050 |
dmap = node_to_dmap(node); |
| 1051 |
|
| 1052 |
/* still in use. */ |
| 1053 |
if (refcount_read(&dmap->refcnt) > 1) |
| 1054 |
return 0; |
| 1055 |
|
| 1056 |
ret = reclaim_one_dmap_locked(inode, dmap); |
| 1057 |
if (ret < 0) |
| 1058 |
return ret; |
| 1059 |
|
| 1060 |
/* Cleanup dmap entry and add back to free list */ |
| 1061 |
spin_lock(&fcd->lock); |
| 1062 |
dmap_reinit_add_to_free_pool(fcd, dmap); |
| 1063 |
spin_unlock(&fcd->lock); |
| 1064 |
return ret; |
| 1065 |
} |
| 1066 |
|
| 1067 |
/* |
| 1068 |
* Free a range of memory. |
| 1069 |
* Locking: |
| 1070 |
* 1. Take mapping->invalidate_lock to block dax faults. |
| 1071 |
* 2. Take fi->dax->sem to protect interval tree and also to make sure |
| 1072 |
* read/write can not reuse a dmap which we might be freeing. |
| 1073 |
*/ |
| 1074 |
static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd, |
| 1075 |
struct inode *inode, |
| 1076 |
unsigned long start_idx, |
| 1077 |
unsigned long end_idx) |
| 1078 |
{ |
| 1079 |
int ret; |
| 1080 |
struct fuse_inode *fi = get_fuse_inode(inode); |
| 1081 |
loff_t dmap_start = start_idx << FUSE_DAX_SHIFT; |
| 1082 |
loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1; |
| 1083 |
|
| 1084 |
filemap_invalidate_lock(inode->i_mapping); |
| 1085 |
ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end); |
| 1086 |
if (ret) { |
| 1087 |
pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n", |
| 1088 |
ret); |
| 1089 |
goto out_mmap_sem; |
| 1090 |
} |
| 1091 |
|
| 1092 |
down_write(&fi->dax->sem); |
| 1093 |
ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx); |
| 1094 |
up_write(&fi->dax->sem); |
| 1095 |
out_mmap_sem: |
| 1096 |
filemap_invalidate_unlock(inode->i_mapping); |
| 1097 |
return ret; |
| 1098 |
} |
| 1099 |
|
| 1100 |
static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd, |
| 1101 |
unsigned long nr_to_free) |
| 1102 |
{ |
| 1103 |
struct fuse_dax_mapping *dmap, *pos, *temp; |
| 1104 |
int ret, nr_freed = 0; |
| 1105 |
unsigned long start_idx = 0, end_idx = 0; |
| 1106 |
struct inode *inode = NULL; |
| 1107 |
|
| 1108 |
/* Pick first busy range and free it for now*/ |
| 1109 |
while (1) { |
| 1110 |
if (nr_freed >= nr_to_free) |
| 1111 |
break; |
| 1112 |
|
| 1113 |
dmap = NULL; |
| 1114 |
spin_lock(&fcd->lock); |
| 1115 |
|
| 1116 |
if (!fcd->nr_busy_ranges) { |
| 1117 |
spin_unlock(&fcd->lock); |
| 1118 |
return 0; |
| 1119 |
} |
| 1120 |
|
| 1121 |
list_for_each_entry_safe(pos, temp, &fcd->busy_ranges, |
| 1122 |
busy_list) { |
| 1123 |
/* skip this range if it's in use. */ |
| 1124 |
if (refcount_read(&pos->refcnt) > 1) |
| 1125 |
continue; |
| 1126 |
|
| 1127 |
inode = igrab(pos->inode); |
| 1128 |
/* |
| 1129 |
* This inode is going away. That will free |
| 1130 |
* up all the ranges anyway, continue to |
| 1131 |
* next range. |
| 1132 |
*/ |
| 1133 |
if (!inode) |
| 1134 |
continue; |
| 1135 |
/* |
| 1136 |
* Take this element off list and add it tail. If |
| 1137 |
* this element can't be freed, it will help with |
| 1138 |
* selecting new element in next iteration of loop. |
| 1139 |
*/ |
| 1140 |
dmap = pos; |
| 1141 |
list_move_tail(&dmap->busy_list, &fcd->busy_ranges); |
| 1142 |
start_idx = end_idx = dmap->itn.start; |
| 1143 |
break; |
| 1144 |
} |
| 1145 |
spin_unlock(&fcd->lock); |
| 1146 |
if (!dmap) |
| 1147 |
return 0; |
| 1148 |
|
| 1149 |
ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx); |
| 1150 |
iput(inode); |
| 1151 |
if (ret) |
| 1152 |
return ret; |
| 1153 |
nr_freed++; |
| 1154 |
} |
| 1155 |
return 0; |
| 1156 |
} |
| 1157 |
|
| 1158 |
static void fuse_dax_free_mem_worker(struct work_struct *work) |
| 1159 |
{ |
| 1160 |
int ret; |
| 1161 |
struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax, |
| 1162 |
free_work.work); |
| 1163 |
ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK); |
| 1164 |
if (ret) { |
| 1165 |
pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n", |
| 1166 |
ret); |
| 1167 |
} |
| 1168 |
|
| 1169 |
/* If number of free ranges are still below threshold, requeue */ |
| 1170 |
kick_dmap_free_worker(fcd, 1); |
| 1171 |
} |
| 1172 |
|
| 1173 |
static void fuse_free_dax_mem_ranges(struct list_head *mem_list) |
| 1174 |
{ |
| 1175 |
struct fuse_dax_mapping *range, *temp; |
| 1176 |
|
| 1177 |
/* Free All allocated elements */ |
| 1178 |
list_for_each_entry_safe(range, temp, mem_list, list) { |
| 1179 |
list_del(&range->list); |
| 1180 |
if (!list_empty(&range->busy_list)) |
| 1181 |
list_del(&range->busy_list); |
| 1182 |
kfree(range); |
| 1183 |
} |
| 1184 |
} |
| 1185 |
|
| 1186 |
void fuse_dax_conn_free(struct fuse_conn *fc) |
| 1187 |
{ |
| 1188 |
if (fc->dax) { |
| 1189 |
fuse_free_dax_mem_ranges(&fc->dax->free_ranges); |
| 1190 |
kfree(fc->dax); |
| 1191 |
fc->dax = NULL; |
| 1192 |
} |
| 1193 |
} |
| 1194 |
|
| 1195 |
static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd) |
| 1196 |
{ |
| 1197 |
long nr_pages, nr_ranges; |
| 1198 |
struct fuse_dax_mapping *range; |
| 1199 |
int ret, id; |
| 1200 |
size_t dax_size = -1; |
| 1201 |
unsigned long i; |
| 1202 |
|
| 1203 |
init_waitqueue_head(&fcd->range_waitq); |
| 1204 |
INIT_LIST_HEAD(&fcd->free_ranges); |
| 1205 |
INIT_LIST_HEAD(&fcd->busy_ranges); |
| 1206 |
INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker); |
| 1207 |
|
| 1208 |
id = dax_read_lock(); |
| 1209 |
nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size), |
| 1210 |
DAX_ACCESS, NULL, NULL); |
| 1211 |
dax_read_unlock(id); |
| 1212 |
if (nr_pages < 0) { |
| 1213 |
pr_debug("dax_direct_access() returned %ld\n", nr_pages); |
| 1214 |
return nr_pages; |
| 1215 |
} |
| 1216 |
|
| 1217 |
nr_ranges = nr_pages/FUSE_DAX_PAGES; |
| 1218 |
pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n", |
| 1219 |
__func__, nr_pages, nr_ranges); |
| 1220 |
|
| 1221 |
for (i = 0; i < nr_ranges; i++) { |
| 1222 |
range = kzalloc_obj(struct fuse_dax_mapping); |
| 1223 |
ret = -ENOMEM; |
| 1224 |
if (!range) |
| 1225 |
goto out_err; |
| 1226 |
|
| 1227 |
/* TODO: This offset only works if virtio-fs driver is not |
| 1228 |
* having some memory hidden at the beginning. This needs |
| 1229 |
* better handling |
| 1230 |
*/ |
| 1231 |
range->window_offset = i * FUSE_DAX_SZ; |
| 1232 |
range->length = FUSE_DAX_SZ; |
| 1233 |
INIT_LIST_HEAD(&range->busy_list); |
| 1234 |
refcount_set(&range->refcnt, 1); |
| 1235 |
list_add_tail(&range->list, &fcd->free_ranges); |
| 1236 |
} |
| 1237 |
|
| 1238 |
fcd->nr_free_ranges = nr_ranges; |
| 1239 |
fcd->nr_ranges = nr_ranges; |
| 1240 |
return 0; |
| 1241 |
out_err: |
| 1242 |
/* Free All allocated elements */ |
| 1243 |
fuse_free_dax_mem_ranges(&fcd->free_ranges); |
| 1244 |
return ret; |
| 1245 |
} |
| 1246 |
|
| 1247 |
int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode, |
| 1248 |
struct dax_device *dax_dev) |
| 1249 |
{ |
| 1250 |
struct fuse_conn_dax *fcd; |
| 1251 |
int err; |
| 1252 |
|
| 1253 |
fc->dax_mode = dax_mode; |
| 1254 |
|
| 1255 |
if (!dax_dev) |
| 1256 |
return 0; |
| 1257 |
|
| 1258 |
fcd = kzalloc_obj(*fcd); |
| 1259 |
if (!fcd) |
| 1260 |
return -ENOMEM; |
| 1261 |
|
| 1262 |
spin_lock_init(&fcd->lock); |
| 1263 |
fcd->dev = dax_dev; |
| 1264 |
err = fuse_dax_mem_range_init(fcd); |
| 1265 |
if (err) { |
| 1266 |
kfree(fcd); |
| 1267 |
return err; |
| 1268 |
} |
| 1269 |
|
| 1270 |
fc->dax = fcd; |
| 1271 |
return 0; |
| 1272 |
} |
| 1273 |
|
| 1274 |
bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) |
| 1275 |
{ |
| 1276 |
struct fuse_conn *fc = get_fuse_conn_super(sb); |
| 1277 |
|
| 1278 |
fi->dax = NULL; |
| 1279 |
if (fc->dax) { |
| 1280 |
fi->dax = kzalloc_obj(*fi->dax, GFP_KERNEL_ACCOUNT); |
| 1281 |
if (!fi->dax) |
| 1282 |
return false; |
| 1283 |
|
| 1284 |
init_rwsem(&fi->dax->sem); |
| 1285 |
fi->dax->tree = RB_ROOT_CACHED; |
| 1286 |
} |
| 1287 |
|
| 1288 |
return true; |
| 1289 |
} |
| 1290 |
|
| 1291 |
static const struct address_space_operations fuse_dax_file_aops = { |
| 1292 |
.direct_IO = noop_direct_IO, |
| 1293 |
.dirty_folio = noop_dirty_folio, |
| 1294 |
}; |
| 1295 |
|
| 1296 |
static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags) |
| 1297 |
{ |
| 1298 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1299 |
enum fuse_dax_mode dax_mode = fc->dax_mode; |
| 1300 |
|
| 1301 |
if (dax_mode == FUSE_DAX_NEVER) |
| 1302 |
return false; |
| 1303 |
|
| 1304 |
/* |
| 1305 |
* fc->dax may be NULL in 'inode' mode when filesystem device doesn't |
| 1306 |
* support DAX, in which case it will silently fallback to 'never' mode. |
| 1307 |
*/ |
| 1308 |
if (!fc->dax) |
| 1309 |
return false; |
| 1310 |
|
| 1311 |
if (dax_mode == FUSE_DAX_ALWAYS) |
| 1312 |
return true; |
| 1313 |
|
| 1314 |
/* dax_mode is FUSE_DAX_INODE* */ |
| 1315 |
return fc->inode_dax && (flags & FUSE_ATTR_DAX); |
| 1316 |
} |
| 1317 |
|
| 1318 |
void fuse_dax_inode_init(struct inode *inode, unsigned int flags) |
| 1319 |
{ |
| 1320 |
if (!fuse_should_enable_dax(inode, flags)) |
| 1321 |
return; |
| 1322 |
|
| 1323 |
inode->i_flags |= S_DAX; |
| 1324 |
inode->i_data.a_ops = &fuse_dax_file_aops; |
| 1325 |
} |
| 1326 |
|
| 1327 |
void fuse_dax_dontcache(struct inode *inode, unsigned int flags) |
| 1328 |
{ |
| 1329 |
struct fuse_conn *fc = get_fuse_conn(inode); |
| 1330 |
|
| 1331 |
if (fuse_is_inode_dax_mode(fc->dax_mode) && |
| 1332 |
((bool) IS_DAX(inode) != (bool) (flags & FUSE_ATTR_DAX))) |
| 1333 |
d_mark_dontcache(inode); |
| 1334 |
} |
| 1335 |
|
| 1336 |
bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment) |
| 1337 |
{ |
| 1338 |
if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) { |
| 1339 |
pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n", |
| 1340 |
map_alignment, FUSE_DAX_SZ); |
| 1341 |
return false; |
| 1342 |
} |
| 1343 |
return true; |
| 1344 |
} |
| 1345 |
|
| 1346 |
void fuse_dax_cancel_work(struct fuse_conn *fc) |
| 1347 |
{ |
| 1348 |
struct fuse_conn_dax *fcd = fc->dax; |
| 1349 |
|
| 1350 |
if (fcd) |
| 1351 |
cancel_delayed_work_sync(&fcd->free_work); |
| 1352 |
|
| 1353 |
} |
| 1354 |
EXPORT_SYMBOL_GPL(fuse_dax_cancel_work); |