projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
fs: port privilege checking helpers to mnt_idmap
[linux-block.git] / fs / fuse / file.c
CommitLineData
b6aeaded
MS		 1/*
2 FUSE: Filesystem in Userspace
1729a16c 3 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
b6aeaded
MS		 4
5 This program can be distributed under the terms of the GNU GPL.
6 See the file COPYING.
7*/
8
9#include "fuse_i.h"
10
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/kernel.h>
e8edc6e0 14#include <linux/sched.h>
7a36094d 15#include <linux/sched/signal.h>
08cbf542 16#include <linux/module.h>
478e0841 17#include <linux/swap.h>
3634a632 18#include <linux/falloc.h>
e2e40f2c 19#include <linux/uio.h>
31070f6c 20#include <linux/fs.h>
b6aeaded 21
b9d54c6f
MS		 22static int fuse_send_open(struct fuse_mount *fm, u64 nodeid,
23 unsigned int open_flags, int opcode,
24 struct fuse_open_out *outargp)
b6aeaded 25{
b6aeaded 26 struct fuse_open_in inarg;
7078187a 27 FUSE_ARGS(args);
fd72faac
MS		 28
29 memset(&inarg, 0, sizeof(inarg));
b9d54c6f 30 inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
fcee216b 31 if (!fm->fc->atomic_o_trunc)
6ff958ed 32 inarg.flags &= ~O_TRUNC;
643a666a
VG		 33
34 if (fm->fc->handle_killpriv_v2 &&
35 (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) {
36 inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID;
37 }
38
d5b48543
MS		 39 args.opcode = opcode;
40 args.nodeid = nodeid;
41 args.in_numargs = 1;
42 args.in_args[0].size = sizeof(inarg);
43 args.in_args[0].value = &inarg;
44 args.out_numargs = 1;
45 args.out_args[0].size = sizeof(*outargp);
46 args.out_args[0].value = outargp;
fd72faac 47
fcee216b 48 return fuse_simple_request(fm, &args);
fd72faac
MS		 49}
50
4cb54866
MS		 51struct fuse_release_args {
52 struct fuse_args args;
53 struct fuse_release_in inarg;
54 struct inode *inode;
55};
56
fcee216b 57struct fuse_file *fuse_file_alloc(struct fuse_mount *fm)
fd72faac
MS		 58{
59 struct fuse_file *ff;
6b2db28a 60
dc69e98c 61 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
6b2db28a
TH		 62 if (unlikely(!ff))
63 return NULL;
64
fcee216b 65 ff->fm = fm;
dc69e98c
KK		 66 ff->release_args = kzalloc(sizeof(*ff->release_args),
67 GFP_KERNEL_ACCOUNT);
4cb54866 68 if (!ff->release_args) {
6b2db28a
TH		 69 kfree(ff);
70 return NULL;
fd72faac 71 }
6b2db28a
TH		 72
73 INIT_LIST_HEAD(&ff->write_entry);
5d7bc7e8 74 mutex_init(&ff->readdir.lock);
4e8c2eb5 75 refcount_set(&ff->count, 1);
6b2db28a
TH		 76 RB_CLEAR_NODE(&ff->polled_node);
77 init_waitqueue_head(&ff->poll_wait);
78
fcee216b 79 ff->kh = atomic64_inc_return(&fm->fc->khctr);
6b2db28a 80
fd72faac
MS		 81 return ff;
82}
83
84void fuse_file_free(struct fuse_file *ff)
85{
4cb54866 86 kfree(ff->release_args);
5d7bc7e8 87 mutex_destroy(&ff->readdir.lock);
fd72faac
MS		 88 kfree(ff);
89}
90
267d8444 91static struct fuse_file *fuse_file_get(struct fuse_file *ff)
c756e0a4 92{
4e8c2eb5 93 refcount_inc(&ff->count);
c756e0a4
MS		 94 return ff;
95}
96
fcee216b 97static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
4cb54866 98 int error)
819c4b3b 99{
4cb54866
MS		 100 struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
101
102 iput(ra->inode);
103 kfree(ra);
819c4b3b
MS		 104}
105
2e64ff15 106static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir)
c756e0a4 107{
4e8c2eb5 108 if (refcount_dec_and_test(&ff->count)) {
4cb54866 109 struct fuse_args *args = &ff->release_args->args;
8b0797a4 110
fcee216b 111 if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) {
4cb54866 112 /* Do nothing when client does not implement 'open' */
fcee216b 113 fuse_release_end(ff->fm, args, 0);
7678ac50 114 } else if (sync) {
fcee216b
MR		 115 fuse_simple_request(ff->fm, args);
116 fuse_release_end(ff->fm, args, 0);
5a18ec17 117 } else {
4cb54866 118 args->end = fuse_release_end;
fcee216b 119 if (fuse_simple_background(ff->fm, args,
4cb54866 120 GFP_KERNEL | __GFP_NOFAIL))
fcee216b 121 fuse_release_end(ff->fm, args, -ENOTCONN);
5a18ec17 122 }
c756e0a4
MS		 123 kfree(ff);
124 }
125}
126
b9d54c6f
MS		 127struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
128 unsigned int open_flags, bool isdir)
91fe96b4 129{
fcee216b 130 struct fuse_conn *fc = fm->fc;
91fe96b4 131 struct fuse_file *ff;
91fe96b4
MS		 132 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
133
fcee216b 134 ff = fuse_file_alloc(fm);
91fe96b4 135 if (!ff)
b9d54c6f 136 return ERR_PTR(-ENOMEM);
91fe96b4 137
7678ac50 138 ff->fh = 0;
fabf7e02
CA		 139 /* Default for no-open */
140 ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
d9a9ea94 141 if (isdir ? !fc->no_opendir : !fc->no_open) {
7678ac50
AG		 142 struct fuse_open_out outarg;
143 int err;
144
b9d54c6f 145 err = fuse_send_open(fm, nodeid, open_flags, opcode, &outarg);
7678ac50
AG		 146 if (!err) {
147 ff->fh = outarg.fh;
148 ff->open_flags = outarg.open_flags;
149
d9a9ea94 150 } else if (err != -ENOSYS) {
7678ac50 151 fuse_file_free(ff);
b9d54c6f 152 return ERR_PTR(err);
7678ac50 153 } else {
d9a9ea94
CA		 154 if (isdir)
155 fc->no_opendir = 1;
156 else
157 fc->no_open = 1;
7678ac50 158 }
91fe96b4
MS		 159 }
160
161 if (isdir)
7678ac50 162 ff->open_flags &= ~FOPEN_DIRECT_IO;
91fe96b4 163
91fe96b4 164 ff->nodeid = nodeid;
91fe96b4 165
b9d54c6f
MS		 166 return ff;
167}
168
169int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
170 bool isdir)
171{
172 struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir);
173
174 if (!IS_ERR(ff))
175 file->private_data = ff;
176
177 return PTR_ERR_OR_ZERO(ff);
91fe96b4 178}
08cbf542 179EXPORT_SYMBOL_GPL(fuse_do_open);
91fe96b4 180
650b22b9
PE		 181static void fuse_link_write_file(struct file *file)
182{
183 struct inode *inode = file_inode(file);
650b22b9
PE		 184 struct fuse_inode *fi = get_fuse_inode(inode);
185 struct fuse_file *ff = file->private_data;
186 /*
187 * file may be written through mmap, so chain it onto the
188 * inodes's write_file list
189 */
f15ecfef 190 spin_lock(&fi->lock);
650b22b9
PE		 191 if (list_empty(&ff->write_entry))
192 list_add(&ff->write_entry, &fi->write_files);
f15ecfef 193 spin_unlock(&fi->lock);
650b22b9
PE		 194}
195
c7b7143c 196void fuse_finish_open(struct inode *inode, struct file *file)
fd72faac 197{
c7b7143c 198 struct fuse_file *ff = file->private_data;
a0822c55 199 struct fuse_conn *fc = get_fuse_conn(inode);
c7b7143c 200
bbd84f33
KS		 201 if (ff->open_flags & FOPEN_STREAM)
202 stream_open(inode, file);
203 else if (ff->open_flags & FOPEN_NONSEEKABLE)
a7c1b990 204 nonseekable_open(inode, file);
76224355 205
a0822c55
KS		 206 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
207 struct fuse_inode *fi = get_fuse_inode(inode);
208
f15ecfef 209 spin_lock(&fi->lock);
4510d86f 210 fi->attr_version = atomic64_inc_return(&fc->attr_version);
a0822c55 211 i_size_write(inode, 0);
f15ecfef 212 spin_unlock(&fi->lock);
20235b43 213 file_update_time(file);
fa5eee57 214 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
a0822c55 215 }
4d99ff8f
PE		 216 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
217 fuse_link_write_file(file);
fd72faac
MS		 218}
219
91fe96b4 220int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
fd72faac 221{
fcee216b
MR		 222 struct fuse_mount *fm = get_fuse_mount(inode);
223 struct fuse_conn *fc = fm->fc;
b6aeaded 224 int err;
e4648309 225 bool is_wb_truncate = (file->f_flags & O_TRUNC) &&
75caeecd
MP		 226 fc->atomic_o_trunc &&
227 fc->writeback_cache;
6ae330ca
VG		 228 bool dax_truncate = (file->f_flags & O_TRUNC) &&
229 fc->atomic_o_trunc && FUSE_IS_DAX(inode);
b6aeaded 230
5d069dbe
MS		 231 if (fuse_is_bad(inode))
232 return -EIO;
233
b6aeaded
MS		 234 err = generic_file_open(inode, file);
235 if (err)
236 return err;
237
2fdbb8dd 238 if (is_wb_truncate || dax_truncate)
5955102c 239 inode_lock(inode);
75caeecd 240
6ae330ca 241 if (dax_truncate) {
8bcbbe9c 242 filemap_invalidate_lock(inode->i_mapping);
6ae330ca
VG		 243 err = fuse_dax_break_layouts(inode, 0, 0);
244 if (err)
2fdbb8dd 245 goto out_inode_unlock;
6ae330ca 246 }
b6aeaded 247
2fdbb8dd
MS		 248 if (is_wb_truncate || dax_truncate)
249 fuse_set_nowrite(inode);
250
fcee216b 251 err = fuse_do_open(fm, get_node_id(inode), file, isdir);
75caeecd
MP		 252 if (!err)
253 fuse_finish_open(inode, file);
91fe96b4 254
2fdbb8dd
MS		 255 if (is_wb_truncate || dax_truncate)
256 fuse_release_nowrite(inode);
257 if (!err) {
258 struct fuse_file *ff = file->private_data;
259
260 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC))
261 truncate_pagecache(inode, 0);
262 else if (!(ff->open_flags & FOPEN_KEEP_CACHE))
263 invalidate_inode_pages2(inode->i_mapping);
264 }
6ae330ca 265 if (dax_truncate)
8bcbbe9c 266 filemap_invalidate_unlock(inode->i_mapping);
2fdbb8dd
MS		 267out_inode_unlock:
268 if (is_wb_truncate || dax_truncate)
5955102c 269 inode_unlock(inode);
75caeecd
MP		 270
271 return err;
b6aeaded
MS		 272}
273
ebf84d0c 274static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
54d601cb 275 unsigned int flags, int opcode)
64c6d8ed 276{
fcee216b 277 struct fuse_conn *fc = ff->fm->fc;
4cb54866 278 struct fuse_release_args *ra = ff->release_args;
b6aeaded 279
f15ecfef
KT		 280 /* Inode is NULL on error path of fuse_create_open() */
281 if (likely(fi)) {
282 spin_lock(&fi->lock);
283 list_del(&ff->write_entry);
284 spin_unlock(&fi->lock);
285 }
8b0797a4 286 spin_lock(&fc->lock);
8b0797a4
MS		 287 if (!RB_EMPTY_NODE(&ff->polled_node))
288 rb_erase(&ff->polled_node, &fc->polled_files);
289 spin_unlock(&fc->lock);
290
357ccf2b 291 wake_up_interruptible_all(&ff->poll_wait);
8b0797a4 292
4cb54866
MS		 293 ra->inarg.fh = ff->fh;
294 ra->inarg.flags = flags;
295 ra->args.in_numargs = 1;
296 ra->args.in_args[0].size = sizeof(struct fuse_release_in);
297 ra->args.in_args[0].value = &ra->inarg;
298 ra->args.opcode = opcode;
299 ra->args.nodeid = ff->nodeid;
300 ra->args.force = true;
301 ra->args.nocreds = true;
fd72faac
MS		 302}
303
b9d54c6f
MS		 304void fuse_file_release(struct inode *inode, struct fuse_file *ff,
305 unsigned int open_flags, fl_owner_t id, bool isdir)
fd72faac 306{
b9d54c6f 307 struct fuse_inode *fi = get_fuse_inode(inode);
4cb54866 308 struct fuse_release_args *ra = ff->release_args;
2e64ff15 309 int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
6b2db28a 310
b9d54c6f 311 fuse_prepare_release(fi, ff, open_flags, opcode);
6b2db28a 312
37fb3a30 313 if (ff->flock) {
4cb54866 314 ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
b9d54c6f 315 ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id);
37fb3a30 316 }
baebccbe 317 /* Hold inode until release is finished */
b9d54c6f 318 ra->inode = igrab(inode);
6b2db28a 319
6b2db28a
TH		 320 /*
321 * Normally this will send the RELEASE request, however if
322 * some asynchronous READ or WRITE requests are outstanding,
323 * the sending will be delayed.
5a18ec17
MS		 324 *
325 * Make the release synchronous if this is a fuseblk mount,
326 * synchronous RELEASE is allowed (and desirable) in this case
327 * because the server can be trusted not to screw up.
6b2db28a 328 */
fcee216b 329 fuse_file_put(ff, ff->fm->fc->destroy, isdir);
b6aeaded
MS		 330}
331
b9d54c6f
MS		 332void fuse_release_common(struct file *file, bool isdir)
333{
334 fuse_file_release(file_inode(file), file->private_data, file->f_flags,
335 (fl_owner_t) file, isdir);
336}
337
04730fef
MS		 338static int fuse_open(struct inode *inode, struct file *file)
339{
91fe96b4 340 return fuse_open_common(inode, file, false);
04730fef
MS		 341}
342
343static int fuse_release(struct inode *inode, struct file *file)
344{
035ff33c
MS		 345 struct fuse_conn *fc = get_fuse_conn(inode);
346
347 /*
348 * Dirty pages might remain despite write_inode_now() call from
349 * fuse_flush() due to writes racing with the close.
350 */
351 if (fc->writeback_cache)
352 write_inode_now(inode, 1);
353
2e64ff15 354 fuse_release_common(file, false);
8b0797a4
MS		 355
356 /* return value is ignored by VFS */
357 return 0;
358}
359
54d601cb
MS		 360void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff,
361 unsigned int flags)
8b0797a4 362{
4e8c2eb5 363 WARN_ON(refcount_read(&ff->count) > 1);
ebf84d0c 364 fuse_prepare_release(fi, ff, flags, FUSE_RELEASE);
267d8444
MS		 365 /*
366 * iput(NULL) is a no-op and since the refcount is 1 and everything's
367 * synchronous, we are fine with not doing igrab() here"
368 */
2e64ff15 369 fuse_file_put(ff, true, false);
04730fef 370}
08cbf542 371EXPORT_SYMBOL_GPL(fuse_sync_release);
04730fef 372
71421259 373/*
9c8ef561
MS		 374 * Scramble the ID space with XTEA, so that the value of the files_struct
375 * pointer is not exposed to userspace.
71421259 376 */
f3332114 377u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
71421259 378{
9c8ef561
MS		 379 u32 *k = fc->scramble_key;
380 u64 v = (unsigned long) id;
381 u32 v0 = v;
382 u32 v1 = v >> 32;
383 u32 sum = 0;
384 int i;
385
386 for (i = 0; i < 32; i++) {
387 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
388 sum += 0x9E3779B9;
389 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
390 }
391
392 return (u64) v0 + ((u64) v1 << 32);
71421259
MS		 393}
394
33826ebb
MS		 395struct fuse_writepage_args {
396 struct fuse_io_args ia;
6b2fb799 397 struct rb_node writepages_entry;
33826ebb
MS		 398 struct list_head queue_entry;
399 struct fuse_writepage_args *next;
400 struct inode *inode;
660585b5 401 struct fuse_sync_bucket *bucket;
33826ebb
MS		 402};
403
404static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi,
2fe93bd4
MS		 405 pgoff_t idx_from, pgoff_t idx_to)
406{
6b2fb799
MP		 407 struct rb_node *n;
408
409 n = fi->writepages.rb_node;
2fe93bd4 410
6b2fb799
MP		 411 while (n) {
412 struct fuse_writepage_args *wpa;
2fe93bd4
MS		 413 pgoff_t curr_index;
414
6b2fb799 415 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);
33826ebb
MS		 416 WARN_ON(get_fuse_inode(wpa->inode) != fi);
417 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;
6b2fb799
MP		 418 if (idx_from >= curr_index + wpa->ia.ap.num_pages)
419 n = n->rb_right;
420 else if (idx_to < curr_index)
421 n = n->rb_left;
422 else
33826ebb 423 return wpa;
2fe93bd4
MS		 424 }
425 return NULL;
426}
427
3be5a52b 428/*
ea8cd333 429 * Check if any page in a range is under writeback
3be5a52b
MS		 430 *
431 * This is currently done by walking the list of writepage requests
432 * for the inode, which can be pretty inefficient.
433 */
ea8cd333
PE		 434static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
435 pgoff_t idx_to)
3be5a52b 436{
3be5a52b 437 struct fuse_inode *fi = get_fuse_inode(inode);
2fe93bd4 438 bool found;
3be5a52b 439
f15ecfef 440 spin_lock(&fi->lock);
2fe93bd4 441 found = fuse_find_writeback(fi, idx_from, idx_to);
f15ecfef 442 spin_unlock(&fi->lock);
3be5a52b
MS		 443
444 return found;
445}
446
ea8cd333
PE		 447static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
448{
449 return fuse_range_is_writeback(inode, index, index);
450}
451
3be5a52b
MS		 452/*
453 * Wait for page writeback to be completed.
454 *
455 * Since fuse doesn't rely on the VM writeback tracking, this has to
456 * use some other means.
457 */
17b2cbe2 458static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
3be5a52b
MS		 459{
460 struct fuse_inode *fi = get_fuse_inode(inode);
461
462 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
3be5a52b
MS		 463}
464
fe38d7df
MP		 465/*
466 * Wait for all pending writepages on the inode to finish.
467 *
468 * This is currently done by blocking further writes with FUSE_NOWRITE
469 * and waiting for all sent writes to complete.
470 *
471 * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
472 * could conflict with truncation.
473 */
474static void fuse_sync_writes(struct inode *inode)
475{
476 fuse_set_nowrite(inode);
477 fuse_release_nowrite(inode);
478}
479
75e1fcc0 480static int fuse_flush(struct file *file, fl_owner_t id)
b6aeaded 481{
6131ffaa 482 struct inode *inode = file_inode(file);
fcee216b 483 struct fuse_mount *fm = get_fuse_mount(inode);
b6aeaded 484 struct fuse_file *ff = file->private_data;
b6aeaded 485 struct fuse_flush_in inarg;
c500ebaa 486 FUSE_ARGS(args);
b6aeaded
MS		 487 int err;
488
5d069dbe 489 if (fuse_is_bad(inode))
248d86e8
MS		 490 return -EIO;
491
a390ccb3
AG		 492 if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache)
493 return 0;
494
1e18bda8 495 err = write_inode_now(inode, 1);
fe38d7df
MP		 496 if (err)
497 return err;
498
5955102c 499 inode_lock(inode);
fe38d7df 500 fuse_sync_writes(inode);
5955102c 501 inode_unlock(inode);
fe38d7df 502
4a7f4e88 503 err = filemap_check_errors(file->f_mapping);
9ebce595
MP		 504 if (err)
505 return err;
506
614c026e 507 err = 0;
fcee216b 508 if (fm->fc->no_flush)
614c026e
MS		 509 goto inval_attr_out;
510
b6aeaded
MS		 511 memset(&inarg, 0, sizeof(inarg));
512 inarg.fh = ff->fh;
fcee216b 513 inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
c500ebaa
MS		 514 args.opcode = FUSE_FLUSH;
515 args.nodeid = get_node_id(inode);
516 args.in_numargs = 1;
517 args.in_args[0].size = sizeof(inarg);
518 args.in_args[0].value = &inarg;
519 args.force = true;
520
fcee216b 521 err = fuse_simple_request(fm, &args);
b6aeaded 522 if (err == -ENOSYS) {
fcee216b 523 fm->fc->no_flush = 1;
b6aeaded
MS		 524 err = 0;
525 }
cf576c58
EG		 526
527inval_attr_out:
528 /*
529 * In memory i_blocks is not maintained by fuse, if writeback cache is
530 * enabled, i_blocks from cached attr may not be accurate.
531 */
fcee216b 532 if (!err && fm->fc->writeback_cache)
fa5eee57 533 fuse_invalidate_attr_mask(inode, STATX_BLOCKS);
b6aeaded
MS		 534 return err;
535}
536
02c24a82 537int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
a9c2d1e8 538 int datasync, int opcode)
b6aeaded 539{
7ea80859 540 struct inode *inode = file->f_mapping->host;
fcee216b 541 struct fuse_mount *fm = get_fuse_mount(inode);
b6aeaded 542 struct fuse_file *ff = file->private_data;
7078187a 543 FUSE_ARGS(args);
b6aeaded 544 struct fuse_fsync_in inarg;
a9c2d1e8
MS		 545
546 memset(&inarg, 0, sizeof(inarg));
547 inarg.fh = ff->fh;
154603fe 548 inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
d5b48543
MS		 549 args.opcode = opcode;
550 args.nodeid = get_node_id(inode);
551 args.in_numargs = 1;
552 args.in_args[0].size = sizeof(inarg);
553 args.in_args[0].value = &inarg;
fcee216b 554 return fuse_simple_request(fm, &args);
a9c2d1e8
MS		 555}
556
557static int fuse_fsync(struct file *file, loff_t start, loff_t end,
558 int datasync)
559{
560 struct inode *inode = file->f_mapping->host;
561 struct fuse_conn *fc = get_fuse_conn(inode);
b6aeaded
MS		 562 int err;
563
5d069dbe 564 if (fuse_is_bad(inode))
248d86e8
MS		 565 return -EIO;
566
5955102c 567 inode_lock(inode);
02c24a82 568
3be5a52b
MS		 569 /*
570 * Start writeback against all dirty pages of the inode, then
571 * wait for all outstanding writes, before sending the FSYNC
572 * request.
573 */
7e51fe1d 574 err = file_write_and_wait_range(file, start, end);
3be5a52b 575 if (err)
02c24a82 576 goto out;
3be5a52b
MS		 577
578 fuse_sync_writes(inode);
ac7f052b
AK		 579
580 /*
581 * Due to implementation of fuse writeback
7e51fe1d 582 * file_write_and_wait_range() does not catch errors.
ac7f052b
AK		 583 * We have to do this directly after fuse_sync_writes()
584 */
7e51fe1d 585 err = file_check_and_advance_wb_err(file);
ac7f052b
AK		 586 if (err)
587 goto out;
588
1e18bda8
MS		 589 err = sync_inode_metadata(inode, 1);
590 if (err)
591 goto out;
3be5a52b 592
a9c2d1e8 593 if (fc->no_fsync)
22401e7b 594 goto out;
b0aa7606 595
a9c2d1e8 596 err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
b6aeaded 597 if (err == -ENOSYS) {
a9c2d1e8 598 fc->no_fsync = 1;
b6aeaded
MS		 599 err = 0;
600 }
02c24a82 601out:
5955102c 602 inode_unlock(inode);
b6aeaded 603
a9c2d1e8 604 return err;
82547981
MS		 605}
606
00793ca5
MS		 607void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
608 size_t count, int opcode)
609{
610 struct fuse_file *ff = file->private_data;
611 struct fuse_args *args = &ia->ap.args;
612
613 ia->read.in.fh = ff->fh;
614 ia->read.in.offset = pos;
615 ia->read.in.size = count;
616 ia->read.in.flags = file->f_flags;
617 args->opcode = opcode;
618 args->nodeid = ff->nodeid;
619 args->in_numargs = 1;
620 args->in_args[0].size = sizeof(ia->read.in);
621 args->in_args[0].value = &ia->read.in;
622 args->out_argvar = true;
623 args->out_numargs = 1;
624 args->out_args[0].size = count;
625}
626
45ac96ed
MS		 627static void fuse_release_user_pages(struct fuse_args_pages *ap,
628 bool should_dirty)
187c5c36 629{
45ac96ed 630 unsigned int i;
187c5c36 631
45ac96ed 632 for (i = 0; i < ap->num_pages; i++) {
8fba54ae 633 if (should_dirty)
45ac96ed
MS		 634 set_page_dirty_lock(ap->pages[i]);
635 put_page(ap->pages[i]);
187c5c36
MP		 636 }
637}
638
744742d6
SF		 639static void fuse_io_release(struct kref *kref)
640{
641 kfree(container_of(kref, struct fuse_io_priv, refcnt));
642}
643
9d5722b7
CH		 644static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
645{
646 if (io->err)
647 return io->err;
648
649 if (io->bytes >= 0 && io->write)
650 return -EIO;
651
652 return io->bytes < 0 ? io->size : io->bytes;
653}
654
01e9d11a
MP		 655/**
656 * In case of short read, the caller sets 'pos' to the position of
657 * actual end of fuse request in IO request. Otherwise, if bytes_requested
658 * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
659 *
660 * An example:
c4e0cd4e 661 * User requested DIO read of 64K. It was split into two 32K fuse requests,
01e9d11a
MP		 662 * both submitted asynchronously. The first of them was ACKed by userspace as
663 * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
664 * second request was ACKed as short, e.g. only 1K was read, resulting in
665 * pos == 33K.
666 *
667 * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
668 * will be equal to the length of the longest contiguous fragment of
669 * transferred data starting from the beginning of IO request.
670 */
671static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
672{
673 int left;
674
675 spin_lock(&io->lock);
676 if (err)
677 io->err = io->err ? : err;
678 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
679 io->bytes = pos;
680
681 left = --io->reqs;
7879c4e5 682 if (!left && io->blocking)
9d5722b7 683 complete(io->done);
01e9d11a
MP		 684 spin_unlock(&io->lock);
685
7879c4e5 686 if (!left && !io->blocking) {
9d5722b7 687 ssize_t res = fuse_get_res_by_io(io);
01e9d11a 688
9d5722b7
CH		 689 if (res >= 0) {
690 struct inode *inode = file_inode(io->iocb->ki_filp);
691 struct fuse_conn *fc = get_fuse_conn(inode);
692 struct fuse_inode *fi = get_fuse_inode(inode);
01e9d11a 693
f15ecfef 694 spin_lock(&fi->lock);
4510d86f 695 fi->attr_version = atomic64_inc_return(&fc->attr_version);
f15ecfef 696 spin_unlock(&fi->lock);
01e9d11a
MP		 697 }
698
6b19b766 699 io->iocb->ki_complete(io->iocb, res);
01e9d11a 700 }
744742d6
SF		 701
702 kref_put(&io->refcnt, fuse_io_release);
01e9d11a
MP		 703}
704
45ac96ed
MS		 705static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
706 unsigned int npages)
707{
708 struct fuse_io_args *ia;
709
710 ia = kzalloc(sizeof(*ia), GFP_KERNEL);
711 if (ia) {
712 ia->io = io;
713 ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,
714 &ia->ap.descs);
715 if (!ia->ap.pages) {
716 kfree(ia);
717 ia = NULL;
718 }
719 }
720 return ia;
721}
722
723static void fuse_io_free(struct fuse_io_args *ia)
01e9d11a 724{
45ac96ed
MS		 725 kfree(ia->ap.pages);
726 kfree(ia);
727}
728
fcee216b 729static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
45ac96ed
MS		 730 int err)
731{
732 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
733 struct fuse_io_priv *io = ia->io;
01e9d11a
MP		 734 ssize_t pos = -1;
735
45ac96ed 736 fuse_release_user_pages(&ia->ap, io->should_dirty);
01e9d11a 737
45ac96ed
MS		 738 if (err) {
739 /* Nothing */
740 } else if (io->write) {
741 if (ia->write.out.size > ia->write.in.size) {
742 err = -EIO;
743 } else if (ia->write.in.size != ia->write.out.size) {
744 pos = ia->write.in.offset - io->offset +
745 ia->write.out.size;
746 }
01e9d11a 747 } else {
45ac96ed
MS		 748 u32 outsize = args->out_args[0].size;
749
750 if (ia->read.in.size != outsize)
751 pos = ia->read.in.offset - io->offset + outsize;
01e9d11a
MP		 752 }
753
45ac96ed
MS		 754 fuse_aio_complete(io, err, pos);
755 fuse_io_free(ia);
01e9d11a
MP		 756}
757
fcee216b 758static ssize_t fuse_async_req_send(struct fuse_mount *fm,
45ac96ed 759 struct fuse_io_args *ia, size_t num_bytes)
01e9d11a 760{
45ac96ed
MS		 761 ssize_t err;
762 struct fuse_io_priv *io = ia->io;
763
01e9d11a 764 spin_lock(&io->lock);
744742d6 765 kref_get(&io->refcnt);
01e9d11a
MP		 766 io->size += num_bytes;
767 io->reqs++;
768 spin_unlock(&io->lock);
769
45ac96ed 770 ia->ap.args.end = fuse_aio_complete_req;
bb737bbe 771 ia->ap.args.may_block = io->should_dirty;
fcee216b 772 err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
f1ebdeff 773 if (err)
fcee216b 774 fuse_aio_complete_req(fm, &ia->ap.args, err);
01e9d11a 775
f1ebdeff 776 return num_bytes;
01e9d11a
MP		 777}
778
45ac96ed
MS		 779static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
780 fl_owner_t owner)
04730fef 781{
45ac96ed 782 struct file *file = ia->io->iocb->ki_filp;
2106cb18 783 struct fuse_file *ff = file->private_data;
fcee216b 784 struct fuse_mount *fm = ff->fm;
f3332114 785
45ac96ed 786 fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
f3332114 787 if (owner != NULL) {
45ac96ed 788 ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
fcee216b 789 ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
f3332114 790 }
36cf66ed 791
45ac96ed 792 if (ia->io->async)
fcee216b 793 return fuse_async_req_send(fm, ia, count);
36cf66ed 794
fcee216b 795 return fuse_simple_request(fm, &ia->ap.args);
04730fef
MS		 796}
797
5c5c5e51
MS		 798static void fuse_read_update_size(struct inode *inode, loff_t size,
799 u64 attr_ver)
800{
801 struct fuse_conn *fc = get_fuse_conn(inode);
802 struct fuse_inode *fi = get_fuse_inode(inode);
803
f15ecfef 804 spin_lock(&fi->lock);
484ce657 805 if (attr_ver >= fi->attr_version && size < inode->i_size &&
06a7c3c2 806 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
4510d86f 807 fi->attr_version = atomic64_inc_return(&fc->attr_version);
5c5c5e51
MS		 808 i_size_write(inode, size);
809 }
f15ecfef 810 spin_unlock(&fi->lock);
5c5c5e51
MS		 811}
812
a0d45d84 813static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
134831e3 814 struct fuse_args_pages *ap)
a92adc82 815{
8373200b
PE		 816 struct fuse_conn *fc = get_fuse_conn(inode);
817
a73d47f5
MS		 818 /*
819 * If writeback_cache is enabled, a short read means there's a hole in
820 * the file. Some data after the hole is in page cache, but has not
821 * reached the client fs yet. So the hole is not present there.
822 */
823 if (!fc->writeback_cache) {
134831e3 824 loff_t pos = page_offset(ap->pages[0]) + num_read;
8373200b
PE		 825 fuse_read_update_size(inode, pos, attr_ver);
826 }
a92adc82
PE		 827}
828
482fce55 829static int fuse_do_readpage(struct file *file, struct page *page)
b6aeaded
MS		 830{
831 struct inode *inode = page->mapping->host;
fcee216b 832 struct fuse_mount *fm = get_fuse_mount(inode);
5c5c5e51 833 loff_t pos = page_offset(page);
00793ca5
MS		 834 struct fuse_page_desc desc = { .length = PAGE_SIZE };
835 struct fuse_io_args ia = {
836 .ap.args.page_zeroing = true,
837 .ap.args.out_pages = true,
838 .ap.num_pages = 1,
839 .ap.pages = &page,
840 .ap.descs = &desc,
841 };
842 ssize_t res;
5c5c5e51 843 u64 attr_ver;
248d86e8 844
3be5a52b 845 /*
25985edc 846 * Page writeback can extend beyond the lifetime of the
3be5a52b
MS		 847 * page-cache page, so make sure we read a properly synced
848 * page.
849 */
850 fuse_wait_on_page_writeback(inode, page->index);
851
fcee216b 852 attr_ver = fuse_get_attr_version(fm->fc);
5c5c5e51 853
2f139829
MS		 854 /* Don't overflow end offset */
855 if (pos + (desc.length - 1) == LLONG_MAX)
856 desc.length--;
857
00793ca5 858 fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
fcee216b 859 res = fuse_simple_request(fm, &ia.ap.args);
00793ca5
MS		 860 if (res < 0)
861 return res;
862 /*
863 * Short read means EOF. If file size is larger, truncate it
864 */
865 if (res < desc.length)
134831e3 866 fuse_short_read(inode, attr_ver, res, &ia.ap);
5c5c5e51 867
00793ca5 868 SetPageUptodate(page);
482fce55 869
00793ca5 870 return 0;
482fce55
MP		 871}
872
5efd00e4 873static int fuse_read_folio(struct file *file, struct folio *folio)
482fce55 874{
5efd00e4 875 struct page *page = &folio->page;
482fce55
MP		 876 struct inode *inode = page->mapping->host;
877 int err;
878
879 err = -EIO;
5d069dbe 880 if (fuse_is_bad(inode))
482fce55
MP		 881 goto out;
882
883 err = fuse_do_readpage(file, page);
451418fc 884 fuse_invalidate_atime(inode);
b6aeaded
MS		 885 out:
886 unlock_page(page);
887 return err;
888}
889
fcee216b 890static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
134831e3 891 int err)
db50b96c 892{
c1aa96a5 893 int i;
134831e3
MS		 894 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
895 struct fuse_args_pages *ap = &ia->ap;
896 size_t count = ia->read.in.size;
897 size_t num_read = args->out_args[0].size;
ce534fb0 898 struct address_space *mapping = NULL;
c1aa96a5 899
134831e3
MS		 900 for (i = 0; mapping == NULL && i < ap->num_pages; i++)
901 mapping = ap->pages[i]->mapping;
5c5c5e51 902
ce534fb0
MS		 903 if (mapping) {
904 struct inode *inode = mapping->host;
905
906 /*
907 * Short read means EOF. If file size is larger, truncate it
908 */
134831e3
MS		 909 if (!err && num_read < count)
910 fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
ce534fb0 911
451418fc 912 fuse_invalidate_atime(inode);
ce534fb0 913 }
c1aa96a5 914
134831e3
MS		 915 for (i = 0; i < ap->num_pages; i++) {
916 struct page *page = ap->pages[i];
917
918 if (!err)
db50b96c 919 SetPageUptodate(page);
c1aa96a5
MS		 920 else
921 SetPageError(page);
db50b96c 922 unlock_page(page);
09cbfeaf 923 put_page(page);
db50b96c 924 }
134831e3
MS		 925 if (ia->ff)
926 fuse_file_put(ia->ff, false, false);
927
928 fuse_io_free(ia);
c1aa96a5
MS		 929}
930
134831e3 931static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)
c1aa96a5 932{
2106cb18 933 struct fuse_file *ff = file->private_data;
fcee216b 934 struct fuse_mount *fm = ff->fm;
134831e3
MS		 935 struct fuse_args_pages *ap = &ia->ap;
936 loff_t pos = page_offset(ap->pages[0]);
937 size_t count = ap->num_pages << PAGE_SHIFT;
7df1e988 938 ssize_t res;
134831e3
MS		 939 int err;
940
941 ap->args.out_pages = true;
942 ap->args.page_zeroing = true;
943 ap->args.page_replace = true;
2f139829
MS		 944
945 /* Don't overflow end offset */
946 if (pos + (count - 1) == LLONG_MAX) {
947 count--;
948 ap->descs[ap->num_pages - 1].length--;
949 }
950 WARN_ON((loff_t) (pos + count) < 0);
951
134831e3 952 fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
fcee216b
MR		 953 ia->read.attr_ver = fuse_get_attr_version(fm->fc);
954 if (fm->fc->async_read) {
134831e3
MS		 955 ia->ff = fuse_file_get(ff);
956 ap->args.end = fuse_readpages_end;
fcee216b 957 err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
134831e3
MS		 958 if (!err)
959 return;
9cd68455 960 } else {
fcee216b 961 res = fuse_simple_request(fm, &ap->args);
7df1e988 962 err = res < 0 ? res : 0;
9cd68455 963 }
fcee216b 964 fuse_readpages_end(fm, &ap->args, err);
db50b96c
MS		 965}
966
76a0294e 967static void fuse_readahead(struct readahead_control *rac)
db50b96c 968{
76a0294e 969 struct inode *inode = rac->mapping->host;
db50b96c 970 struct fuse_conn *fc = get_fuse_conn(inode);
76a0294e 971 unsigned int i, max_pages, nr_pages = 0;
db50b96c 972
5d069dbe 973 if (fuse_is_bad(inode))
76a0294e 974 return;
248d86e8 975
76a0294e
MWO		 976 max_pages = min_t(unsigned int, fc->max_pages,
977 fc->max_read / PAGE_SIZE);
db50b96c 978
76a0294e
MWO		 979 for (;;) {
980 struct fuse_io_args *ia;
981 struct fuse_args_pages *ap;
982
670d21c6
N		 983 if (fc->num_background >= fc->congestion_threshold &&
984 rac->ra->async_size >= readahead_count(rac))
985 /*
986 * Congested and only async pages left, so skip the
987 * rest.
988 */
989 break;
990
76a0294e
MWO		 991 nr_pages = readahead_count(rac) - nr_pages;
992 if (nr_pages > max_pages)
993 nr_pages = max_pages;
994 if (nr_pages == 0)
995 break;
996 ia = fuse_io_alloc(NULL, nr_pages);
997 if (!ia)
998 return;
999 ap = &ia->ap;
1000 nr_pages = __readahead_batch(rac, ap->pages, nr_pages);
1001 for (i = 0; i < nr_pages; i++) {
1002 fuse_wait_on_page_writeback(inode,
1003 readahead_index(rac) + i);
1004 ap->descs[i].length = PAGE_SIZE;
1005 }
1006 ap->num_pages = nr_pages;
1007 fuse_send_readpages(ia, rac->file);
d3406ffa 1008 }
db50b96c
MS		 1009}
1010
55752a3a 1011static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
bcb4be80
MS		 1012{
1013 struct inode *inode = iocb->ki_filp->f_mapping->host;
a8894274 1014 struct fuse_conn *fc = get_fuse_conn(inode);
bcb4be80 1015
a8894274
BF		 1016 /*
1017 * In auto invalidate mode, always update attributes on read.
1018 * Otherwise, only update if we attempt to read past EOF (to ensure
1019 * i_size is up to date).
1020 */
1021 if (fc->auto_inval_data ||
37c20f16 1022 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
bcb4be80 1023 int err;
c6c745b8 1024 err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE);
bcb4be80
MS		 1025 if (err)
1026 return err;
1027 }
1028
37c20f16 1029 return generic_file_read_iter(iocb, to);
bcb4be80
MS		 1030}
1031
338f2e3f
MS		 1032static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
1033 loff_t pos, size_t count)
1034{
1035 struct fuse_args *args = &ia->ap.args;
1036
1037 ia->write.in.fh = ff->fh;
1038 ia->write.in.offset = pos;
1039 ia->write.in.size = count;
1040 args->opcode = FUSE_WRITE;
1041 args->nodeid = ff->nodeid;
1042 args->in_numargs = 2;
fcee216b 1043 if (ff->fm->fc->minor < 9)
338f2e3f
MS		 1044 args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
1045 else
1046 args->in_args[0].size = sizeof(ia->write.in);
1047 args->in_args[0].value = &ia->write.in;
1048 args->in_args[1].size = count;
1049 args->out_numargs = 1;
1050 args->out_args[0].size = sizeof(ia->write.out);
1051 args->out_args[0].value = &ia->write.out;
1052}
1053
1054static unsigned int fuse_write_flags(struct kiocb *iocb)
1055{
1056 unsigned int flags = iocb->ki_filp->f_flags;
1057
91b94c5d 1058 if (iocb_is_dsync(iocb))
338f2e3f
MS		 1059 flags |= O_DSYNC;
1060 if (iocb->ki_flags & IOCB_SYNC)
1061 flags |= O_SYNC;
1062
1063 return flags;
1064}
1065
45ac96ed
MS		 1066static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
1067 size_t count, fl_owner_t owner)
b25e82e5 1068{
45ac96ed 1069 struct kiocb *iocb = ia->io->iocb;
e1c0eecb 1070 struct file *file = iocb->ki_filp;
2106cb18 1071 struct fuse_file *ff = file->private_data;
fcee216b 1072 struct fuse_mount *fm = ff->fm;
45ac96ed
MS		 1073 struct fuse_write_in *inarg = &ia->write.in;
1074 ssize_t err;
2d698b07 1075
45ac96ed 1076 fuse_write_args_fill(ia, ff, pos, count);
338f2e3f 1077 inarg->flags = fuse_write_flags(iocb);
f3332114 1078 if (owner != NULL) {
f3332114 1079 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
fcee216b 1080 inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
f3332114 1081 }
36cf66ed 1082
45ac96ed 1083 if (ia->io->async)
fcee216b 1084 return fuse_async_req_send(fm, ia, count);
45ac96ed 1085
fcee216b 1086 err = fuse_simple_request(fm, &ia->ap.args);
45ac96ed
MS		 1087 if (!err && ia->write.out.size > count)
1088 err = -EIO;
36cf66ed 1089
45ac96ed 1090 return err ?: ia->write.out.size;
b6aeaded
MS		 1091}
1092
d347739a 1093bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written)
854512ec
MS		 1094{
1095 struct fuse_conn *fc = get_fuse_conn(inode);
1096 struct fuse_inode *fi = get_fuse_inode(inode);
b0aa7606 1097 bool ret = false;
854512ec 1098
f15ecfef 1099 spin_lock(&fi->lock);
4510d86f 1100 fi->attr_version = atomic64_inc_return(&fc->attr_version);
d347739a 1101 if (written > 0 && pos > inode->i_size) {
854512ec 1102 i_size_write(inode, pos);
b0aa7606
MP		 1103 ret = true;
1104 }
f15ecfef 1105 spin_unlock(&fi->lock);
b0aa7606 1106
d347739a
MS		 1107 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
1108
b0aa7606 1109 return ret;
854512ec
MS		 1110}
1111
338f2e3f
MS		 1112static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
1113 struct kiocb *iocb, struct inode *inode,
1114 loff_t pos, size_t count)
ea9b9907 1115{
338f2e3f
MS		 1116 struct fuse_args_pages *ap = &ia->ap;
1117 struct file *file = iocb->ki_filp;
1118 struct fuse_file *ff = file->private_data;
fcee216b 1119 struct fuse_mount *fm = ff->fm;
338f2e3f 1120 unsigned int offset, i;
4f06dd92 1121 bool short_write;
338f2e3f 1122 int err;
ea9b9907 1123
338f2e3f
MS		 1124 for (i = 0; i < ap->num_pages; i++)
1125 fuse_wait_on_page_writeback(inode, ap->pages[i]->index);
ea9b9907 1126
338f2e3f
MS		 1127 fuse_write_args_fill(ia, ff, pos, count);
1128 ia->write.in.flags = fuse_write_flags(iocb);
b8667395
VG		 1129 if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID))
1130 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
ea9b9907 1131
fcee216b 1132 err = fuse_simple_request(fm, &ap->args);
8aab336b
MS		 1133 if (!err && ia->write.out.size > count)
1134 err = -EIO;
338f2e3f 1135
4f06dd92 1136 short_write = ia->write.out.size < count;
338f2e3f
MS		 1137 offset = ap->descs[0].offset;
1138 count = ia->write.out.size;
1139 for (i = 0; i < ap->num_pages; i++) {
1140 struct page *page = ap->pages[i];
ea9b9907 1141
4f06dd92
VG		 1142 if (err) {
1143 ClearPageUptodate(page);
1144 } else {
1145 if (count >= PAGE_SIZE - offset)
1146 count -= PAGE_SIZE - offset;
1147 else {
1148 if (short_write)
1149 ClearPageUptodate(page);
1150 count = 0;
1151 }
1152 offset = 0;
1153 }
1154 if (ia->write.page_locked && (i == ap->num_pages - 1))
1155 unlock_page(page);
09cbfeaf 1156 put_page(page);
ea9b9907
NP		 1157 }
1158
338f2e3f 1159 return err;
ea9b9907
NP		 1160}
1161
4f06dd92 1162static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
338f2e3f
MS		 1163 struct address_space *mapping,
1164 struct iov_iter *ii, loff_t pos,
1165 unsigned int max_pages)
ea9b9907 1166{
4f06dd92 1167 struct fuse_args_pages *ap = &ia->ap;
ea9b9907 1168 struct fuse_conn *fc = get_fuse_conn(mapping->host);
09cbfeaf 1169 unsigned offset = pos & (PAGE_SIZE - 1);
ea9b9907
NP		 1170 size_t count = 0;
1171 int err;
1172
338f2e3f
MS		 1173 ap->args.in_pages = true;
1174 ap->descs[0].offset = offset;
ea9b9907
NP		 1175
1176 do {
1177 size_t tmp;
1178 struct page *page;
09cbfeaf
KS		 1179 pgoff_t index = pos >> PAGE_SHIFT;
1180 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
ea9b9907
NP		 1181 iov_iter_count(ii));
1182
1183 bytes = min_t(size_t, bytes, fc->max_write - count);
1184
1185 again:
1186 err = -EFAULT;
a6294593 1187 if (fault_in_iov_iter_readable(ii, bytes))
ea9b9907
NP		 1188 break;
1189
1190 err = -ENOMEM;
b7446e7c 1191 page = grab_cache_page_write_begin(mapping, index);
ea9b9907
NP		 1192 if (!page)
1193 break;
1194
931e80e4 1195 if (mapping_writably_mapped(mapping))
1196 flush_dcache_page(page);
1197
f0b65f39 1198 tmp = copy_page_from_iter_atomic(page, offset, bytes, ii);
ea9b9907
NP		 1199 flush_dcache_page(page);
1200
1201 if (!tmp) {
1202 unlock_page(page);
09cbfeaf 1203 put_page(page);
ea9b9907
NP		 1204 goto again;
1205 }
1206
1207 err = 0;
338f2e3f
MS		 1208 ap->pages[ap->num_pages] = page;
1209 ap->descs[ap->num_pages].length = tmp;
1210 ap->num_pages++;
ea9b9907 1211
ea9b9907
NP		 1212 count += tmp;
1213 pos += tmp;
1214 offset += tmp;
09cbfeaf 1215 if (offset == PAGE_SIZE)
ea9b9907
NP		 1216 offset = 0;
1217
4f06dd92
VG		 1218 /* If we copied full page, mark it uptodate */
1219 if (tmp == PAGE_SIZE)
1220 SetPageUptodate(page);
1221
1222 if (PageUptodate(page)) {
1223 unlock_page(page);
1224 } else {
1225 ia->write.page_locked = true;
1226 break;
1227 }
78bb6cb9
MS		 1228 if (!fc->big_writes)
1229 break;
ea9b9907 1230 } while (iov_iter_count(ii) && count < fc->max_write &&
338f2e3f 1231 ap->num_pages < max_pages && offset == 0);
ea9b9907
NP		 1232
1233 return count > 0 ? count : err;
1234}
1235
5da784cc
CS		 1236static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1237 unsigned int max_pages)
d07f09f5 1238{
5da784cc 1239 return min_t(unsigned int,
09cbfeaf
KS		 1240 ((pos + len - 1) >> PAGE_SHIFT) -
1241 (pos >> PAGE_SHIFT) + 1,
5da784cc 1242 max_pages);
d07f09f5
MP		 1243}
1244
e1c0eecb 1245static ssize_t fuse_perform_write(struct kiocb *iocb,
ea9b9907
NP		 1246 struct address_space *mapping,
1247 struct iov_iter *ii, loff_t pos)
1248{
1249 struct inode *inode = mapping->host;
1250 struct fuse_conn *fc = get_fuse_conn(inode);
06a7c3c2 1251 struct fuse_inode *fi = get_fuse_inode(inode);
ea9b9907
NP		 1252 int err = 0;
1253 ssize_t res = 0;
1254
06a7c3c2
MP		 1255 if (inode->i_size < pos + iov_iter_count(ii))
1256 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1257
ea9b9907 1258 do {
ea9b9907 1259 ssize_t count;
338f2e3f
MS		 1260 struct fuse_io_args ia = {};
1261 struct fuse_args_pages *ap = &ia.ap;
5da784cc
CS		 1262 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1263 fc->max_pages);
ea9b9907 1264
338f2e3f
MS		 1265 ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs);
1266 if (!ap->pages) {
1267 err = -ENOMEM;
ea9b9907
NP		 1268 break;
1269 }
1270
4f06dd92 1271 count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);
ea9b9907
NP		 1272 if (count <= 0) {
1273 err = count;
1274 } else {
338f2e3f
MS		 1275 err = fuse_send_write_pages(&ia, iocb, inode,
1276 pos, count);
ea9b9907 1277 if (!err) {
338f2e3f
MS		 1278 size_t num_written = ia.write.out.size;
1279
ea9b9907
NP		 1280 res += num_written;
1281 pos += num_written;
1282
1283 /* break out of the loop on short write */
1284 if (num_written != count)
1285 err = -EIO;
1286 }
1287 }
338f2e3f 1288 kfree(ap->pages);
ea9b9907
NP		 1289 } while (!err && iov_iter_count(ii));
1290
d347739a 1291 fuse_write_update_attr(inode, pos, res);
06a7c3c2 1292 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
ea9b9907
NP		 1293
1294 return res > 0 ? res : err;
1295}
1296
55752a3a 1297static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
ea9b9907
NP		 1298{
1299 struct file *file = iocb->ki_filp;
1300 struct address_space *mapping = file->f_mapping;
ea9b9907 1301 ssize_t written = 0;
4273b793 1302 ssize_t written_buffered = 0;
ea9b9907
NP		 1303 struct inode *inode = mapping->host;
1304 ssize_t err;
8981bdfd 1305 struct fuse_conn *fc = get_fuse_conn(inode);
4273b793 1306 loff_t endbyte = 0;
ea9b9907 1307
8981bdfd 1308 if (fc->writeback_cache) {
4d99ff8f 1309 /* Update size (EOF optimization) and mode (SUID clearing) */
c6c745b8
MS		 1310 err = fuse_update_attributes(mapping->host, file,
1311 STATX_SIZE | STATX_MODE);
4d99ff8f
PE		 1312 if (err)
1313 return err;
1314
8981bdfd 1315 if (fc->handle_killpriv_v2 &&
9452e93e
CB		 1316 setattr_should_drop_suidgid(&nop_mnt_idmap,
1317 file_inode(file))) {
8981bdfd
VG		 1318 goto writethrough;
1319 }
1320
84c3d55c 1321 return generic_file_write_iter(iocb, from);
4d99ff8f
PE		 1322 }
1323
8981bdfd 1324writethrough:
5955102c 1325 inode_lock(inode);
ea9b9907
NP		 1326
1327 /* We can write back this queue in page reclaim */
de1414a6 1328 current->backing_dev_info = inode_to_bdi(inode);
ea9b9907 1329
3309dd04
AV		 1330 err = generic_write_checks(iocb, from);
1331 if (err <= 0)
ea9b9907
NP		 1332 goto out;
1333
5fa8e0a1 1334 err = file_remove_privs(file);
ea9b9907
NP		 1335 if (err)
1336 goto out;
1337
c3b2da31
JB		 1338 err = file_update_time(file);
1339 if (err)
1340 goto out;
ea9b9907 1341
2ba48ce5 1342 if (iocb->ki_flags & IOCB_DIRECT) {
3309dd04 1343 loff_t pos = iocb->ki_pos;
1af5bb49 1344 written = generic_file_direct_write(iocb, from);
84c3d55c 1345 if (written < 0 || !iov_iter_count(from))
4273b793
AA		 1346 goto out;
1347
1348 pos += written;
ea9b9907 1349
e1c0eecb 1350 written_buffered = fuse_perform_write(iocb, mapping, from, pos);
4273b793
AA		 1351 if (written_buffered < 0) {
1352 err = written_buffered;
1353 goto out;
1354 }
1355 endbyte = pos + written_buffered - 1;
1356
1357 err = filemap_write_and_wait_range(file->f_mapping, pos,
1358 endbyte);
1359 if (err)
1360 goto out;
1361
1362 invalidate_mapping_pages(file->f_mapping,
09cbfeaf
KS		 1363 pos >> PAGE_SHIFT,
1364 endbyte >> PAGE_SHIFT);
4273b793
AA		 1365
1366 written += written_buffered;
1367 iocb->ki_pos = pos + written_buffered;
1368 } else {
e1c0eecb 1369 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
4273b793 1370 if (written >= 0)
3309dd04 1371 iocb->ki_pos += written;
4273b793 1372 }
ea9b9907
NP		 1373out:
1374 current->backing_dev_info = NULL;
5955102c 1375 inode_unlock(inode);
e1c0eecb
MS		 1376 if (written > 0)
1377 written = generic_write_sync(iocb, written);
ea9b9907
NP		 1378
1379 return written ? written : err;
1380}
1381
7c190c8b
MP		 1382static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1383{
1384 return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1385}
1386
1387static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1388 size_t max_size)
1389{
1390 return min(iov_iter_single_seg_count(ii), max_size);
1391}
1392
45ac96ed
MS		 1393static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1394 size_t *nbytesp, int write,
1395 unsigned int max_pages)
413ef8cb 1396{
7c190c8b 1397 size_t nbytes = 0; /* # bytes already packed in req */
742f9927 1398 ssize_t ret = 0;
b98d023a 1399
f4975c67 1400 /* Special case for kernel I/O: can copy directly into the buffer */
00e23707 1401 if (iov_iter_is_kvec(ii)) {
7c190c8b
MP		 1402 unsigned long user_addr = fuse_get_user_addr(ii);
1403 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1404
f4975c67 1405 if (write)
45ac96ed 1406 ap->args.in_args[1].value = (void *) user_addr;
f4975c67 1407 else
45ac96ed 1408 ap->args.out_args[0].value = (void *) user_addr;
f4975c67 1409
b98d023a
MP		 1410 iov_iter_advance(ii, frag_size);
1411 *nbytesp = frag_size;
f4975c67
MS		 1412 return 0;
1413 }
413ef8cb 1414
45ac96ed 1415 while (nbytes < *nbytesp && ap->num_pages < max_pages) {
7c190c8b 1416 unsigned npages;
f67da30c 1417 size_t start;
1ef255e2 1418 ret = iov_iter_get_pages2(ii, &ap->pages[ap->num_pages],
2c80929c 1419 *nbytesp - nbytes,
45ac96ed 1420 max_pages - ap->num_pages,
c7f3888a 1421 &start);
7c190c8b 1422 if (ret < 0)
742f9927 1423 break;
7c190c8b 1424
c9c37e2e 1425 nbytes += ret;
7c190c8b 1426
c9c37e2e 1427 ret += start;
6c88632b 1428 npages = DIV_ROUND_UP(ret, PAGE_SIZE);
7c190c8b 1429
45ac96ed
MS		 1430 ap->descs[ap->num_pages].offset = start;
1431 fuse_page_descs_length_init(ap->descs, ap->num_pages, npages);
7c190c8b 1432
45ac96ed
MS		 1433 ap->num_pages += npages;
1434 ap->descs[ap->num_pages - 1].length -=
c9c37e2e 1435 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
7c190c8b 1436 }
f4975c67 1437
0c4bcfde 1438 ap->args.user_pages = true;
f4975c67 1439 if (write)
cabdb4fa 1440 ap->args.in_pages = true;
f4975c67 1441 else
cabdb4fa 1442 ap->args.out_pages = true;
f4975c67 1443
7c190c8b 1444 *nbytesp = nbytes;
f4975c67 1445
2c932d4c 1446 return ret < 0 ? ret : 0;
413ef8cb
MS		 1447}
1448
d22a943f
AV		 1449ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1450 loff_t *ppos, int flags)
413ef8cb 1451{
ea8cd333
PE		 1452 int write = flags & FUSE_DIO_WRITE;
1453 int cuse = flags & FUSE_DIO_CUSE;
e1c0eecb 1454 struct file *file = io->iocb->ki_filp;
ea8cd333 1455 struct inode *inode = file->f_mapping->host;
2106cb18 1456 struct fuse_file *ff = file->private_data;
fcee216b 1457 struct fuse_conn *fc = ff->fm->fc;
413ef8cb
MS		 1458 size_t nmax = write ? fc->max_write : fc->max_read;
1459 loff_t pos = *ppos;
d22a943f 1460 size_t count = iov_iter_count(iter);
09cbfeaf
KS		 1461 pgoff_t idx_from = pos >> PAGE_SHIFT;
1462 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
413ef8cb 1463 ssize_t res = 0;
742f9927 1464 int err = 0;
45ac96ed
MS		 1465 struct fuse_io_args *ia;
1466 unsigned int max_pages;
248d86e8 1467
45ac96ed
MS		 1468 max_pages = iov_iter_npages(iter, fc->max_pages);
1469 ia = fuse_io_alloc(io, max_pages);
1470 if (!ia)
1471 return -ENOMEM;
413ef8cb 1472
ea8cd333
PE		 1473 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1474 if (!write)
5955102c 1475 inode_lock(inode);
ea8cd333
PE		 1476 fuse_sync_writes(inode);
1477 if (!write)
5955102c 1478 inode_unlock(inode);
ea8cd333
PE		 1479 }
1480
fcb14cb1 1481 io->should_dirty = !write && user_backed_iter(iter);
413ef8cb 1482 while (count) {
45ac96ed 1483 ssize_t nres;
2106cb18 1484 fl_owner_t owner = current->files;
f4975c67 1485 size_t nbytes = min(count, nmax);
45ac96ed
MS		 1486
1487 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1488 max_pages);
742f9927 1489 if (err && !nbytes)
413ef8cb 1490 break;
f4975c67 1491
4a2abf99 1492 if (write) {
45ac96ed 1493 if (!capable(CAP_FSETID))
10c52c84 1494 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
4a2abf99 1495
45ac96ed 1496 nres = fuse_send_write(ia, pos, nbytes, owner);
4a2abf99 1497 } else {
45ac96ed 1498 nres = fuse_send_read(ia, pos, nbytes, owner);
4a2abf99 1499 }
2106cb18 1500
45ac96ed
MS		 1501 if (!io->async || nres < 0) {
1502 fuse_release_user_pages(&ia->ap, io->should_dirty);
1503 fuse_io_free(ia);
1504 }
1505 ia = NULL;
1506 if (nres < 0) {
f658adee 1507 iov_iter_revert(iter, nbytes);
45ac96ed 1508 err = nres;
413ef8cb
MS		 1509 break;
1510 }
45ac96ed
MS		 1511 WARN_ON(nres > nbytes);
1512
413ef8cb
MS		 1513 count -= nres;
1514 res += nres;
1515 pos += nres;
f658adee
MS		 1516 if (nres != nbytes) {
1517 iov_iter_revert(iter, nbytes - nres);
413ef8cb 1518 break;
f658adee 1519 }
56cf34ff 1520 if (count) {
45ac96ed
MS		 1521 max_pages = iov_iter_npages(iter, fc->max_pages);
1522 ia = fuse_io_alloc(io, max_pages);
1523 if (!ia)
56cf34ff
MS		 1524 break;
1525 }
413ef8cb 1526 }
45ac96ed
MS		 1527 if (ia)
1528 fuse_io_free(ia);
d09cb9d7 1529 if (res > 0)
413ef8cb 1530 *ppos = pos;
413ef8cb 1531
742f9927 1532 return res > 0 ? res : err;
413ef8cb 1533}
08cbf542 1534EXPORT_SYMBOL_GPL(fuse_direct_io);
413ef8cb 1535
36cf66ed 1536static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
d22a943f
AV		 1537 struct iov_iter *iter,
1538 loff_t *ppos)
413ef8cb 1539{
d09cb9d7 1540 ssize_t res;
e1c0eecb 1541 struct inode *inode = file_inode(io->iocb->ki_filp);
d09cb9d7 1542
d22a943f 1543 res = fuse_direct_io(io, iter, ppos, 0);
d09cb9d7 1544
9a2eb24d 1545 fuse_invalidate_atime(inode);
d09cb9d7
MS		 1546
1547 return res;
413ef8cb
MS		 1548}
1549
23c94e1c
MR		 1550static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1551
15316263 1552static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
b98d023a 1553{
23c94e1c
MR		 1554 ssize_t res;
1555
1556 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
23c94e1c
MR		 1557 res = fuse_direct_IO(iocb, to);
1558 } else {
1559 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1560
1561 res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1562 }
1563
1564 return res;
b98d023a
MP		 1565}
1566
15352405
DS		 1567static bool fuse_direct_write_extending_i_size(struct kiocb *iocb,
1568 struct iov_iter *iter)
1569{
1570 struct inode *inode = file_inode(iocb->ki_filp);
1571
1572 return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode);
1573}
1574
15316263 1575static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
4273b793 1576{
e1c0eecb 1577 struct inode *inode = file_inode(iocb->ki_filp);
15352405
DS		 1578 struct file *file = iocb->ki_filp;
1579 struct fuse_file *ff = file->private_data;
e1c0eecb 1580 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
15316263 1581 ssize_t res;
15352405
DS		 1582 bool exclusive_lock =
1583 !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES) ||
1584 iocb->ki_flags & IOCB_APPEND ||
1585 fuse_direct_write_extending_i_size(iocb, from);
1586
1587 /*
1588 * Take exclusive lock if
1589 * - Parallel direct writes are disabled - a user space decision
1590 * - Parallel direct writes are enabled and i_size is being extended.
1591 * This might not be needed at all, but needs further investigation.
1592 */
1593 if (exclusive_lock)
1594 inode_lock(inode);
1595 else {
1596 inode_lock_shared(inode);
1597
1598 /* A race with truncate might have come up as the decision for
1599 * the lock type was done without holding the lock, check again.
1600 */
1601 if (fuse_direct_write_extending_i_size(iocb, from)) {
1602 inode_unlock_shared(inode);
1603 inode_lock(inode);
1604 exclusive_lock = true;
1605 }
1606 }
4273b793 1607
3309dd04 1608 res = generic_write_checks(iocb, from);
23c94e1c
MR		 1609 if (res > 0) {
1610 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1611 res = fuse_direct_IO(iocb, from);
1612 } else {
1613 res = fuse_direct_io(&io, from, &iocb->ki_pos,
1614 FUSE_DIO_WRITE);
d347739a 1615 fuse_write_update_attr(inode, iocb->ki_pos, res);
23c94e1c
MR		 1616 }
1617 }
15352405
DS		 1618 if (exclusive_lock)
1619 inode_unlock(inode);
1620 else
1621 inode_unlock_shared(inode);
4273b793
AA		 1622
1623 return res;
1624}
1625
55752a3a
MS		 1626static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1627{
2f7b6f5b
MS		 1628 struct file *file = iocb->ki_filp;
1629 struct fuse_file *ff = file->private_data;
c2d0ad00 1630 struct inode *inode = file_inode(file);
2f7b6f5b 1631
5d069dbe 1632 if (fuse_is_bad(inode))
2f7b6f5b 1633 return -EIO;
55752a3a 1634
c2d0ad00
VG		 1635 if (FUSE_IS_DAX(inode))
1636 return fuse_dax_read_iter(iocb, to);
1637
55752a3a
MS		 1638 if (!(ff->open_flags & FOPEN_DIRECT_IO))
1639 return fuse_cache_read_iter(iocb, to);
1640 else
1641 return fuse_direct_read_iter(iocb, to);
1642}
1643
1644static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1645{
2f7b6f5b
MS		 1646 struct file *file = iocb->ki_filp;
1647 struct fuse_file *ff = file->private_data;
c2d0ad00 1648 struct inode *inode = file_inode(file);
2f7b6f5b 1649
5d069dbe 1650 if (fuse_is_bad(inode))
2f7b6f5b 1651 return -EIO;
55752a3a 1652
c2d0ad00
VG		 1653 if (FUSE_IS_DAX(inode))
1654 return fuse_dax_write_iter(iocb, from);
1655
55752a3a
MS		 1656 if (!(ff->open_flags & FOPEN_DIRECT_IO))
1657 return fuse_cache_write_iter(iocb, from);
1658 else
1659 return fuse_direct_write_iter(iocb, from);
1660}
1661
33826ebb 1662static void fuse_writepage_free(struct fuse_writepage_args *wpa)
b6aeaded 1663{
33826ebb 1664 struct fuse_args_pages *ap = &wpa->ia.ap;
385b1268
PE		 1665 int i;
1666
660585b5
MS		 1667 if (wpa->bucket)
1668 fuse_sync_bucket_dec(wpa->bucket);
1669
33826ebb
MS		 1670 for (i = 0; i < ap->num_pages; i++)
1671 __free_page(ap->pages[i]);
1672
1673 if (wpa->ia.ff)
1674 fuse_file_put(wpa->ia.ff, false, false);
8b284dc4 1675
33826ebb
MS		 1676 kfree(ap->pages);
1677 kfree(wpa);
3be5a52b
MS		 1678}
1679
fcee216b 1680static void fuse_writepage_finish(struct fuse_mount *fm,
33826ebb 1681 struct fuse_writepage_args *wpa)
3be5a52b 1682{
33826ebb
MS		 1683 struct fuse_args_pages *ap = &wpa->ia.ap;
1684 struct inode *inode = wpa->inode;
3be5a52b 1685 struct fuse_inode *fi = get_fuse_inode(inode);
de1414a6 1686 struct backing_dev_info *bdi = inode_to_bdi(inode);
385b1268 1687 int i;
3be5a52b 1688
33826ebb 1689 for (i = 0; i < ap->num_pages; i++) {
93f78d88 1690 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
33826ebb 1691 dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP);
93f78d88 1692 wb_writeout_inc(&bdi->wb);
385b1268 1693 }
3be5a52b
MS		 1694 wake_up(&fi->page_waitq);
1695}
1696
f15ecfef 1697/* Called under fi->lock, may release and reacquire it */
fcee216b 1698static void fuse_send_writepage(struct fuse_mount *fm,
33826ebb 1699 struct fuse_writepage_args *wpa, loff_t size)
f15ecfef
KT		 1700__releases(fi->lock)
1701__acquires(fi->lock)
3be5a52b 1702{
33826ebb
MS		 1703 struct fuse_writepage_args *aux, *next;
1704 struct fuse_inode *fi = get_fuse_inode(wpa->inode);
1705 struct fuse_write_in *inarg = &wpa->ia.write.in;
1706 struct fuse_args *args = &wpa->ia.ap.args;
1707 __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;
1708 int err;
3be5a52b 1709
33826ebb 1710 fi->writectr++;
385b1268
PE		 1711 if (inarg->offset + data_size <= size) {
1712 inarg->size = data_size;
3be5a52b 1713 } else if (inarg->offset < size) {
385b1268 1714 inarg->size = size - inarg->offset;
3be5a52b
MS		 1715 } else {
1716 /* Got truncated off completely */
1717 goto out_free;
b6aeaded 1718 }
3be5a52b 1719
33826ebb
MS		 1720 args->in_args[1].size = inarg->size;
1721 args->force = true;
1722 args->nocreds = true;
1723
fcee216b 1724 err = fuse_simple_background(fm, args, GFP_ATOMIC);
33826ebb
MS		 1725 if (err == -ENOMEM) {
1726 spin_unlock(&fi->lock);
fcee216b 1727 err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
33826ebb
MS		 1728 spin_lock(&fi->lock);
1729 }
1730
f15ecfef 1731 /* Fails on broken connection only */
33826ebb 1732 if (unlikely(err))
f15ecfef
KT		 1733 goto out_free;
1734
3be5a52b
MS		 1735 return;
1736
1737 out_free:
33826ebb 1738 fi->writectr--;
69a6487a 1739 rb_erase(&wpa->writepages_entry, &fi->writepages);
fcee216b 1740 fuse_writepage_finish(fm, wpa);
f15ecfef 1741 spin_unlock(&fi->lock);
e2653bd5
MS		 1742
1743 /* After fuse_writepage_finish() aux request list is private */
33826ebb
MS		 1744 for (aux = wpa->next; aux; aux = next) {
1745 next = aux->next;
1746 aux->next = NULL;
1747 fuse_writepage_free(aux);
e2653bd5
MS		 1748 }
1749
33826ebb 1750 fuse_writepage_free(wpa);
f15ecfef 1751 spin_lock(&fi->lock);
b6aeaded
MS		 1752}
1753
3be5a52b
MS		 1754/*
1755 * If fi->writectr is positive (no truncate or fsync going on) send
1756 * all queued writepage requests.
1757 *
f15ecfef 1758 * Called with fi->lock
3be5a52b
MS		 1759 */
1760void fuse_flush_writepages(struct inode *inode)
f15ecfef
KT		 1761__releases(fi->lock)
1762__acquires(fi->lock)
b6aeaded 1763{
fcee216b 1764 struct fuse_mount *fm = get_fuse_mount(inode);
3be5a52b 1765 struct fuse_inode *fi = get_fuse_inode(inode);
9de5be06 1766 loff_t crop = i_size_read(inode);
33826ebb 1767 struct fuse_writepage_args *wpa;
3be5a52b
MS		 1768
1769 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
33826ebb
MS		 1770 wpa = list_entry(fi->queued_writes.next,
1771 struct fuse_writepage_args, queue_entry);
1772 list_del_init(&wpa->queue_entry);
fcee216b 1773 fuse_send_writepage(fm, wpa, crop);
3be5a52b
MS		 1774 }
1775}
1776
c146024e
MS		 1777static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root,
1778 struct fuse_writepage_args *wpa)
6b2fb799
MP		 1779{
1780 pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;
1781 pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;
1782 struct rb_node **p = &root->rb_node;
1783 struct rb_node *parent = NULL;
1784
1785 WARN_ON(!wpa->ia.ap.num_pages);
1786 while (*p) {
1787 struct fuse_writepage_args *curr;
1788 pgoff_t curr_index;
1789
1790 parent = *p;
1791 curr = rb_entry(parent, struct fuse_writepage_args,
1792 writepages_entry);
1793 WARN_ON(curr->inode != wpa->inode);
1794 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;
1795
1796 if (idx_from >= curr_index + curr->ia.ap.num_pages)
1797 p = &(*p)->rb_right;
1798 else if (idx_to < curr_index)
1799 p = &(*p)->rb_left;
1800 else
c146024e 1801 return curr;
6b2fb799
MP		 1802 }
1803
1804 rb_link_node(&wpa->writepages_entry, parent, p);
1805 rb_insert_color(&wpa->writepages_entry, root);
c146024e
MS		 1806 return NULL;
1807}
1808
1809static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa)
1810{
1811 WARN_ON(fuse_insert_writeback(root, wpa));
6b2fb799
MP		 1812}
1813
fcee216b 1814static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
33826ebb 1815 int error)
3be5a52b 1816{
33826ebb
MS		 1817 struct fuse_writepage_args *wpa =
1818 container_of(args, typeof(*wpa), ia.ap.args);
1819 struct inode *inode = wpa->inode;
3be5a52b 1820 struct fuse_inode *fi = get_fuse_inode(inode);
3466958b 1821 struct fuse_conn *fc = get_fuse_conn(inode);
3be5a52b 1822
33826ebb 1823 mapping_set_error(inode->i_mapping, error);
3466958b
VG		 1824 /*
1825 * A writeback finished and this might have updated mtime/ctime on
1826 * server making local mtime/ctime stale. Hence invalidate attrs.
1827 * Do this only if writeback_cache is not enabled. If writeback_cache
1828 * is enabled, we trust local ctime/mtime.
1829 */
1830 if (!fc->writeback_cache)
fa5eee57 1831 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY);
f15ecfef 1832 spin_lock(&fi->lock);
69a6487a 1833 rb_erase(&wpa->writepages_entry, &fi->writepages);
33826ebb 1834 while (wpa->next) {
fcee216b 1835 struct fuse_mount *fm = get_fuse_mount(inode);
33826ebb
MS		 1836 struct fuse_write_in *inarg = &wpa->ia.write.in;
1837 struct fuse_writepage_args *next = wpa->next;
1838
1839 wpa->next = next->next;
1840 next->next = NULL;
1841 next->ia.ff = fuse_file_get(wpa->ia.ff);
6b2fb799 1842 tree_insert(&fi->writepages, next);
6eaf4782
MP		 1843
1844 /*
1845 * Skip fuse_flush_writepages() to make it easy to crop requests
1846 * based on primary request size.
1847 *
1848 * 1st case (trivial): there are no concurrent activities using
1849 * fuse_set/release_nowrite. Then we're on safe side because
1850 * fuse_flush_writepages() would call fuse_send_writepage()
1851 * anyway.
1852 *
1853 * 2nd case: someone called fuse_set_nowrite and it is waiting
1854 * now for completion of all in-flight requests. This happens
1855 * rarely and no more than once per page, so this should be
1856 * okay.
1857 *
1858 * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle
1859 * of fuse_set_nowrite..fuse_release_nowrite section. The fact
1860 * that fuse_set_nowrite returned implies that all in-flight
1861 * requests were completed along with all of their secondary
1862 * requests. Further primary requests are blocked by negative
1863 * writectr. Hence there cannot be any in-flight requests and
1864 * no invocations of fuse_writepage_end() while we're in
1865 * fuse_set_nowrite..fuse_release_nowrite section.
1866 */
fcee216b 1867 fuse_send_writepage(fm, next, inarg->offset + inarg->size);
8b284dc4 1868 }
3be5a52b 1869 fi->writectr--;
fcee216b 1870 fuse_writepage_finish(fm, wpa);
f15ecfef 1871 spin_unlock(&fi->lock);
33826ebb 1872 fuse_writepage_free(wpa);
3be5a52b
MS		 1873}
1874
a9667ac8 1875static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi)
adcadfa8 1876{
84840efc 1877 struct fuse_file *ff;
adcadfa8 1878
f15ecfef 1879 spin_lock(&fi->lock);
84840efc
MS		 1880 ff = list_first_entry_or_null(&fi->write_files, struct fuse_file,
1881 write_entry);
1882 if (ff)
72523425 1883 fuse_file_get(ff);
f15ecfef 1884 spin_unlock(&fi->lock);
adcadfa8
PE		 1885
1886 return ff;
1887}
1888
a9667ac8 1889static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi)
1e18bda8 1890{
a9667ac8 1891 struct fuse_file *ff = __fuse_write_file_get(fi);
1e18bda8
MS		 1892 WARN_ON(!ff);
1893 return ff;
1894}
1895
1896int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1897{
1e18bda8
MS		 1898 struct fuse_inode *fi = get_fuse_inode(inode);
1899 struct fuse_file *ff;
1900 int err;
1901
5c791fe1
MS		 1902 /*
1903 * Inode is always written before the last reference is dropped and
1904 * hence this should not be reached from reclaim.
1905 *
1906 * Writing back the inode from reclaim can deadlock if the request
1907 * processing itself needs an allocation. Allocations triggering
1908 * reclaim while serving a request can't be prevented, because it can
1909 * involve any number of unrelated userspace processes.
1910 */
1911 WARN_ON(wbc->for_reclaim);
1912
a9667ac8 1913 ff = __fuse_write_file_get(fi);
ab9e13f7 1914 err = fuse_flush_times(inode, ff);
1e18bda8 1915 if (ff)
2e64ff15 1916 fuse_file_put(ff, false, false);
1e18bda8
MS		 1917
1918 return err;
1919}
1920
33826ebb
MS		 1921static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
1922{
1923 struct fuse_writepage_args *wpa;
1924 struct fuse_args_pages *ap;
1925
1926 wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
1927 if (wpa) {
1928 ap = &wpa->ia.ap;
1929 ap->num_pages = 0;
1930 ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs);
1931 if (!ap->pages) {
1932 kfree(wpa);
1933 wpa = NULL;
1934 }
1935 }
1936 return wpa;
1937
1938}
1939
660585b5
MS		 1940static void fuse_writepage_add_to_bucket(struct fuse_conn *fc,
1941 struct fuse_writepage_args *wpa)
1942{
1943 if (!fc->sync_fs)
1944 return;
1945
1946 rcu_read_lock();
1947 /* Prevent resurrection of dead bucket in unlikely race with syncfs */
1948 do {
1949 wpa->bucket = rcu_dereference(fc->curr_bucket);
1950 } while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count)));
1951 rcu_read_unlock();
1952}
1953
3be5a52b
MS		 1954static int fuse_writepage_locked(struct page *page)
1955{
1956 struct address_space *mapping = page->mapping;
1957 struct inode *inode = mapping->host;
1958 struct fuse_conn *fc = get_fuse_conn(inode);
1959 struct fuse_inode *fi = get_fuse_inode(inode);
33826ebb
MS		 1960 struct fuse_writepage_args *wpa;
1961 struct fuse_args_pages *ap;
3be5a52b 1962 struct page *tmp_page;
72523425 1963 int error = -ENOMEM;
3be5a52b
MS		 1964
1965 set_page_writeback(page);
1966
33826ebb
MS		 1967 wpa = fuse_writepage_args_alloc();
1968 if (!wpa)
3be5a52b 1969 goto err;
33826ebb 1970 ap = &wpa->ia.ap;
3be5a52b
MS		 1971
1972 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1973 if (!tmp_page)
1974 goto err_free;
1975
72523425 1976 error = -EIO;
a9667ac8 1977 wpa->ia.ff = fuse_write_file_get(fi);
33826ebb 1978 if (!wpa->ia.ff)
27f1b363 1979 goto err_nofile;
72523425 1980
660585b5 1981 fuse_writepage_add_to_bucket(fc, wpa);
33826ebb 1982 fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);
3be5a52b
MS		 1983
1984 copy_highpage(tmp_page, page);
33826ebb
MS		 1985 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
1986 wpa->next = NULL;
1987 ap->args.in_pages = true;
1988 ap->num_pages = 1;
1989 ap->pages[0] = tmp_page;
1990 ap->descs[0].offset = 0;
1991 ap->descs[0].length = PAGE_SIZE;
1992 ap->args.end = fuse_writepage_end;
1993 wpa->inode = inode;
3be5a52b 1994
93f78d88 1995 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
11fb9989 1996 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
3be5a52b 1997
f15ecfef 1998 spin_lock(&fi->lock);
6b2fb799 1999 tree_insert(&fi->writepages, wpa);
33826ebb 2000 list_add_tail(&wpa->queue_entry, &fi->queued_writes);
3be5a52b 2001 fuse_flush_writepages(inode);
f15ecfef 2002 spin_unlock(&fi->lock);
3be5a52b 2003
4a4ac4eb
MP		 2004 end_page_writeback(page);
2005
3be5a52b
MS		 2006 return 0;
2007
27f1b363
MP		 2008err_nofile:
2009 __free_page(tmp_page);
3be5a52b 2010err_free:
33826ebb 2011 kfree(wpa);
3be5a52b 2012err:
9183976e 2013 mapping_set_error(page->mapping, error);
3be5a52b 2014 end_page_writeback(page);
72523425 2015 return error;
3be5a52b
MS		 2016}
2017
2018static int fuse_writepage(struct page *page, struct writeback_control *wbc)
2019{
670d21c6 2020 struct fuse_conn *fc = get_fuse_conn(page->mapping->host);
3be5a52b
MS		 2021 int err;
2022
ff17be08
MS		 2023 if (fuse_page_is_writeback(page->mapping->host, page->index)) {
2024 /*
2025 * ->writepages() should be called for sync() and friends. We
2026 * should only get here on direct reclaim and then we are
2027 * allowed to skip a page which is already in flight
2028 */
2029 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
2030
2031 redirty_page_for_writepage(wbc, page);
d5880c7a 2032 unlock_page(page);
ff17be08
MS		 2033 return 0;
2034 }
2035
670d21c6
N		 2036 if (wbc->sync_mode == WB_SYNC_NONE &&
2037 fc->num_background >= fc->congestion_threshold)
2038 return AOP_WRITEPAGE_ACTIVATE;
2039
3be5a52b
MS		 2040 err = fuse_writepage_locked(page);
2041 unlock_page(page);
2042
2043 return err;
2044}
2045
26d614df 2046struct fuse_fill_wb_data {
33826ebb 2047 struct fuse_writepage_args *wpa;
26d614df
PE		 2048 struct fuse_file *ff;
2049 struct inode *inode;
2d033eaa 2050 struct page **orig_pages;
33826ebb 2051 unsigned int max_pages;
26d614df
PE		 2052};
2053
33826ebb
MS		 2054static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
2055{
2056 struct fuse_args_pages *ap = &data->wpa->ia.ap;
2057 struct fuse_conn *fc = get_fuse_conn(data->inode);
2058 struct page **pages;
2059 struct fuse_page_desc *descs;
2060 unsigned int npages = min_t(unsigned int,
2061 max_t(unsigned int, data->max_pages * 2,
2062 FUSE_DEFAULT_MAX_PAGES_PER_REQ),
2063 fc->max_pages);
2064 WARN_ON(npages <= data->max_pages);
2065
2066 pages = fuse_pages_alloc(npages, GFP_NOFS, &descs);
2067 if (!pages)
2068 return false;
2069
2070 memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages);
2071 memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages);
2072 kfree(ap->pages);
2073 ap->pages = pages;
2074 ap->descs = descs;
2075 data->max_pages = npages;
2076
2077 return true;
2078}
2079
26d614df
PE		 2080static void fuse_writepages_send(struct fuse_fill_wb_data *data)
2081{
33826ebb 2082 struct fuse_writepage_args *wpa = data->wpa;
26d614df 2083 struct inode *inode = data->inode;
26d614df 2084 struct fuse_inode *fi = get_fuse_inode(inode);
33826ebb 2085 int num_pages = wpa->ia.ap.num_pages;
2d033eaa 2086 int i;
26d614df 2087
33826ebb 2088 wpa->ia.ff = fuse_file_get(data->ff);
f15ecfef 2089 spin_lock(&fi->lock);
33826ebb 2090 list_add_tail(&wpa->queue_entry, &fi->queued_writes);
26d614df 2091 fuse_flush_writepages(inode);
f15ecfef 2092 spin_unlock(&fi->lock);
2d033eaa
MP		 2093
2094 for (i = 0; i < num_pages; i++)
2095 end_page_writeback(data->orig_pages[i]);
26d614df
PE		 2096}
2097
7f305ca1 2098/*
c146024e
MS		 2099 * Check under fi->lock if the page is under writeback, and insert it onto the
2100 * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's
419234d5
MS		 2101 * one already added for a page at this offset. If there's none, then insert
2102 * this new request onto the auxiliary list, otherwise reuse the existing one by
c146024e 2103 * swapping the new temp page with the old one.
7f305ca1 2104 */
c146024e
MS		 2105static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,
2106 struct page *page)
8b284dc4 2107{
33826ebb
MS		 2108 struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);
2109 struct fuse_writepage_args *tmp;
2110 struct fuse_writepage_args *old_wpa;
2111 struct fuse_args_pages *new_ap = &new_wpa->ia.ap;
8b284dc4 2112
33826ebb 2113 WARN_ON(new_ap->num_pages != 0);
c146024e 2114 new_ap->num_pages = 1;
8b284dc4 2115
f15ecfef 2116 spin_lock(&fi->lock);
c146024e 2117 old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);
33826ebb 2118 if (!old_wpa) {
f15ecfef 2119 spin_unlock(&fi->lock);
c146024e 2120 return true;
f6011081 2121 }
8b284dc4 2122
33826ebb 2123 for (tmp = old_wpa->next; tmp; tmp = tmp->next) {
7f305ca1
MS		 2124 pgoff_t curr_index;
2125
33826ebb
MS		 2126 WARN_ON(tmp->inode != new_wpa->inode);
2127 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;
419234d5 2128 if (curr_index == page->index) {
33826ebb
MS		 2129 WARN_ON(tmp->ia.ap.num_pages != 1);
2130 swap(tmp->ia.ap.pages[0], new_ap->pages[0]);
7f305ca1 2131 break;
8b284dc4
MS		 2132 }
2133 }
2134
7f305ca1 2135 if (!tmp) {
33826ebb
MS		 2136 new_wpa->next = old_wpa->next;
2137 old_wpa->next = new_wpa;
7f305ca1 2138 }
41b6e41f 2139
f15ecfef 2140 spin_unlock(&fi->lock);
7f305ca1
MS		 2141
2142 if (tmp) {
33826ebb 2143 struct backing_dev_info *bdi = inode_to_bdi(new_wpa->inode);
8b284dc4 2144
93f78d88 2145 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
33826ebb 2146 dec_node_page_state(new_ap->pages[0], NR_WRITEBACK_TEMP);
93f78d88 2147 wb_writeout_inc(&bdi->wb);
33826ebb 2148 fuse_writepage_free(new_wpa);
8b284dc4 2149 }
7f305ca1 2150
c146024e 2151 return false;
8b284dc4
MS		 2152}
2153
6ddf3af9
MS		 2154static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page,
2155 struct fuse_args_pages *ap,
2156 struct fuse_fill_wb_data *data)
2157{
2158 WARN_ON(!ap->num_pages);
2159
2160 /*
2161 * Being under writeback is unlikely but possible. For example direct
2162 * read to an mmaped fuse file will set the page dirty twice; once when
2163 * the pages are faulted with get_user_pages(), and then after the read
2164 * completed.
2165 */
2166 if (fuse_page_is_writeback(data->inode, page->index))
2167 return true;
2168
2169 /* Reached max pages */
2170 if (ap->num_pages == fc->max_pages)
2171 return true;
2172
2173 /* Reached max write bytes */
2174 if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write)
2175 return true;
2176
2177 /* Discontinuity */
2178 if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)
2179 return true;
2180
2181 /* Need to grow the pages array? If so, did the expansion fail? */
2182 if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data))
2183 return true;
2184
2185 return false;
2186}
2187
26d614df
PE		 2188static int fuse_writepages_fill(struct page *page,
2189 struct writeback_control *wbc, void *_data)
2190{
2191 struct fuse_fill_wb_data *data = _data;
33826ebb
MS		 2192 struct fuse_writepage_args *wpa = data->wpa;
2193 struct fuse_args_pages *ap = &wpa->ia.ap;
26d614df 2194 struct inode *inode = data->inode;
f15ecfef 2195 struct fuse_inode *fi = get_fuse_inode(inode);
26d614df
PE		 2196 struct fuse_conn *fc = get_fuse_conn(inode);
2197 struct page *tmp_page;
2198 int err;
2199
2200 if (!data->ff) {
2201 err = -EIO;
a9667ac8 2202 data->ff = fuse_write_file_get(fi);
26d614df
PE		 2203 if (!data->ff)
2204 goto out_unlock;
2205 }
2206
6ddf3af9 2207 if (wpa && fuse_writepage_need_send(fc, page, ap, data)) {
8b284dc4 2208 fuse_writepages_send(data);
33826ebb 2209 data->wpa = NULL;
26d614df 2210 }
e52a8250 2211
26d614df
PE		 2212 err = -ENOMEM;
2213 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
2214 if (!tmp_page)
2215 goto out_unlock;
2216
2217 /*
2218 * The page must not be redirtied until the writeout is completed
2219 * (i.e. userspace has sent a reply to the write request). Otherwise
2220 * there could be more than one temporary page instance for each real
2221 * page.
2222 *
2223 * This is ensured by holding the page lock in page_mkwrite() while
2224 * checking fuse_page_is_writeback(). We already hold the page lock
2225 * since clear_page_dirty_for_io() and keep it held until we add the
33826ebb 2226 * request to the fi->writepages list and increment ap->num_pages.
26d614df
PE		 2227 * After this fuse_page_is_writeback() will indicate that the page is
2228 * under writeback, so we can release the page lock.
2229 */
33826ebb 2230 if (data->wpa == NULL) {
26d614df 2231 err = -ENOMEM;
33826ebb
MS		 2232 wpa = fuse_writepage_args_alloc();
2233 if (!wpa) {
26d614df
PE		 2234 __free_page(tmp_page);
2235 goto out_unlock;
2236 }
660585b5
MS		 2237 fuse_writepage_add_to_bucket(fc, wpa);
2238
33826ebb 2239 data->max_pages = 1;
26d614df 2240
33826ebb
MS		 2241 ap = &wpa->ia.ap;
2242 fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);
2243 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2244 wpa->next = NULL;
2245 ap->args.in_pages = true;
2246 ap->args.end = fuse_writepage_end;
2247 ap->num_pages = 0;
2248 wpa->inode = inode;
26d614df
PE		 2249 }
2250 set_page_writeback(page);
2251
2252 copy_highpage(tmp_page, page);
33826ebb
MS		 2253 ap->pages[ap->num_pages] = tmp_page;
2254 ap->descs[ap->num_pages].offset = 0;
2255 ap->descs[ap->num_pages].length = PAGE_SIZE;
c146024e 2256 data->orig_pages[ap->num_pages] = page;
26d614df 2257
93f78d88 2258 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
11fb9989 2259 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
8b284dc4
MS		 2260
2261 err = 0;
c146024e
MS		 2262 if (data->wpa) {
2263 /*
2264 * Protected by fi->lock against concurrent access by
2265 * fuse_page_is_writeback().
2266 */
2267 spin_lock(&fi->lock);
2268 ap->num_pages++;
2269 spin_unlock(&fi->lock);
2270 } else if (fuse_writepage_add(wpa, page)) {
2271 data->wpa = wpa;
2272 } else {
8b284dc4 2273 end_page_writeback(page);
8b284dc4 2274 }
26d614df
PE		 2275out_unlock:
2276 unlock_page(page);
2277
2278 return err;
2279}
2280
2281static int fuse_writepages(struct address_space *mapping,
2282 struct writeback_control *wbc)
2283{
2284 struct inode *inode = mapping->host;
5da784cc 2285 struct fuse_conn *fc = get_fuse_conn(inode);
26d614df
PE		 2286 struct fuse_fill_wb_data data;
2287 int err;
2288
2289 err = -EIO;
5d069dbe 2290 if (fuse_is_bad(inode))
26d614df
PE		 2291 goto out;
2292
670d21c6
N		 2293 if (wbc->sync_mode == WB_SYNC_NONE &&
2294 fc->num_background >= fc->congestion_threshold)
2295 return 0;
2296
26d614df 2297 data.inode = inode;
33826ebb 2298 data.wpa = NULL;
26d614df
PE		 2299 data.ff = NULL;
2300
2d033eaa 2301 err = -ENOMEM;
5da784cc 2302 data.orig_pages = kcalloc(fc->max_pages,
f2b3455e 2303 sizeof(struct page *),
2d033eaa
MP		 2304 GFP_NOFS);
2305 if (!data.orig_pages)
2306 goto out;
2307
26d614df 2308 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
33826ebb 2309 if (data.wpa) {
33826ebb 2310 WARN_ON(!data.wpa->ia.ap.num_pages);
26d614df 2311 fuse_writepages_send(&data);
26d614df
PE		 2312 }
2313 if (data.ff)
2e64ff15 2314 fuse_file_put(data.ff, false, false);
2d033eaa
MP		 2315
2316 kfree(data.orig_pages);
26d614df
PE		 2317out:
2318 return err;
2319}
2320
6b12c1b3
PE		 2321/*
2322 * It's worthy to make sure that space is reserved on disk for the write,
2323 * but how to implement it without killing performance need more thinking.
2324 */
2325static int fuse_write_begin(struct file *file, struct address_space *mapping,
9d6b0cd7 2326 loff_t pos, unsigned len, struct page **pagep, void **fsdata)
6b12c1b3 2327{
09cbfeaf 2328 pgoff_t index = pos >> PAGE_SHIFT;
a455589f 2329 struct fuse_conn *fc = get_fuse_conn(file_inode(file));
6b12c1b3
PE		 2330 struct page *page;
2331 loff_t fsize;
2332 int err = -ENOMEM;
2333
2334 WARN_ON(!fc->writeback_cache);
2335
b7446e7c 2336 page = grab_cache_page_write_begin(mapping, index);
6b12c1b3
PE		 2337 if (!page)
2338 goto error;
2339
2340 fuse_wait_on_page_writeback(mapping->host, page->index);
2341
09cbfeaf 2342 if (PageUptodate(page) || len == PAGE_SIZE)
6b12c1b3
PE		 2343 goto success;
2344 /*
2345 * Check if the start this page comes after the end of file, in which
2346 * case the readpage can be optimized away.
2347 */
2348 fsize = i_size_read(mapping->host);
09cbfeaf
KS		 2349 if (fsize <= (pos & PAGE_MASK)) {
2350 size_t off = pos & ~PAGE_MASK;
6b12c1b3
PE		 2351 if (off)
2352 zero_user_segment(page, 0, off);
2353 goto success;
2354 }
2355 err = fuse_do_readpage(file, page);
2356 if (err)
2357 goto cleanup;
2358success:
2359 *pagep = page;
2360 return 0;
2361
2362cleanup:
2363 unlock_page(page);
09cbfeaf 2364 put_page(page);
6b12c1b3
PE		 2365error:
2366 return err;
2367}
2368
2369static int fuse_write_end(struct file *file, struct address_space *mapping,
2370 loff_t pos, unsigned len, unsigned copied,
2371 struct page *page, void *fsdata)
2372{
2373 struct inode *inode = page->mapping->host;
2374
59c3b76c
MS		 2375 /* Haven't copied anything? Skip zeroing, size extending, dirtying. */
2376 if (!copied)
2377 goto unlock;
2378
8c56e03d 2379 pos += copied;
6b12c1b3
PE		 2380 if (!PageUptodate(page)) {
2381 /* Zero any unwritten bytes at the end of the page */
8c56e03d 2382 size_t endoff = pos & ~PAGE_MASK;
6b12c1b3 2383 if (endoff)
09cbfeaf 2384 zero_user_segment(page, endoff, PAGE_SIZE);
6b12c1b3
PE		 2385 SetPageUptodate(page);
2386 }
2387
8c56e03d
MS		 2388 if (pos > inode->i_size)
2389 i_size_write(inode, pos);
2390
6b12c1b3 2391 set_page_dirty(page);
59c3b76c
MS		 2392
2393unlock:
6b12c1b3 2394 unlock_page(page);
09cbfeaf 2395 put_page(page);
6b12c1b3
PE		 2396
2397 return copied;
2398}
2399
2bf06b8e 2400static int fuse_launder_folio(struct folio *folio)
3be5a52b
MS		 2401{
2402 int err = 0;
2bf06b8e
MWO		 2403 if (folio_clear_dirty_for_io(folio)) {
2404 struct inode *inode = folio->mapping->host;
3993382b
MS		 2405
2406 /* Serialize with pending writeback for the same page */
2bf06b8e
MWO		 2407 fuse_wait_on_page_writeback(inode, folio->index);
2408 err = fuse_writepage_locked(&folio->page);
3be5a52b 2409 if (!err)
2bf06b8e 2410 fuse_wait_on_page_writeback(inode, folio->index);
3be5a52b
MS		 2411 }
2412 return err;
2413}
2414
2415/*
36ea2337
MS		 2416 * Write back dirty data/metadata now (there may not be any suitable
2417 * open files later for data)
3be5a52b
MS		 2418 */
2419static void fuse_vma_close(struct vm_area_struct *vma)
2420{
36ea2337
MS		 2421 int err;
2422
2423 err = write_inode_now(vma->vm_file->f_mapping->host, 1);
2424 mapping_set_error(vma->vm_file->f_mapping, err);
3be5a52b
MS		 2425}
2426
2427/*
2428 * Wait for writeback against this page to complete before allowing it
2429 * to be marked dirty again, and hence written back again, possibly
2430 * before the previous writepage completed.
2431 *
2432 * Block here, instead of in ->writepage(), so that the userspace fs
2433 * can only block processes actually operating on the filesystem.
2434 *
2435 * Otherwise unprivileged userspace fs would be able to block
2436 * unrelated:
2437 *
2438 * - page migration
2439 * - sync(2)
2440 * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
2441 */
46fb504a 2442static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
3be5a52b 2443{
c2ec175c 2444 struct page *page = vmf->page;
11bac800 2445 struct inode *inode = file_inode(vmf->vma->vm_file);
cca24370 2446
11bac800 2447 file_update_time(vmf->vma->vm_file);
cca24370
MS		 2448 lock_page(page);
2449 if (page->mapping != inode->i_mapping) {
2450 unlock_page(page);
2451 return VM_FAULT_NOPAGE;
2452 }
3be5a52b
MS		 2453
2454 fuse_wait_on_page_writeback(inode, page->index);
cca24370 2455 return VM_FAULT_LOCKED;
3be5a52b
MS		 2456}
2457
f0f37e2f 2458static const struct vm_operations_struct fuse_file_vm_ops = {
3be5a52b
MS		 2459 .close = fuse_vma_close,
2460 .fault = filemap_fault,
f1820361 2461 .map_pages = filemap_map_pages,
3be5a52b
MS		 2462 .page_mkwrite = fuse_page_mkwrite,
2463};
2464
2465static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2466{
55752a3a
MS		 2467 struct fuse_file *ff = file->private_data;
2468
2a9a609a
SH		 2469 /* DAX mmap is superior to direct_io mmap */
2470 if (FUSE_IS_DAX(file_inode(file)))
2471 return fuse_dax_mmap(file, vma);
2472
55752a3a
MS		 2473 if (ff->open_flags & FOPEN_DIRECT_IO) {
2474 /* Can't provide the coherency needed for MAP_SHARED */
2475 if (vma->vm_flags & VM_MAYSHARE)
2476 return -ENODEV;
2477
2478 invalidate_inode_pages2(file->f_mapping);
2479
2480 return generic_file_mmap(file, vma);
2481 }
2482
650b22b9
PE		 2483 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2484 fuse_link_write_file(file);
2485
3be5a52b
MS		 2486 file_accessed(file);
2487 vma->vm_ops = &fuse_file_vm_ops;
b6aeaded
MS		 2488 return 0;
2489}
2490
0b6e9ea0
SF		 2491static int convert_fuse_file_lock(struct fuse_conn *fc,
2492 const struct fuse_file_lock *ffl,
71421259
MS		 2493 struct file_lock *fl)
2494{
2495 switch (ffl->type) {
2496 case F_UNLCK:
2497 break;
2498
2499 case F_RDLCK:
2500 case F_WRLCK:
2501 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2502 ffl->end < ffl->start)
2503 return -EIO;
2504
2505 fl->fl_start = ffl->start;
2506 fl->fl_end = ffl->end;
0b6e9ea0
SF		 2507
2508 /*
9d5b86ac
BC		 2509 * Convert pid into init's pid namespace. The locks API will
2510 * translate it into the caller's pid namespace.
0b6e9ea0
SF		 2511 */
2512 rcu_read_lock();
9d5b86ac 2513 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
0b6e9ea0 2514 rcu_read_unlock();
71421259
MS		 2515 break;
2516
2517 default:
2518 return -EIO;
2519 }
2520 fl->fl_type = ffl->type;
2521 return 0;
2522}
2523
7078187a 2524static void fuse_lk_fill(struct fuse_args *args, struct file *file,
a9ff4f87 2525 const struct file_lock *fl, int opcode, pid_t pid,
7078187a 2526 int flock, struct fuse_lk_in *inarg)
71421259 2527{
6131ffaa 2528 struct inode *inode = file_inode(file);
9c8ef561 2529 struct fuse_conn *fc = get_fuse_conn(inode);
71421259 2530 struct fuse_file *ff = file->private_data;
7078187a
MS		 2531
2532 memset(inarg, 0, sizeof(*inarg));
2533 inarg->fh = ff->fh;
2534 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2535 inarg->lk.start = fl->fl_start;
2536 inarg->lk.end = fl->fl_end;
2537 inarg->lk.type = fl->fl_type;
2538 inarg->lk.pid = pid;
a9ff4f87 2539 if (flock)
7078187a 2540 inarg->lk_flags |= FUSE_LK_FLOCK;
d5b48543
MS		 2541 args->opcode = opcode;
2542 args->nodeid = get_node_id(inode);
2543 args->in_numargs = 1;
2544 args->in_args[0].size = sizeof(*inarg);
2545 args->in_args[0].value = inarg;
71421259
MS		 2546}
2547
2548static int fuse_getlk(struct file *file, struct file_lock *fl)
2549{
6131ffaa 2550 struct inode *inode = file_inode(file);
fcee216b 2551 struct fuse_mount *fm = get_fuse_mount(inode);
7078187a
MS		 2552 FUSE_ARGS(args);
2553 struct fuse_lk_in inarg;
71421259
MS		 2554 struct fuse_lk_out outarg;
2555 int err;
2556
7078187a 2557 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
d5b48543
MS		 2558 args.out_numargs = 1;
2559 args.out_args[0].size = sizeof(outarg);
2560 args.out_args[0].value = &outarg;
fcee216b 2561 err = fuse_simple_request(fm, &args);
71421259 2562 if (!err)
fcee216b 2563 err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
71421259
MS		 2564
2565 return err;
2566}
2567
a9ff4f87 2568static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
71421259 2569{
6131ffaa 2570 struct inode *inode = file_inode(file);
fcee216b 2571 struct fuse_mount *fm = get_fuse_mount(inode);
7078187a
MS		 2572 FUSE_ARGS(args);
2573 struct fuse_lk_in inarg;
71421259 2574 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
0b6e9ea0 2575 struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
fcee216b 2576 pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
71421259
MS		 2577 int err;
2578
8fb47a4f 2579 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
48e90761
MS		 2580 /* NLM needs asynchronous locks, which we don't support yet */
2581 return -ENOLCK;
2582 }
2583
71421259 2584 /* Unlock on close is handled by the flush method */
50f2112c 2585 if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
71421259
MS		 2586 return 0;
2587
0b6e9ea0 2588 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
fcee216b 2589 err = fuse_simple_request(fm, &args);
71421259 2590
a4d27e75
MS		 2591 /* locking is restartable */
2592 if (err == -EINTR)
2593 err = -ERESTARTSYS;
7078187a 2594
71421259
MS		 2595 return err;
2596}
2597
2598static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2599{
6131ffaa 2600 struct inode *inode = file_inode(file);
71421259
MS		 2601 struct fuse_conn *fc = get_fuse_conn(inode);
2602 int err;
2603
48e90761
MS		 2604 if (cmd == F_CANCELLK) {
2605 err = 0;
2606 } else if (cmd == F_GETLK) {
71421259 2607 if (fc->no_lock) {
9d6a8c5c 2608 posix_test_lock(file, fl);
71421259
MS		 2609 err = 0;
2610 } else
2611 err = fuse_getlk(file, fl);
2612 } else {
2613 if (fc->no_lock)
48e90761 2614 err = posix_lock_file(file, fl, NULL);
71421259 2615 else
a9ff4f87 2616 err = fuse_setlk(file, fl, 0);
71421259
MS		 2617 }
2618 return err;
2619}
2620
a9ff4f87
MS		 2621static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2622{
6131ffaa 2623 struct inode *inode = file_inode(file);
a9ff4f87
MS		 2624 struct fuse_conn *fc = get_fuse_conn(inode);
2625 int err;
2626
37fb3a30 2627 if (fc->no_flock) {
4f656367 2628 err = locks_lock_file_wait(file, fl);
a9ff4f87 2629 } else {
37fb3a30
MS		 2630 struct fuse_file *ff = file->private_data;
2631
a9ff4f87 2632 /* emulate flock with POSIX locks */
37fb3a30 2633 ff->flock = true;
a9ff4f87
MS		 2634 err = fuse_setlk(file, fl, 1);
2635 }
2636
2637 return err;
2638}
2639
b2d2272f
MS		 2640static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2641{
2642 struct inode *inode = mapping->host;
fcee216b 2643 struct fuse_mount *fm = get_fuse_mount(inode);
7078187a 2644 FUSE_ARGS(args);
b2d2272f
MS		 2645 struct fuse_bmap_in inarg;
2646 struct fuse_bmap_out outarg;
2647 int err;
2648
fcee216b 2649 if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
b2d2272f
MS		 2650 return 0;
2651
b2d2272f
MS		 2652 memset(&inarg, 0, sizeof(inarg));
2653 inarg.block = block;
2654 inarg.blocksize = inode->i_sb->s_blocksize;
d5b48543
MS		 2655 args.opcode = FUSE_BMAP;
2656 args.nodeid = get_node_id(inode);
2657 args.in_numargs = 1;
2658 args.in_args[0].size = sizeof(inarg);
2659 args.in_args[0].value = &inarg;
2660 args.out_numargs = 1;
2661 args.out_args[0].size = sizeof(outarg);
2662 args.out_args[0].value = &outarg;
fcee216b 2663 err = fuse_simple_request(fm, &args);
b2d2272f 2664 if (err == -ENOSYS)
fcee216b 2665 fm->fc->no_bmap = 1;
b2d2272f
MS		 2666
2667 return err ? 0 : outarg.block;
2668}
2669
0b5da8db
R		 2670static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2671{
2672 struct inode *inode = file->f_mapping->host;
fcee216b 2673 struct fuse_mount *fm = get_fuse_mount(inode);
0b5da8db
R		 2674 struct fuse_file *ff = file->private_data;
2675 FUSE_ARGS(args);
2676 struct fuse_lseek_in inarg = {
2677 .fh = ff->fh,
2678 .offset = offset,
2679 .whence = whence
2680 };
2681 struct fuse_lseek_out outarg;
2682 int err;
2683
fcee216b 2684 if (fm->fc->no_lseek)
0b5da8db
R		 2685 goto fallback;
2686
d5b48543
MS		 2687 args.opcode = FUSE_LSEEK;
2688 args.nodeid = ff->nodeid;
2689 args.in_numargs = 1;
2690 args.in_args[0].size = sizeof(inarg);
2691 args.in_args[0].value = &inarg;
2692 args.out_numargs = 1;
2693 args.out_args[0].size = sizeof(outarg);
2694 args.out_args[0].value = &outarg;
fcee216b 2695 err = fuse_simple_request(fm, &args);
0b5da8db
R		 2696 if (err) {
2697 if (err == -ENOSYS) {
fcee216b 2698 fm->fc->no_lseek = 1;
0b5da8db
R		 2699 goto fallback;
2700 }
2701 return err;
2702 }
2703
2704 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2705
2706fallback:
c6c745b8 2707 err = fuse_update_attributes(inode, file, STATX_SIZE);
0b5da8db
R		 2708 if (!err)
2709 return generic_file_llseek(file, offset, whence);
2710 else
2711 return err;
2712}
2713
965c8e59 2714static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
5559b8f4
MS		 2715{
2716 loff_t retval;
6131ffaa 2717 struct inode *inode = file_inode(file);
5559b8f4 2718
0b5da8db
R		 2719 switch (whence) {
2720 case SEEK_SET:
2721 case SEEK_CUR:
2722 /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
965c8e59 2723 retval = generic_file_llseek(file, offset, whence);
0b5da8db
R		 2724 break;
2725 case SEEK_END:
5955102c 2726 inode_lock(inode);
c6c745b8 2727 retval = fuse_update_attributes(inode, file, STATX_SIZE);
0b5da8db
R		 2728 if (!retval)
2729 retval = generic_file_llseek(file, offset, whence);
5955102c 2730 inode_unlock(inode);
0b5da8db
R		 2731 break;
2732 case SEEK_HOLE:
2733 case SEEK_DATA:
5955102c 2734 inode_lock(inode);
0b5da8db 2735 retval = fuse_lseek(file, offset, whence);
5955102c 2736 inode_unlock(inode);
0b5da8db
R		 2737 break;
2738 default:
2739 retval = -EINVAL;
2740 }
c07c3d19 2741
5559b8f4
MS		 2742 return retval;
2743}
2744
95668a69
TH		 2745/*
2746 * All files which have been polled are linked to RB tree
2747 * fuse_conn->polled_files which is indexed by kh. Walk the tree and
2748 * find the matching one.
2749 */
2750static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2751 struct rb_node **parent_out)
2752{
2753 struct rb_node **link = &fc->polled_files.rb_node;
2754 struct rb_node *last = NULL;
2755
2756 while (*link) {
2757 struct fuse_file *ff;
2758
2759 last = *link;
2760 ff = rb_entry(last, struct fuse_file, polled_node);
2761
2762 if (kh < ff->kh)
2763 link = &last->rb_left;
2764 else if (kh > ff->kh)
2765 link = &last->rb_right;
2766 else
2767 return link;
2768 }
2769
2770 if (parent_out)
2771 *parent_out = last;
2772 return link;
2773}
2774
2775/*
2776 * The file is about to be polled. Make sure it's on the polled_files
2777 * RB tree. Note that files once added to the polled_files tree are
2778 * not removed before the file is released. This is because a file
2779 * polled once is likely to be polled again.
2780 */
2781static void fuse_register_polled_file(struct fuse_conn *fc,
2782 struct fuse_file *ff)
2783{
2784 spin_lock(&fc->lock);
2785 if (RB_EMPTY_NODE(&ff->polled_node)) {
3f649ab7 2786 struct rb_node **link, *parent;
95668a69
TH		 2787
2788 link = fuse_find_polled_node(fc, ff->kh, &parent);
2789 BUG_ON(*link);
2790 rb_link_node(&ff->polled_node, parent, link);
2791 rb_insert_color(&ff->polled_node, &fc->polled_files);
2792 }
2793 spin_unlock(&fc->lock);
2794}
2795
076ccb76 2796__poll_t fuse_file_poll(struct file *file, poll_table *wait)
95668a69 2797{
95668a69 2798 struct fuse_file *ff = file->private_data;
fcee216b 2799 struct fuse_mount *fm = ff->fm;
95668a69
TH		 2800 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2801 struct fuse_poll_out outarg;
7078187a 2802 FUSE_ARGS(args);
95668a69
TH		 2803 int err;
2804
fcee216b 2805 if (fm->fc->no_poll)
95668a69
TH		 2806 return DEFAULT_POLLMASK;
2807
2808 poll_wait(file, &ff->poll_wait, wait);
c71d227f 2809 inarg.events = mangle_poll(poll_requested_events(wait));
95668a69
TH		 2810
2811 /*
2812 * Ask for notification iff there's someone waiting for it.
2813 * The client may ignore the flag and always notify.
2814 */
2815 if (waitqueue_active(&ff->poll_wait)) {
2816 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
fcee216b 2817 fuse_register_polled_file(fm->fc, ff);
95668a69
TH		 2818 }
2819
d5b48543
MS		 2820 args.opcode = FUSE_POLL;
2821 args.nodeid = ff->nodeid;
2822 args.in_numargs = 1;
2823 args.in_args[0].size = sizeof(inarg);
2824 args.in_args[0].value = &inarg;
2825 args.out_numargs = 1;
2826 args.out_args[0].size = sizeof(outarg);
2827 args.out_args[0].value = &outarg;
fcee216b 2828 err = fuse_simple_request(fm, &args);
95668a69
TH		 2829
2830 if (!err)
c71d227f 2831 return demangle_poll(outarg.revents);
95668a69 2832 if (err == -ENOSYS) {
fcee216b 2833 fm->fc->no_poll = 1;
95668a69
TH		 2834 return DEFAULT_POLLMASK;
2835 }
a9a08845 2836 return EPOLLERR;
95668a69 2837}
08cbf542 2838EXPORT_SYMBOL_GPL(fuse_file_poll);
95668a69
TH		 2839
2840/*
2841 * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
2842 * wakes up the poll waiters.
2843 */
2844int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2845 struct fuse_notify_poll_wakeup_out *outarg)
2846{
2847 u64 kh = outarg->kh;
2848 struct rb_node **link;
2849
2850 spin_lock(&fc->lock);
2851
2852 link = fuse_find_polled_node(fc, kh, NULL);
2853 if (*link) {
2854 struct fuse_file *ff;
2855
2856 ff = rb_entry(*link, struct fuse_file, polled_node);
2857 wake_up_interruptible_sync(&ff->poll_wait);
2858 }
2859
2860 spin_unlock(&fc->lock);
2861 return 0;
2862}
2863
efb9fa9e
MP		 2864static void fuse_do_truncate(struct file *file)
2865{
2866 struct inode *inode = file->f_mapping->host;
2867 struct iattr attr;
2868
2869 attr.ia_valid = ATTR_SIZE;
2870 attr.ia_size = i_size_read(inode);
2871
2872 attr.ia_file = file;
2873 attr.ia_valid |= ATTR_FILE;
2874
62490330 2875 fuse_do_setattr(file_dentry(file), &attr, file);
efb9fa9e
MP		 2876}
2877
5da784cc 2878static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
e5c5f05d 2879{
5da784cc 2880 return round_up(off, fc->max_pages << PAGE_SHIFT);
e5c5f05d
MP		 2881}
2882
4273b793 2883static ssize_t
c8b8e32d 2884fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
4273b793 2885{
9d5722b7 2886 DECLARE_COMPLETION_ONSTACK(wait);
4273b793 2887 ssize_t ret = 0;
60b9df7a
MS		 2888 struct file *file = iocb->ki_filp;
2889 struct fuse_file *ff = file->private_data;
4273b793 2890 loff_t pos = 0;
bcba24cc
MP		 2891 struct inode *inode;
2892 loff_t i_size;
933a3752 2893 size_t count = iov_iter_count(iter), shortened = 0;
c8b8e32d 2894 loff_t offset = iocb->ki_pos;
36cf66ed 2895 struct fuse_io_priv *io;
4273b793 2896
4273b793 2897 pos = offset;
bcba24cc
MP		 2898 inode = file->f_mapping->host;
2899 i_size = i_size_read(inode);
4273b793 2900
933a3752 2901 if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
9fe55eea
SW		 2902 return 0;
2903
bcba24cc 2904 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
36cf66ed
MP		 2905 if (!io)
2906 return -ENOMEM;
bcba24cc 2907 spin_lock_init(&io->lock);
744742d6 2908 kref_init(&io->refcnt);
bcba24cc
MP		 2909 io->reqs = 1;
2910 io->bytes = -1;
2911 io->size = 0;
2912 io->offset = offset;
6f673763 2913 io->write = (iov_iter_rw(iter) == WRITE);
bcba24cc 2914 io->err = 0;
bcba24cc
MP		 2915 /*
2916 * By default, we want to optimize all I/Os with async request
60b9df7a 2917 * submission to the client filesystem if supported.
bcba24cc 2918 */
69456535 2919 io->async = ff->fm->fc->async_dio;
bcba24cc 2920 io->iocb = iocb;
7879c4e5 2921 io->blocking = is_sync_kiocb(iocb);
bcba24cc 2922
933a3752
AV		 2923 /* optimization for short read */
2924 if (io->async && !io->write && offset + count > i_size) {
69456535 2925 iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
933a3752
AV		 2926 shortened = count - iov_iter_count(iter);
2927 count -= shortened;
2928 }
2929
bcba24cc 2930 /*
7879c4e5
AS		 2931 * We cannot asynchronously extend the size of a file.
2932 * In such case the aio will behave exactly like sync io.
bcba24cc 2933 */
933a3752 2934 if ((offset + count > i_size) && io->write)
7879c4e5 2935 io->blocking = true;
4273b793 2936
7879c4e5 2937 if (io->async && io->blocking) {
744742d6
SF		 2938 /*
2939 * Additional reference to keep io around after
2940 * calling fuse_aio_complete()
2941 */
2942 kref_get(&io->refcnt);
9d5722b7 2943 io->done = &wait;
744742d6 2944 }
9d5722b7 2945
6f673763 2946 if (iov_iter_rw(iter) == WRITE) {
6b775b18 2947 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
fa5eee57 2948 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
812408fb 2949 } else {
d22a943f 2950 ret = __fuse_direct_read(io, iter, &pos);
812408fb 2951 }
933a3752 2952 iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
36cf66ed 2953
bcba24cc 2954 if (io->async) {
ebacb812
LC		 2955 bool blocking = io->blocking;
2956
bcba24cc
MP		 2957 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2958
2959 /* we have a non-extending, async request, so return */
ebacb812 2960 if (!blocking)
bcba24cc
MP		 2961 return -EIOCBQUEUED;
2962
9d5722b7
CH		 2963 wait_for_completion(&wait);
2964 ret = fuse_get_res_by_io(io);
bcba24cc
MP		 2965 }
2966
744742d6 2967 kref_put(&io->refcnt, fuse_io_release);
9d5722b7 2968
6f673763 2969 if (iov_iter_rw(iter) == WRITE) {
d347739a 2970 fuse_write_update_attr(inode, pos, ret);
15352405 2971 /* For extending writes we already hold exclusive lock */
d347739a 2972 if (ret < 0 && offset + count > i_size)
efb9fa9e
MP		 2973 fuse_do_truncate(file);
2974 }
4273b793
AA		 2975
2976 return ret;
2977}
2978
26eb3bae
MS		 2979static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
2980{
e388164e 2981 int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
26eb3bae
MS		 2982
2983 if (!err)
2984 fuse_sync_writes(inode);
2985
2986 return err;
2987}
2988
cdadb11c
MS		 2989static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
2990 loff_t length)
05ba1f08
AP		 2991{
2992 struct fuse_file *ff = file->private_data;
1c68271c 2993 struct inode *inode = file_inode(file);
0ab08f57 2994 struct fuse_inode *fi = get_fuse_inode(inode);
fcee216b 2995 struct fuse_mount *fm = ff->fm;
7078187a 2996 FUSE_ARGS(args);
05ba1f08
AP		 2997 struct fuse_fallocate_in inarg = {
2998 .fh = ff->fh,
2999 .offset = offset,
3000 .length = length,
3001 .mode = mode
3002 };
3003 int err;
44361e8c
MS		 3004 bool block_faults = FUSE_IS_DAX(inode) &&
3005 (!(mode & FALLOC_FL_KEEP_SIZE) ||
3006 (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)));
6ae330ca 3007
6b1bdb56
RJ		 3008 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3009 FALLOC_FL_ZERO_RANGE))
4adb8302
MS		 3010 return -EOPNOTSUPP;
3011
fcee216b 3012 if (fm->fc->no_fallocate)
519c6040
MS		 3013 return -EOPNOTSUPP;
3014
44361e8c
MS		 3015 inode_lock(inode);
3016 if (block_faults) {
3017 filemap_invalidate_lock(inode->i_mapping);
3018 err = fuse_dax_break_layouts(inode, 0, 0);
3019 if (err)
3020 goto out;
3021 }
6ae330ca 3022
44361e8c
MS		 3023 if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) {
3024 loff_t endbyte = offset + length - 1;
26eb3bae 3025
44361e8c
MS		 3026 err = fuse_writeback_range(inode, offset, endbyte);
3027 if (err)
3028 goto out;
3634a632
BF		 3029 }
3030
0cbade02
LB		 3031 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
3032 offset + length > i_size_read(inode)) {
3033 err = inode_newsize_ok(inode, offset + length);
3034 if (err)
35d6fcbb 3035 goto out;
0cbade02
LB		 3036 }
3037
4a6f278d
MS		 3038 err = file_modified(file);
3039 if (err)
3040 goto out;
3041
0ab08f57
MP		 3042 if (!(mode & FALLOC_FL_KEEP_SIZE))
3043 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3044
d5b48543
MS		 3045 args.opcode = FUSE_FALLOCATE;
3046 args.nodeid = ff->nodeid;
3047 args.in_numargs = 1;
3048 args.in_args[0].size = sizeof(inarg);
3049 args.in_args[0].value = &inarg;
fcee216b 3050 err = fuse_simple_request(fm, &args);
519c6040 3051 if (err == -ENOSYS) {
fcee216b 3052 fm->fc->no_fallocate = 1;
519c6040
MS		 3053 err = -EOPNOTSUPP;
3054 }
bee6c307
BF		 3055 if (err)
3056 goto out;
3057
3058 /* we could have extended the file */
b0aa7606 3059 if (!(mode & FALLOC_FL_KEEP_SIZE)) {
20235b43 3060 if (fuse_write_update_attr(inode, offset + length, length))
93d2269d 3061 file_update_time(file);
b0aa7606 3062 }
bee6c307 3063
6b1bdb56 3064 if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))
bee6c307
BF		 3065 truncate_pagecache_range(inode, offset, offset + length - 1);
3066
fa5eee57 3067 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
bee6c307 3068
3634a632 3069out:
0ab08f57
MP		 3070 if (!(mode & FALLOC_FL_KEEP_SIZE))
3071 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3072
6ae330ca 3073 if (block_faults)
8bcbbe9c 3074 filemap_invalidate_unlock(inode->i_mapping);
6ae330ca 3075
44361e8c 3076 inode_unlock(inode);
3634a632 3077
5c791fe1
MS		 3078 fuse_flush_time_update(inode);
3079
05ba1f08
AP		 3080 return err;
3081}
05ba1f08 3082
64bf5ff5
DC		 3083static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
3084 struct file *file_out, loff_t pos_out,
3085 size_t len, unsigned int flags)
88bc7d50
NV		 3086{
3087 struct fuse_file *ff_in = file_in->private_data;
3088 struct fuse_file *ff_out = file_out->private_data;
a2bc9236 3089 struct inode *inode_in = file_inode(file_in);
88bc7d50
NV		 3090 struct inode *inode_out = file_inode(file_out);
3091 struct fuse_inode *fi_out = get_fuse_inode(inode_out);
fcee216b
MR		 3092 struct fuse_mount *fm = ff_in->fm;
3093 struct fuse_conn *fc = fm->fc;
88bc7d50
NV		 3094 FUSE_ARGS(args);
3095 struct fuse_copy_file_range_in inarg = {
3096 .fh_in = ff_in->fh,
3097 .off_in = pos_in,
3098 .nodeid_out = ff_out->nodeid,
3099 .fh_out = ff_out->fh,
3100 .off_out = pos_out,
3101 .len = len,
3102 .flags = flags
3103 };
3104 struct fuse_write_out outarg;
3105 ssize_t err;
3106 /* mark unstable when write-back is not used, and file_out gets
3107 * extended */
3108 bool is_unstable = (!fc->writeback_cache) &&
3109 ((pos_out + len) > inode_out->i_size);
3110
3111 if (fc->no_copy_file_range)
3112 return -EOPNOTSUPP;
3113
5dae222a
AG		 3114 if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3115 return -EXDEV;
3116
2c4656df
MS		 3117 inode_lock(inode_in);
3118 err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
3119 inode_unlock(inode_in);
3120 if (err)
3121 return err;
a2bc9236 3122
88bc7d50
NV		 3123 inode_lock(inode_out);
3124
fe0da9c0
AG		 3125 err = file_modified(file_out);
3126 if (err)
3127 goto out;
3128
9b46418c
MS		 3129 /*
3130 * Write out dirty pages in the destination file before sending the COPY
3131 * request to userspace. After the request is completed, truncate off
3132 * pages (including partial ones) from the cache that have been copied,
3133 * since these contain stale data at that point.
3134 *
3135 * This should be mostly correct, but if the COPY writes to partial
3136 * pages (at the start or end) and the parts not covered by the COPY are
3137 * written through a memory map after calling fuse_writeback_range(),
3138 * then these partial page modifications will be lost on truncation.
3139 *
3140 * It is unlikely that someone would rely on such mixed style
3141 * modifications. Yet this does give less guarantees than if the
3142 * copying was performed with write(2).
3143 *
8bcbbe9c 3144 * To fix this a mapping->invalidate_lock could be used to prevent new
9b46418c
MS		 3145 * faults while the copy is ongoing.
3146 */
2c4656df
MS		 3147 err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
3148 if (err)
3149 goto out;
88bc7d50
NV		 3150
3151 if (is_unstable)
3152 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3153
d5b48543
MS		 3154 args.opcode = FUSE_COPY_FILE_RANGE;
3155 args.nodeid = ff_in->nodeid;
3156 args.in_numargs = 1;
3157 args.in_args[0].size = sizeof(inarg);
3158 args.in_args[0].value = &inarg;
3159 args.out_numargs = 1;
3160 args.out_args[0].size = sizeof(outarg);
3161 args.out_args[0].value = &outarg;
fcee216b 3162 err = fuse_simple_request(fm, &args);
88bc7d50
NV		 3163 if (err == -ENOSYS) {
3164 fc->no_copy_file_range = 1;
3165 err = -EOPNOTSUPP;
3166 }
3167 if (err)
3168 goto out;
3169
9b46418c
MS		 3170 truncate_inode_pages_range(inode_out->i_mapping,
3171 ALIGN_DOWN(pos_out, PAGE_SIZE),
3172 ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1);
3173
20235b43
MS		 3174 file_update_time(file_out);
3175 fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size);
88bc7d50
NV		 3176
3177 err = outarg.size;
3178out:
3179 if (is_unstable)
3180 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3181
3182 inode_unlock(inode_out);
fe0da9c0 3183 file_accessed(file_in);
88bc7d50 3184
5c791fe1
MS		 3185 fuse_flush_time_update(inode_out);
3186
88bc7d50
NV		 3187 return err;
3188}
3189
64bf5ff5
DC		 3190static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3191 struct file *dst_file, loff_t dst_off,
3192 size_t len, unsigned int flags)
3193{
3194 ssize_t ret;
3195
3196 ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3197 len, flags);
3198
5dae222a 3199 if (ret == -EOPNOTSUPP || ret == -EXDEV)
64bf5ff5
DC		 3200 ret = generic_copy_file_range(src_file, src_off, dst_file,
3201 dst_off, len, flags);
3202 return ret;
3203}
3204
4b6f5d20 3205static const struct file_operations fuse_file_operations = {
5559b8f4 3206 .llseek = fuse_file_llseek,
37c20f16 3207 .read_iter = fuse_file_read_iter,
84c3d55c 3208 .write_iter = fuse_file_write_iter,
b6aeaded
MS		 3209 .mmap = fuse_file_mmap,
3210 .open = fuse_open,
3211 .flush = fuse_flush,
3212 .release = fuse_release,
3213 .fsync = fuse_fsync,
71421259 3214 .lock = fuse_file_lock,
2a9a609a 3215 .get_unmapped_area = thp_get_unmapped_area,
a9ff4f87 3216 .flock = fuse_file_flock,
3c3db095
MS		 3217 .splice_read = generic_file_splice_read,
3218 .splice_write = iter_file_splice_write,
59efec7b
TH		 3219 .unlocked_ioctl = fuse_file_ioctl,
3220 .compat_ioctl = fuse_file_compat_ioctl,
95668a69 3221 .poll = fuse_file_poll,
05ba1f08 3222 .fallocate = fuse_file_fallocate,
d4136d60 3223 .copy_file_range = fuse_copy_file_range,
413ef8cb
MS		 3224};
3225
f5e54d6e 3226static const struct address_space_operations fuse_file_aops = {
5efd00e4 3227 .read_folio = fuse_read_folio,
76a0294e 3228 .readahead = fuse_readahead,
3be5a52b 3229 .writepage = fuse_writepage,
26d614df 3230 .writepages = fuse_writepages,
2bf06b8e 3231 .launder_folio = fuse_launder_folio,
187c82cb 3232 .dirty_folio = filemap_dirty_folio,
b2d2272f 3233 .bmap = fuse_bmap,
4273b793 3234 .direct_IO = fuse_direct_IO,
6b12c1b3
PE		 3235 .write_begin = fuse_write_begin,
3236 .write_end = fuse_write_end,
b6aeaded
MS		 3237};
3238
93a497b9 3239void fuse_init_file_inode(struct inode *inode, unsigned int flags)
b6aeaded 3240{
ab2257e9
MS		 3241 struct fuse_inode *fi = get_fuse_inode(inode);
3242
45323fb7
MS		 3243 inode->i_fop = &fuse_file_operations;
3244 inode->i_data.a_ops = &fuse_file_aops;
ab2257e9
MS		 3245
3246 INIT_LIST_HEAD(&fi->write_files);
3247 INIT_LIST_HEAD(&fi->queued_writes);
3248 fi->writectr = 0;
3249 init_waitqueue_head(&fi->page_waitq);
6b2fb799 3250 fi->writepages = RB_ROOT;
c2d0ad00
VG		 3251
3252 if (IS_ENABLED(CONFIG_FUSE_DAX))
93a497b9 3253 fuse_dax_inode_init(inode, flags);
b6aeaded 3254}
Linux 6.x block layer and io_uring tree(s)