projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'pci-v6.16-fixes-3' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci
[linux-block.git] / fs / fuse / file.c
CommitLineData
b6aeaded
MS		 1/*
2 FUSE: Filesystem in Userspace
1729a16c 3 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
b6aeaded
MS		 4
5 This program can be distributed under the terms of the GNU GPL.
6 See the file COPYING.
7*/
8
9#include "fuse_i.h"
10
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/kernel.h>
e8edc6e0 14#include <linux/sched.h>
7a36094d 15#include <linux/sched/signal.h>
08cbf542 16#include <linux/module.h>
478e0841 17#include <linux/swap.h>
3634a632 18#include <linux/falloc.h>
e2e40f2c 19#include <linux/uio.h>
31070f6c 20#include <linux/fs.h>
5970e15d 21#include <linux/filelock.h>
705bcfcb 22#include <linux/splice.h>
2e3f7dd0 23#include <linux/task_io_accounting_ops.h>
b6aeaded 24
b9d54c6f
MS		 25static int fuse_send_open(struct fuse_mount *fm, u64 nodeid,
26 unsigned int open_flags, int opcode,
27 struct fuse_open_out *outargp)
b6aeaded 28{
b6aeaded 29 struct fuse_open_in inarg;
7078187a 30 FUSE_ARGS(args);
fd72faac
MS		 31
32 memset(&inarg, 0, sizeof(inarg));
b9d54c6f 33 inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
fcee216b 34 if (!fm->fc->atomic_o_trunc)
6ff958ed 35 inarg.flags &= ~O_TRUNC;
643a666a
VG		 36
37 if (fm->fc->handle_killpriv_v2 &&
38 (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) {
39 inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID;
40 }
41
d5b48543
MS		 42 args.opcode = opcode;
43 args.nodeid = nodeid;
44 args.in_numargs = 1;
45 args.in_args[0].size = sizeof(inarg);
46 args.in_args[0].value = &inarg;
47 args.out_numargs = 1;
48 args.out_args[0].size = sizeof(*outargp);
49 args.out_args[0].value = outargp;
fd72faac 50
fcee216b 51 return fuse_simple_request(fm, &args);
fd72faac
MS		 52}
53
e26ee4ef 54struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release)
fd72faac
MS		 55{
56 struct fuse_file *ff;
6b2db28a 57
dc69e98c 58 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
6b2db28a
TH		 59 if (unlikely(!ff))
60 return NULL;
61
fcee216b 62 ff->fm = fm;
e26ee4ef 63 if (release) {
fc8ff397
AG		 64 ff->args = kzalloc(sizeof(*ff->args), GFP_KERNEL_ACCOUNT);
65 if (!ff->args) {
e26ee4ef
AG		 66 kfree(ff);
67 return NULL;
68 }
fd72faac 69 }
6b2db28a
TH		 70
71 INIT_LIST_HEAD(&ff->write_entry);
4e8c2eb5 72 refcount_set(&ff->count, 1);
6b2db28a
TH		 73 RB_CLEAR_NODE(&ff->polled_node);
74 init_waitqueue_head(&ff->poll_wait);
75
fcee216b 76 ff->kh = atomic64_inc_return(&fm->fc->khctr);
6b2db28a 77
fd72faac
MS		 78 return ff;
79}
80
81void fuse_file_free(struct fuse_file *ff)
82{
fc8ff397 83 kfree(ff->args);
fd72faac
MS		 84 kfree(ff);
85}
86
267d8444 87static struct fuse_file *fuse_file_get(struct fuse_file *ff)
c756e0a4 88{
4e8c2eb5 89 refcount_inc(&ff->count);
c756e0a4
MS		 90 return ff;
91}
92
fcee216b 93static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
4cb54866 94 int error)
819c4b3b 95{
4cb54866
MS		 96 struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
97
98 iput(ra->inode);
99 kfree(ra);
819c4b3b
MS		 100}
101
e26ee4ef 102static void fuse_file_put(struct fuse_file *ff, bool sync)
c756e0a4 103{
4e8c2eb5 104 if (refcount_dec_and_test(&ff->count)) {
fc8ff397 105 struct fuse_release_args *ra = &ff->args->release_args;
e26ee4ef 106 struct fuse_args *args = (ra ? &ra->args : NULL);
8b0797a4 107
cb098dd2
AG		 108 if (ra && ra->inode)
109 fuse_file_io_release(ff, ra->inode);
110
e26ee4ef
AG		 111 if (!args) {
112 /* Do nothing when server does not implement 'open' */
7678ac50 113 } else if (sync) {
fcee216b
MR		 114 fuse_simple_request(ff->fm, args);
115 fuse_release_end(ff->fm, args, 0);
5a18ec17 116 } else {
4cb54866 117 args->end = fuse_release_end;
fcee216b 118 if (fuse_simple_background(ff->fm, args,
4cb54866 119 GFP_KERNEL | __GFP_NOFAIL))
fcee216b 120 fuse_release_end(ff->fm, args, -ENOTCONN);
5a18ec17 121 }
c756e0a4
MS		 122 kfree(ff);
123 }
124}
125
b9d54c6f
MS		 126struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
127 unsigned int open_flags, bool isdir)
91fe96b4 128{
fcee216b 129 struct fuse_conn *fc = fm->fc;
91fe96b4 130 struct fuse_file *ff;
91fe96b4 131 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
e26ee4ef 132 bool open = isdir ? !fc->no_opendir : !fc->no_open;
91fe96b4 133
e26ee4ef 134 ff = fuse_file_alloc(fm, open);
91fe96b4 135 if (!ff)
b9d54c6f 136 return ERR_PTR(-ENOMEM);
91fe96b4 137
7678ac50 138 ff->fh = 0;
fabf7e02
CA		 139 /* Default for no-open */
140 ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
e26ee4ef 141 if (open) {
fc8ff397
AG		 142 /* Store outarg for fuse_finish_open() */
143 struct fuse_open_out *outargp = &ff->args->open_outarg;
7678ac50
AG		 144 int err;
145
fc8ff397 146 err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp);
7678ac50 147 if (!err) {
fc8ff397
AG		 148 ff->fh = outargp->fh;
149 ff->open_flags = outargp->open_flags;
d9a9ea94 150 } else if (err != -ENOSYS) {
7678ac50 151 fuse_file_free(ff);
b9d54c6f 152 return ERR_PTR(err);
7678ac50 153 } else {
e26ee4ef 154 /* No release needed */
fc8ff397
AG		 155 kfree(ff->args);
156 ff->args = NULL;
d9a9ea94
CA		 157 if (isdir)
158 fc->no_opendir = 1;
159 else
160 fc->no_open = 1;
7678ac50 161 }
91fe96b4
MS		 162 }
163
164 if (isdir)
7678ac50 165 ff->open_flags &= ~FOPEN_DIRECT_IO;
91fe96b4 166
91fe96b4 167 ff->nodeid = nodeid;
91fe96b4 168
b9d54c6f
MS		 169 return ff;
170}
171
172int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
173 bool isdir)
174{
175 struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir);
176
177 if (!IS_ERR(ff))
178 file->private_data = ff;
179
180 return PTR_ERR_OR_ZERO(ff);
91fe96b4 181}
08cbf542 182EXPORT_SYMBOL_GPL(fuse_do_open);
91fe96b4 183
650b22b9
PE		 184static void fuse_link_write_file(struct file *file)
185{
186 struct inode *inode = file_inode(file);
650b22b9
PE		 187 struct fuse_inode *fi = get_fuse_inode(inode);
188 struct fuse_file *ff = file->private_data;
189 /*
190 * file may be written through mmap, so chain it onto the
191 * inodes's write_file list
192 */
f15ecfef 193 spin_lock(&fi->lock);
650b22b9
PE		 194 if (list_empty(&ff->write_entry))
195 list_add(&ff->write_entry, &fi->write_files);
f15ecfef 196 spin_unlock(&fi->lock);
650b22b9
PE		 197}
198
d2c487f1 199int fuse_finish_open(struct inode *inode, struct file *file)
fd72faac 200{
c7b7143c 201 struct fuse_file *ff = file->private_data;
a0822c55 202 struct fuse_conn *fc = get_fuse_conn(inode);
cb098dd2
AG		 203 int err;
204
205 err = fuse_file_io_open(file, inode);
206 if (err)
207 return err;
c7b7143c 208
bbd84f33
KS		 209 if (ff->open_flags & FOPEN_STREAM)
210 stream_open(inode, file);
211 else if (ff->open_flags & FOPEN_NONSEEKABLE)
a7c1b990 212 nonseekable_open(inode, file);
76224355 213
4d99ff8f
PE		 214 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
215 fuse_link_write_file(file);
d2c487f1
AG		 216
217 return 0;
fd72faac
MS		 218}
219
0c9d7089
AG		 220static void fuse_truncate_update_attr(struct inode *inode, struct file *file)
221{
222 struct fuse_conn *fc = get_fuse_conn(inode);
223 struct fuse_inode *fi = get_fuse_inode(inode);
224
225 spin_lock(&fi->lock);
226 fi->attr_version = atomic64_inc_return(&fc->attr_version);
227 i_size_write(inode, 0);
228 spin_unlock(&fi->lock);
229 file_update_time(file);
230 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
231}
232
7de64d52 233static int fuse_open(struct inode *inode, struct file *file)
fd72faac 234{
fcee216b 235 struct fuse_mount *fm = get_fuse_mount(inode);
d2c487f1 236 struct fuse_inode *fi = get_fuse_inode(inode);
fcee216b 237 struct fuse_conn *fc = fm->fc;
d2c487f1 238 struct fuse_file *ff;
b6aeaded 239 int err;
0c9d7089
AG		 240 bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc;
241 bool is_wb_truncate = is_truncate && fc->writeback_cache;
242 bool dax_truncate = is_truncate && FUSE_IS_DAX(inode);
b6aeaded 243
5d069dbe
MS		 244 if (fuse_is_bad(inode))
245 return -EIO;
246
b6aeaded
MS		 247 err = generic_file_open(inode, file);
248 if (err)
249 return err;
250
2fdbb8dd 251 if (is_wb_truncate || dax_truncate)
5955102c 252 inode_lock(inode);
75caeecd 253
6ae330ca 254 if (dax_truncate) {
8bcbbe9c 255 filemap_invalidate_lock(inode->i_mapping);
7851bf64 256 err = fuse_dax_break_layouts(inode, 0, -1);
6ae330ca 257 if (err)
2fdbb8dd 258 goto out_inode_unlock;
6ae330ca 259 }
b6aeaded 260
2fdbb8dd
MS		 261 if (is_wb_truncate || dax_truncate)
262 fuse_set_nowrite(inode);
263
7de64d52 264 err = fuse_do_open(fm, get_node_id(inode), file, false);
0c9d7089 265 if (!err) {
d2c487f1
AG		 266 ff = file->private_data;
267 err = fuse_finish_open(inode, file);
268 if (err)
269 fuse_sync_release(fi, ff, file->f_flags);
270 else if (is_truncate)
0c9d7089
AG		 271 fuse_truncate_update_attr(inode, file);
272 }
91fe96b4 273
2fdbb8dd
MS		 274 if (is_wb_truncate || dax_truncate)
275 fuse_release_nowrite(inode);
276 if (!err) {
0c9d7089 277 if (is_truncate)
2fdbb8dd
MS		 278 truncate_pagecache(inode, 0);
279 else if (!(ff->open_flags & FOPEN_KEEP_CACHE))
280 invalidate_inode_pages2(inode->i_mapping);
281 }
6ae330ca 282 if (dax_truncate)
8bcbbe9c 283 filemap_invalidate_unlock(inode->i_mapping);
2fdbb8dd
MS		 284out_inode_unlock:
285 if (is_wb_truncate || dax_truncate)
5955102c 286 inode_unlock(inode);
75caeecd
MP		 287
288 return err;
b6aeaded
MS		 289}
290
ebf84d0c 291static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
e26ee4ef 292 unsigned int flags, int opcode, bool sync)
64c6d8ed 293{
fcee216b 294 struct fuse_conn *fc = ff->fm->fc;
fc8ff397 295 struct fuse_release_args *ra = &ff->args->release_args;
b6aeaded 296
4a90451b
AG		 297 if (fuse_file_passthrough(ff))
298 fuse_passthrough_release(ff, fuse_inode_backing(fi));
b6aeaded 299
f15ecfef
KT		 300 /* Inode is NULL on error path of fuse_create_open() */
301 if (likely(fi)) {
302 spin_lock(&fi->lock);
303 list_del(&ff->write_entry);
304 spin_unlock(&fi->lock);
305 }
8b0797a4 306 spin_lock(&fc->lock);
8b0797a4
MS		 307 if (!RB_EMPTY_NODE(&ff->polled_node))
308 rb_erase(&ff->polled_node, &fc->polled_files);
309 spin_unlock(&fc->lock);
310
357ccf2b 311 wake_up_interruptible_all(&ff->poll_wait);
8b0797a4 312
e26ee4ef
AG		 313 if (!ra)
314 return;
315
fc8ff397
AG		 316 /* ff->args was used for open outarg */
317 memset(ff->args, 0, sizeof(*ff->args));
4cb54866
MS		 318 ra->inarg.fh = ff->fh;
319 ra->inarg.flags = flags;
320 ra->args.in_numargs = 1;
321 ra->args.in_args[0].size = sizeof(struct fuse_release_in);
322 ra->args.in_args[0].value = &ra->inarg;
323 ra->args.opcode = opcode;
324 ra->args.nodeid = ff->nodeid;
325 ra->args.force = true;
326 ra->args.nocreds = true;
e26ee4ef
AG		 327
328 /*
329 * Hold inode until release is finished.
330 * From fuse_sync_release() the refcount is 1 and everything's
331 * synchronous, so we are fine with not doing igrab() here.
332 */
333 ra->inode = sync ? NULL : igrab(&fi->inode);
fd72faac
MS		 334}
335
b9d54c6f
MS		 336void fuse_file_release(struct inode *inode, struct fuse_file *ff,
337 unsigned int open_flags, fl_owner_t id, bool isdir)
fd72faac 338{
b9d54c6f 339 struct fuse_inode *fi = get_fuse_inode(inode);
fc8ff397 340 struct fuse_release_args *ra = &ff->args->release_args;
2e64ff15 341 int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
6b2db28a 342
e26ee4ef 343 fuse_prepare_release(fi, ff, open_flags, opcode, false);
6b2db28a 344
e26ee4ef 345 if (ra && ff->flock) {
4cb54866 346 ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
b9d54c6f 347 ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id);
37fb3a30 348 }
6b2db28a 349
6b2db28a
TH		 350 /*
351 * Normally this will send the RELEASE request, however if
352 * some asynchronous READ or WRITE requests are outstanding,
353 * the sending will be delayed.
5a18ec17
MS		 354 *
355 * Make the release synchronous if this is a fuseblk mount,
356 * synchronous RELEASE is allowed (and desirable) in this case
357 * because the server can be trusted not to screw up.
6b2db28a 358 */
e26ee4ef 359 fuse_file_put(ff, ff->fm->fc->destroy);
b6aeaded
MS		 360}
361
b9d54c6f
MS		 362void fuse_release_common(struct file *file, bool isdir)
363{
364 fuse_file_release(file_inode(file), file->private_data, file->f_flags,
365 (fl_owner_t) file, isdir);
366}
367
04730fef
MS		 368static int fuse_release(struct inode *inode, struct file *file)
369{
035ff33c
MS		 370 struct fuse_conn *fc = get_fuse_conn(inode);
371
372 /*
373 * Dirty pages might remain despite write_inode_now() call from
374 * fuse_flush() due to writes racing with the close.
375 */
376 if (fc->writeback_cache)
377 write_inode_now(inode, 1);
378
2e64ff15 379 fuse_release_common(file, false);
8b0797a4
MS		 380
381 /* return value is ignored by VFS */
382 return 0;
383}
384
54d601cb
MS		 385void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff,
386 unsigned int flags)
8b0797a4 387{
4e8c2eb5 388 WARN_ON(refcount_read(&ff->count) > 1);
e26ee4ef
AG		 389 fuse_prepare_release(fi, ff, flags, FUSE_RELEASE, true);
390 fuse_file_put(ff, true);
04730fef 391}
08cbf542 392EXPORT_SYMBOL_GPL(fuse_sync_release);
04730fef 393
71421259 394/*
9c8ef561
MS		 395 * Scramble the ID space with XTEA, so that the value of the files_struct
396 * pointer is not exposed to userspace.
71421259 397 */
f3332114 398u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
71421259 399{
9c8ef561
MS		 400 u32 *k = fc->scramble_key;
401 u64 v = (unsigned long) id;
402 u32 v0 = v;
403 u32 v1 = v >> 32;
404 u32 sum = 0;
405 int i;
406
407 for (i = 0; i < 32; i++) {
408 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
409 sum += 0x9E3779B9;
410 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
411 }
412
413 return (u64) v0 + ((u64) v1 << 32);
71421259
MS		 414}
415
33826ebb
MS		 416struct fuse_writepage_args {
417 struct fuse_io_args ia;
33826ebb 418 struct list_head queue_entry;
33826ebb 419 struct inode *inode;
660585b5 420 struct fuse_sync_bucket *bucket;
33826ebb
MS		 421};
422
fe38d7df
MP		 423/*
424 * Wait for all pending writepages on the inode to finish.
425 *
426 * This is currently done by blocking further writes with FUSE_NOWRITE
427 * and waiting for all sent writes to complete.
428 *
429 * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
430 * could conflict with truncation.
431 */
432static void fuse_sync_writes(struct inode *inode)
433{
434 fuse_set_nowrite(inode);
435 fuse_release_nowrite(inode);
436}
437
91ec6c85 438static int fuse_flush(struct file *file, fl_owner_t id)
5a8bee63 439{
91ec6c85 440 struct inode *inode = file_inode(file);
5a8bee63 441 struct fuse_mount *fm = get_fuse_mount(inode);
91ec6c85
MS		 442 struct fuse_file *ff = file->private_data;
443 struct fuse_flush_in inarg;
444 FUSE_ARGS(args);
445 int err;
446
447 if (fuse_is_bad(inode))
448 return -EIO;
449
450 if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache)
451 return 0;
a390ccb3 452
1e18bda8 453 err = write_inode_now(inode, 1);
fe38d7df 454 if (err)
91ec6c85 455 return err;
fe38d7df 456
91ec6c85 457 err = filemap_check_errors(file->f_mapping);
9ebce595 458 if (err)
91ec6c85 459 return err;
9ebce595 460
614c026e 461 err = 0;
fcee216b 462 if (fm->fc->no_flush)
614c026e
MS		 463 goto inval_attr_out;
464
91ec6c85
MS		 465 memset(&inarg, 0, sizeof(inarg));
466 inarg.fh = ff->fh;
467 inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
468 args.opcode = FUSE_FLUSH;
469 args.nodeid = get_node_id(inode);
470 args.in_numargs = 1;
471 args.in_args[0].size = sizeof(inarg);
472 args.in_args[0].value = &inarg;
473 args.force = true;
474
475 err = fuse_simple_request(fm, &args);
b6aeaded 476 if (err == -ENOSYS) {
fcee216b 477 fm->fc->no_flush = 1;
b6aeaded
MS		 478 err = 0;
479 }
cf576c58
EG		 480
481inval_attr_out:
482 /*
483 * In memory i_blocks is not maintained by fuse, if writeback cache is
484 * enabled, i_blocks from cached attr may not be accurate.
485 */
fcee216b 486 if (!err && fm->fc->writeback_cache)
fa5eee57 487 fuse_invalidate_attr_mask(inode, STATX_BLOCKS);
b6aeaded
MS		 488 return err;
489}
490
02c24a82 491int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
a9c2d1e8 492 int datasync, int opcode)
b6aeaded 493{
7ea80859 494 struct inode *inode = file->f_mapping->host;
fcee216b 495 struct fuse_mount *fm = get_fuse_mount(inode);
b6aeaded 496 struct fuse_file *ff = file->private_data;
7078187a 497 FUSE_ARGS(args);
b6aeaded 498 struct fuse_fsync_in inarg;
a9c2d1e8
MS		 499
500 memset(&inarg, 0, sizeof(inarg));
501 inarg.fh = ff->fh;
154603fe 502 inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
d5b48543
MS		 503 args.opcode = opcode;
504 args.nodeid = get_node_id(inode);
505 args.in_numargs = 1;
506 args.in_args[0].size = sizeof(inarg);
507 args.in_args[0].value = &inarg;
fcee216b 508 return fuse_simple_request(fm, &args);
a9c2d1e8
MS		 509}
510
511static int fuse_fsync(struct file *file, loff_t start, loff_t end,
512 int datasync)
513{
514 struct inode *inode = file->f_mapping->host;
515 struct fuse_conn *fc = get_fuse_conn(inode);
b6aeaded
MS		 516 int err;
517
5d069dbe 518 if (fuse_is_bad(inode))
248d86e8
MS		 519 return -EIO;
520
5955102c 521 inode_lock(inode);
02c24a82 522
3be5a52b
MS		 523 /*
524 * Start writeback against all dirty pages of the inode, then
525 * wait for all outstanding writes, before sending the FSYNC
526 * request.
527 */
7e51fe1d 528 err = file_write_and_wait_range(file, start, end);
3be5a52b 529 if (err)
02c24a82 530 goto out;
3be5a52b
MS		 531
532 fuse_sync_writes(inode);
ac7f052b
AK		 533
534 /*
535 * Due to implementation of fuse writeback
7e51fe1d 536 * file_write_and_wait_range() does not catch errors.
ac7f052b
AK		 537 * We have to do this directly after fuse_sync_writes()
538 */
7e51fe1d 539 err = file_check_and_advance_wb_err(file);
ac7f052b
AK		 540 if (err)
541 goto out;
542
1e18bda8
MS		 543 err = sync_inode_metadata(inode, 1);
544 if (err)
545 goto out;
3be5a52b 546
a9c2d1e8 547 if (fc->no_fsync)
22401e7b 548 goto out;
b0aa7606 549
a9c2d1e8 550 err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
b6aeaded 551 if (err == -ENOSYS) {
a9c2d1e8 552 fc->no_fsync = 1;
b6aeaded
MS		 553 err = 0;
554 }
02c24a82 555out:
5955102c 556 inode_unlock(inode);
b6aeaded 557
a9c2d1e8 558 return err;
82547981
MS		 559}
560
00793ca5
MS		 561void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
562 size_t count, int opcode)
563{
564 struct fuse_file *ff = file->private_data;
565 struct fuse_args *args = &ia->ap.args;
566
567 ia->read.in.fh = ff->fh;
568 ia->read.in.offset = pos;
569 ia->read.in.size = count;
570 ia->read.in.flags = file->f_flags;
571 args->opcode = opcode;
572 args->nodeid = ff->nodeid;
573 args->in_numargs = 1;
574 args->in_args[0].size = sizeof(ia->read.in);
575 args->in_args[0].value = &ia->read.in;
576 args->out_argvar = true;
577 args->out_numargs = 1;
578 args->out_args[0].size = count;
579}
580
41748675 581static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres,
45ac96ed 582 bool should_dirty)
187c5c36 583{
45ac96ed 584 unsigned int i;
187c5c36 585
3b97c365 586 for (i = 0; i < ap->num_folios; i++) {
8fba54ae 587 if (should_dirty)
3b97c365 588 folio_mark_dirty_lock(ap->folios[i]);
738adade 589 if (ap->args.is_pinned)
3b97c365 590 unpin_folio(ap->folios[i]);
187c5c36 591 }
41748675
HT		 592
593 if (nres > 0 && ap->args.invalidate_vmap)
594 invalidate_kernel_vmap_range(ap->args.vmap_base, nres);
187c5c36
MP		 595}
596
744742d6
SF		 597static void fuse_io_release(struct kref *kref)
598{
599 kfree(container_of(kref, struct fuse_io_priv, refcnt));
600}
601
9d5722b7
CH		 602static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
603{
604 if (io->err)
605 return io->err;
606
607 if (io->bytes >= 0 && io->write)
608 return -EIO;
609
610 return io->bytes < 0 ? io->size : io->bytes;
611}
612
06bbb761 613/*
01e9d11a
MP		 614 * In case of short read, the caller sets 'pos' to the position of
615 * actual end of fuse request in IO request. Otherwise, if bytes_requested
616 * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
617 *
618 * An example:
c4e0cd4e 619 * User requested DIO read of 64K. It was split into two 32K fuse requests,
01e9d11a
MP		 620 * both submitted asynchronously. The first of them was ACKed by userspace as
621 * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
622 * second request was ACKed as short, e.g. only 1K was read, resulting in
623 * pos == 33K.
624 *
625 * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
626 * will be equal to the length of the longest contiguous fragment of
627 * transferred data starting from the beginning of IO request.
628 */
629static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
630{
631 int left;
632
633 spin_lock(&io->lock);
634 if (err)
635 io->err = io->err ? : err;
636 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
637 io->bytes = pos;
638
639 left = --io->reqs;
7879c4e5 640 if (!left && io->blocking)
9d5722b7 641 complete(io->done);
01e9d11a
MP		 642 spin_unlock(&io->lock);
643
7879c4e5 644 if (!left && !io->blocking) {
9d5722b7 645 ssize_t res = fuse_get_res_by_io(io);
01e9d11a 646
9d5722b7
CH		 647 if (res >= 0) {
648 struct inode *inode = file_inode(io->iocb->ki_filp);
649 struct fuse_conn *fc = get_fuse_conn(inode);
650 struct fuse_inode *fi = get_fuse_inode(inode);
01e9d11a 651
f15ecfef 652 spin_lock(&fi->lock);
4510d86f 653 fi->attr_version = atomic64_inc_return(&fc->attr_version);
f15ecfef 654 spin_unlock(&fi->lock);
01e9d11a
MP		 655 }
656
6b19b766 657 io->iocb->ki_complete(io->iocb, res);
01e9d11a 658 }
744742d6
SF		 659
660 kref_put(&io->refcnt, fuse_io_release);
01e9d11a
MP		 661}
662
68bfb7eb 663static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
51b02530
JK		 664 unsigned int nfolios)
665{
666 struct fuse_io_args *ia;
667
668 ia = kzalloc(sizeof(*ia), GFP_KERNEL);
669 if (ia) {
670 ia->io = io;
51b02530 671 ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL,
68bfb7eb 672 &ia->ap.descs);
51b02530
JK		 673 if (!ia->ap.folios) {
674 kfree(ia);
675 ia = NULL;
676 }
677 }
678 return ia;
679}
680
68bfb7eb 681static void fuse_io_free(struct fuse_io_args *ia)
51b02530
JK		 682{
683 kfree(ia->ap.folios);
684 kfree(ia);
685}
686
fcee216b 687static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
45ac96ed
MS		 688 int err)
689{
690 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
691 struct fuse_io_priv *io = ia->io;
01e9d11a 692 ssize_t pos = -1;
41748675 693 size_t nres;
01e9d11a 694
45ac96ed
MS		 695 if (err) {
696 /* Nothing */
697 } else if (io->write) {
698 if (ia->write.out.size > ia->write.in.size) {
699 err = -EIO;
41748675
HT		 700 } else {
701 nres = ia->write.out.size;
702 if (ia->write.in.size != ia->write.out.size)
703 pos = ia->write.in.offset - io->offset +
704 ia->write.out.size;
45ac96ed 705 }
01e9d11a 706 } else {
45ac96ed
MS		 707 u32 outsize = args->out_args[0].size;
708
41748675 709 nres = outsize;
45ac96ed
MS		 710 if (ia->read.in.size != outsize)
711 pos = ia->read.in.offset - io->offset + outsize;
01e9d11a
MP		 712 }
713
41748675
HT		 714 fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty);
715
45ac96ed 716 fuse_aio_complete(io, err, pos);
68bfb7eb 717 fuse_io_free(ia);
01e9d11a
MP		 718}
719
fcee216b 720static ssize_t fuse_async_req_send(struct fuse_mount *fm,
45ac96ed 721 struct fuse_io_args *ia, size_t num_bytes)
01e9d11a 722{
45ac96ed
MS		 723 ssize_t err;
724 struct fuse_io_priv *io = ia->io;
725
01e9d11a 726 spin_lock(&io->lock);
744742d6 727 kref_get(&io->refcnt);
01e9d11a
MP		 728 io->size += num_bytes;
729 io->reqs++;
730 spin_unlock(&io->lock);
731
45ac96ed 732 ia->ap.args.end = fuse_aio_complete_req;
bb737bbe 733 ia->ap.args.may_block = io->should_dirty;
fcee216b 734 err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
f1ebdeff 735 if (err)
fcee216b 736 fuse_aio_complete_req(fm, &ia->ap.args, err);
01e9d11a 737
f1ebdeff 738 return num_bytes;
01e9d11a
MP		 739}
740
45ac96ed
MS		 741static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
742 fl_owner_t owner)
04730fef 743{
45ac96ed 744 struct file *file = ia->io->iocb->ki_filp;
2106cb18 745 struct fuse_file *ff = file->private_data;
fcee216b 746 struct fuse_mount *fm = ff->fm;
f3332114 747
45ac96ed 748 fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
f3332114 749 if (owner != NULL) {
45ac96ed 750 ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
fcee216b 751 ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
f3332114 752 }
36cf66ed 753
45ac96ed 754 if (ia->io->async)
fcee216b 755 return fuse_async_req_send(fm, ia, count);
36cf66ed 756
fcee216b 757 return fuse_simple_request(fm, &ia->ap.args);
04730fef
MS		 758}
759
5c5c5e51
MS		 760static void fuse_read_update_size(struct inode *inode, loff_t size,
761 u64 attr_ver)
762{
763 struct fuse_conn *fc = get_fuse_conn(inode);
764 struct fuse_inode *fi = get_fuse_inode(inode);
765
f15ecfef 766 spin_lock(&fi->lock);
484ce657 767 if (attr_ver >= fi->attr_version && size < inode->i_size &&
06a7c3c2 768 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
4510d86f 769 fi->attr_version = atomic64_inc_return(&fc->attr_version);
5c5c5e51
MS		 770 i_size_write(inode, size);
771 }
f15ecfef 772 spin_unlock(&fi->lock);
5c5c5e51
MS		 773}
774
a0d45d84 775static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
134831e3 776 struct fuse_args_pages *ap)
a92adc82 777{
8373200b
PE		 778 struct fuse_conn *fc = get_fuse_conn(inode);
779
a73d47f5
MS		 780 /*
781 * If writeback_cache is enabled, a short read means there's a hole in
782 * the file. Some data after the hole is in page cache, but has not
783 * reached the client fs yet. So the hole is not present there.
784 */
785 if (!fc->writeback_cache) {
51b02530 786 loff_t pos = folio_pos(ap->folios[0]) + num_read;
8373200b
PE		 787 fuse_read_update_size(inode, pos, attr_ver);
788 }
a92adc82
PE		 789}
790
65fe891d 791static int fuse_do_readfolio(struct file *file, struct folio *folio)
b6aeaded 792{
65fe891d 793 struct inode *inode = folio->mapping->host;
fcee216b 794 struct fuse_mount *fm = get_fuse_mount(inode);
65fe891d 795 loff_t pos = folio_pos(folio);
351a24eb 796 struct fuse_folio_desc desc = { .length = folio_size(folio) };
00793ca5
MS		 797 struct fuse_io_args ia = {
798 .ap.args.page_zeroing = true,
799 .ap.args.out_pages = true,
51b02530
JK		 800 .ap.num_folios = 1,
801 .ap.folios = &folio,
68bfb7eb 802 .ap.descs = &desc,
00793ca5
MS		 803 };
804 ssize_t res;
5c5c5e51 805 u64 attr_ver;
248d86e8 806
fcee216b 807 attr_ver = fuse_get_attr_version(fm->fc);
5c5c5e51 808
2f139829
MS		 809 /* Don't overflow end offset */
810 if (pos + (desc.length - 1) == LLONG_MAX)
811 desc.length--;
812
00793ca5 813 fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
fcee216b 814 res = fuse_simple_request(fm, &ia.ap.args);
00793ca5
MS		 815 if (res < 0)
816 return res;
817 /*
818 * Short read means EOF. If file size is larger, truncate it
819 */
820 if (res < desc.length)
134831e3 821 fuse_short_read(inode, attr_ver, res, &ia.ap);
5c5c5e51 822
65fe891d 823 folio_mark_uptodate(folio);
482fce55 824
00793ca5 825 return 0;
482fce55
MP		 826}
827
5efd00e4 828static int fuse_read_folio(struct file *file, struct folio *folio)
482fce55 829{
65fe891d 830 struct inode *inode = folio->mapping->host;
482fce55
MP		 831 int err;
832
833 err = -EIO;
5d069dbe 834 if (fuse_is_bad(inode))
482fce55
MP		 835 goto out;
836
65fe891d 837 err = fuse_do_readfolio(file, folio);
451418fc 838 fuse_invalidate_atime(inode);
b6aeaded 839 out:
65fe891d 840 folio_unlock(folio);
b6aeaded
MS		 841 return err;
842}
843
fcee216b 844static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
134831e3 845 int err)
db50b96c 846{
c1aa96a5 847 int i;
134831e3
MS		 848 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
849 struct fuse_args_pages *ap = &ia->ap;
850 size_t count = ia->read.in.size;
851 size_t num_read = args->out_args[0].size;
ce534fb0 852 struct address_space *mapping = NULL;
c1aa96a5 853
51b02530
JK		 854 for (i = 0; mapping == NULL && i < ap->num_folios; i++)
855 mapping = ap->folios[i]->mapping;
5c5c5e51 856
ce534fb0
MS		 857 if (mapping) {
858 struct inode *inode = mapping->host;
859
860 /*
861 * Short read means EOF. If file size is larger, truncate it
862 */
134831e3
MS		 863 if (!err && num_read < count)
864 fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
ce534fb0 865
451418fc 866 fuse_invalidate_atime(inode);
ce534fb0 867 }
c1aa96a5 868
0c67c37e 869 for (i = 0; i < ap->num_folios; i++) {
51b02530 870 folio_end_read(ap->folios[i], !err);
0c67c37e
JK		 871 folio_put(ap->folios[i]);
872 }
134831e3 873 if (ia->ff)
e26ee4ef 874 fuse_file_put(ia->ff, false);
134831e3 875
68bfb7eb 876 fuse_io_free(ia);
c1aa96a5
MS		 877}
878
906354c8
JK		 879static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
880 unsigned int count)
c1aa96a5 881{
2106cb18 882 struct fuse_file *ff = file->private_data;
fcee216b 883 struct fuse_mount *fm = ff->fm;
134831e3 884 struct fuse_args_pages *ap = &ia->ap;
51b02530 885 loff_t pos = folio_pos(ap->folios[0]);
7df1e988 886 ssize_t res;
134831e3
MS		 887 int err;
888
889 ap->args.out_pages = true;
890 ap->args.page_zeroing = true;
891 ap->args.page_replace = true;
2f139829
MS		 892
893 /* Don't overflow end offset */
894 if (pos + (count - 1) == LLONG_MAX) {
895 count--;
68bfb7eb 896 ap->descs[ap->num_folios - 1].length--;
2f139829
MS		 897 }
898 WARN_ON((loff_t) (pos + count) < 0);
899
134831e3 900 fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
fcee216b
MR		 901 ia->read.attr_ver = fuse_get_attr_version(fm->fc);
902 if (fm->fc->async_read) {
134831e3
MS		 903 ia->ff = fuse_file_get(ff);
904 ap->args.end = fuse_readpages_end;
fcee216b 905 err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
134831e3
MS		 906 if (!err)
907 return;
9cd68455 908 } else {
fcee216b 909 res = fuse_simple_request(fm, &ap->args);
7df1e988 910 err = res < 0 ? res : 0;
9cd68455 911 }
fcee216b 912 fuse_readpages_end(fm, &ap->args, err);
db50b96c
MS		 913}
914
76a0294e 915static void fuse_readahead(struct readahead_control *rac)
db50b96c 916{
76a0294e 917 struct inode *inode = rac->mapping->host;
db50b96c 918 struct fuse_conn *fc = get_fuse_conn(inode);
3eab9d7b 919 unsigned int max_pages, nr_pages;
906354c8 920 struct folio *folio = NULL;
db50b96c 921
5d069dbe 922 if (fuse_is_bad(inode))
76a0294e 923 return;
248d86e8 924
76a0294e
MWO		 925 max_pages = min_t(unsigned int, fc->max_pages,
926 fc->max_read / PAGE_SIZE);
db50b96c 927
3eab9d7b
JB		 928 /*
929 * This is only accurate the first time through, since readahead_folio()
930 * doesn't update readahead_count() from the previous folio until the
931 * next call. Grab nr_pages here so we know how many pages we're going
932 * to have to process. This means that we will exit here with
933 * readahead_count() == folio_nr_pages(last_folio), but we will have
934 * consumed all of the folios, and read_pages() will call
935 * readahead_folio() again which will clean up the rac.
936 */
937 nr_pages = readahead_count(rac);
938
939 while (nr_pages) {
76a0294e
MWO		 940 struct fuse_io_args *ia;
941 struct fuse_args_pages *ap;
3eab9d7b 942 unsigned cur_pages = min(max_pages, nr_pages);
906354c8 943 unsigned int pages = 0;
76a0294e 944
670d21c6
N		 945 if (fc->num_background >= fc->congestion_threshold &&
946 rac->ra->async_size >= readahead_count(rac))
947 /*
948 * Congested and only async pages left, so skip the
949 * rest.
950 */
951 break;
952
68bfb7eb 953 ia = fuse_io_alloc(NULL, cur_pages);
76a0294e 954 if (!ia)
906354c8 955 break;
76a0294e 956 ap = &ia->ap;
3eab9d7b 957
906354c8
JK		 958 while (pages < cur_pages) {
959 unsigned int folio_pages;
960
0c67c37e
JK		 961 /*
962 * This returns a folio with a ref held on it.
963 * The ref needs to be held until the request is
964 * completed, since the splice case (see
965 * fuse_try_move_page()) drops the ref after it's
966 * replaced in the page cache.
967 */
906354c8
JK		 968 if (!folio)
969 folio = __readahead_folio(rac);
970
971 folio_pages = folio_nr_pages(folio);
972 if (folio_pages > cur_pages - pages) {
973 /*
974 * Large folios belonging to fuse will never
975 * have more pages than max_pages.
976 */
977 WARN_ON(!pages);
978 break;
979 }
980
51b02530 981 ap->folios[ap->num_folios] = folio;
68bfb7eb 982 ap->descs[ap->num_folios].length = folio_size(folio);
51b02530 983 ap->num_folios++;
906354c8
JK		 984 pages += folio_pages;
985 folio = NULL;
76a0294e 986 }
906354c8
JK		 987 fuse_send_readpages(ia, rac->file, pages << PAGE_SHIFT);
988 nr_pages -= pages;
989 }
990 if (folio) {
991 folio_end_read(folio, false);
992 folio_put(folio);
d3406ffa 993 }
db50b96c
MS		 994}
995
55752a3a 996static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
bcb4be80
MS		 997{
998 struct inode *inode = iocb->ki_filp->f_mapping->host;
a8894274 999 struct fuse_conn *fc = get_fuse_conn(inode);
bcb4be80 1000
a8894274
BF		 1001 /*
1002 * In auto invalidate mode, always update attributes on read.
1003 * Otherwise, only update if we attempt to read past EOF (to ensure
1004 * i_size is up to date).
1005 */
1006 if (fc->auto_inval_data ||
37c20f16 1007 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
bcb4be80 1008 int err;
c6c745b8 1009 err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE);
bcb4be80
MS		 1010 if (err)
1011 return err;
1012 }
1013
37c20f16 1014 return generic_file_read_iter(iocb, to);
bcb4be80
MS		 1015}
1016
338f2e3f
MS		 1017static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
1018 loff_t pos, size_t count)
1019{
1020 struct fuse_args *args = &ia->ap.args;
1021
1022 ia->write.in.fh = ff->fh;
1023 ia->write.in.offset = pos;
1024 ia->write.in.size = count;
1025 args->opcode = FUSE_WRITE;
1026 args->nodeid = ff->nodeid;
1027 args->in_numargs = 2;
fcee216b 1028 if (ff->fm->fc->minor < 9)
338f2e3f
MS		 1029 args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
1030 else
1031 args->in_args[0].size = sizeof(ia->write.in);
1032 args->in_args[0].value = &ia->write.in;
1033 args->in_args[1].size = count;
1034 args->out_numargs = 1;
1035 args->out_args[0].size = sizeof(ia->write.out);
1036 args->out_args[0].value = &ia->write.out;
1037}
1038
1039static unsigned int fuse_write_flags(struct kiocb *iocb)
1040{
1041 unsigned int flags = iocb->ki_filp->f_flags;
1042
91b94c5d 1043 if (iocb_is_dsync(iocb))
338f2e3f
MS		 1044 flags |= O_DSYNC;
1045 if (iocb->ki_flags & IOCB_SYNC)
1046 flags |= O_SYNC;
1047
1048 return flags;
1049}
1050
45ac96ed
MS		 1051static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
1052 size_t count, fl_owner_t owner)
b25e82e5 1053{
45ac96ed 1054 struct kiocb *iocb = ia->io->iocb;
e1c0eecb 1055 struct file *file = iocb->ki_filp;
2106cb18 1056 struct fuse_file *ff = file->private_data;
fcee216b 1057 struct fuse_mount *fm = ff->fm;
45ac96ed
MS		 1058 struct fuse_write_in *inarg = &ia->write.in;
1059 ssize_t err;
2d698b07 1060
45ac96ed 1061 fuse_write_args_fill(ia, ff, pos, count);
338f2e3f 1062 inarg->flags = fuse_write_flags(iocb);
f3332114 1063 if (owner != NULL) {
f3332114 1064 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
fcee216b 1065 inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
f3332114 1066 }
36cf66ed 1067
45ac96ed 1068 if (ia->io->async)
fcee216b 1069 return fuse_async_req_send(fm, ia, count);
45ac96ed 1070
fcee216b 1071 err = fuse_simple_request(fm, &ia->ap.args);
45ac96ed
MS		 1072 if (!err && ia->write.out.size > count)
1073 err = -EIO;
36cf66ed 1074
45ac96ed 1075 return err ?: ia->write.out.size;
b6aeaded
MS		 1076}
1077
d347739a 1078bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written)
854512ec
MS		 1079{
1080 struct fuse_conn *fc = get_fuse_conn(inode);
1081 struct fuse_inode *fi = get_fuse_inode(inode);
b0aa7606 1082 bool ret = false;
854512ec 1083
f15ecfef 1084 spin_lock(&fi->lock);
4510d86f 1085 fi->attr_version = atomic64_inc_return(&fc->attr_version);
d347739a 1086 if (written > 0 && pos > inode->i_size) {
854512ec 1087 i_size_write(inode, pos);
b0aa7606
MP		 1088 ret = true;
1089 }
f15ecfef 1090 spin_unlock(&fi->lock);
b0aa7606 1091
d347739a
MS		 1092 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
1093
b0aa7606 1094 return ret;
854512ec
MS		 1095}
1096
338f2e3f
MS		 1097static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
1098 struct kiocb *iocb, struct inode *inode,
1099 loff_t pos, size_t count)
ea9b9907 1100{
338f2e3f
MS		 1101 struct fuse_args_pages *ap = &ia->ap;
1102 struct file *file = iocb->ki_filp;
1103 struct fuse_file *ff = file->private_data;
fcee216b 1104 struct fuse_mount *fm = ff->fm;
338f2e3f 1105 unsigned int offset, i;
4f06dd92 1106 bool short_write;
338f2e3f 1107 int err;
ea9b9907 1108
f2ef459b 1109 for (i = 0; i < ap->num_folios; i++)
0c58a97f 1110 folio_wait_writeback(ap->folios[i]);
ea9b9907 1111
338f2e3f
MS		 1112 fuse_write_args_fill(ia, ff, pos, count);
1113 ia->write.in.flags = fuse_write_flags(iocb);
b8667395
VG		 1114 if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID))
1115 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
ea9b9907 1116
fcee216b 1117 err = fuse_simple_request(fm, &ap->args);
8aab336b
MS		 1118 if (!err && ia->write.out.size > count)
1119 err = -EIO;
338f2e3f 1120
4f06dd92 1121 short_write = ia->write.out.size < count;
68bfb7eb 1122 offset = ap->descs[0].offset;
338f2e3f 1123 count = ia->write.out.size;
f2ef459b
JK		 1124 for (i = 0; i < ap->num_folios; i++) {
1125 struct folio *folio = ap->folios[i];
ea9b9907 1126
4f06dd92 1127 if (err) {
785d06af 1128 folio_clear_uptodate(folio);
4f06dd92 1129 } else {
785d06af
JB		 1130 if (count >= folio_size(folio) - offset)
1131 count -= folio_size(folio) - offset;
4f06dd92
VG		 1132 else {
1133 if (short_write)
785d06af 1134 folio_clear_uptodate(folio);
4f06dd92
VG		 1135 count = 0;
1136 }
1137 offset = 0;
1138 }
f2ef459b 1139 if (ia->write.folio_locked && (i == ap->num_folios - 1))
785d06af
JB		 1140 folio_unlock(folio);
1141 folio_put(folio);
ea9b9907
NP		 1142 }
1143
338f2e3f 1144 return err;
ea9b9907
NP		 1145}
1146
4f06dd92 1147static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
338f2e3f
MS		 1148 struct address_space *mapping,
1149 struct iov_iter *ii, loff_t pos,
dbee298c 1150 unsigned int max_folios)
ea9b9907 1151{
4f06dd92 1152 struct fuse_args_pages *ap = &ia->ap;
ea9b9907 1153 struct fuse_conn *fc = get_fuse_conn(mapping->host);
09cbfeaf 1154 unsigned offset = pos & (PAGE_SIZE - 1);
ea9b9907 1155 size_t count = 0;
63c69ad3
JK		 1156 unsigned int num;
1157 int err = 0;
1158
1159 num = min(iov_iter_count(ii), fc->max_write);
ea9b9907 1160
338f2e3f 1161 ap->args.in_pages = true;
68bfb7eb 1162 ap->descs[0].offset = offset;
ea9b9907 1163
dbee298c 1164 while (num && ap->num_folios < max_folios) {
ea9b9907 1165 size_t tmp;
9bafbe7a 1166 struct folio *folio;
09cbfeaf 1167 pgoff_t index = pos >> PAGE_SHIFT;
d60a6015
JK		 1168 unsigned int bytes;
1169 unsigned int folio_offset;
ea9b9907
NP		 1170
1171 again:
9bafbe7a
JB		 1172 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
1173 mapping_gfp_mask(mapping));
1174 if (IS_ERR(folio)) {
1175 err = PTR_ERR(folio);
ea9b9907 1176 break;
9bafbe7a 1177 }
ea9b9907 1178
931e80e4 1179 if (mapping_writably_mapped(mapping))
9bafbe7a 1180 flush_dcache_folio(folio);
931e80e4 1181
d60a6015
JK		 1182 folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
1183 bytes = min(folio_size(folio) - folio_offset, num);
1184
1185 tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii);
9bafbe7a 1186 flush_dcache_folio(folio);
ea9b9907
NP		 1187
1188 if (!tmp) {
9bafbe7a
JB		 1189 folio_unlock(folio);
1190 folio_put(folio);
faa794dd
DH		 1191
1192 /*
1193 * Ensure forward progress by faulting in
1194 * while not holding the folio lock:
1195 */
1196 if (fault_in_iov_iter_readable(ii, bytes)) {
1197 err = -EFAULT;
1198 break;
1199 }
1200
ea9b9907
NP		 1201 goto again;
1202 }
1203
f2ef459b 1204 ap->folios[ap->num_folios] = folio;
d60a6015 1205 ap->descs[ap->num_folios].offset = folio_offset;
68bfb7eb 1206 ap->descs[ap->num_folios].length = tmp;
f2ef459b 1207 ap->num_folios++;
ea9b9907 1208
ea9b9907
NP		 1209 count += tmp;
1210 pos += tmp;
63c69ad3 1211 num -= tmp;
ea9b9907 1212 offset += tmp;
d60a6015 1213 if (offset == folio_size(folio))
ea9b9907
NP		 1214 offset = 0;
1215
d60a6015
JK		 1216 /* If we copied full folio, mark it uptodate */
1217 if (tmp == folio_size(folio))
9bafbe7a 1218 folio_mark_uptodate(folio);
4f06dd92 1219
9bafbe7a
JB		 1220 if (folio_test_uptodate(folio)) {
1221 folio_unlock(folio);
4f06dd92 1222 } else {
f2ef459b 1223 ia->write.folio_locked = true;
4f06dd92
VG		 1224 break;
1225 }
63c69ad3 1226 if (!fc->big_writes || offset != 0)
78bb6cb9 1227 break;
63c69ad3 1228 }
ea9b9907
NP		 1229
1230 return count > 0 ? count : err;
1231}
1232
5da784cc
CS		 1233static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1234 unsigned int max_pages)
d07f09f5 1235{
5da784cc 1236 return min_t(unsigned int,
09cbfeaf
KS		 1237 ((pos + len - 1) >> PAGE_SHIFT) -
1238 (pos >> PAGE_SHIFT) + 1,
5da784cc 1239 max_pages);
d07f09f5
MP		 1240}
1241
596df33d 1242static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii)
ea9b9907 1243{
596df33d 1244 struct address_space *mapping = iocb->ki_filp->f_mapping;
ea9b9907
NP		 1245 struct inode *inode = mapping->host;
1246 struct fuse_conn *fc = get_fuse_conn(inode);
06a7c3c2 1247 struct fuse_inode *fi = get_fuse_inode(inode);
596df33d 1248 loff_t pos = iocb->ki_pos;
ea9b9907
NP		 1249 int err = 0;
1250 ssize_t res = 0;
1251
06a7c3c2
MP		 1252 if (inode->i_size < pos + iov_iter_count(ii))
1253 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1254
ea9b9907 1255 do {
ea9b9907 1256 ssize_t count;
338f2e3f
MS		 1257 struct fuse_io_args ia = {};
1258 struct fuse_args_pages *ap = &ia.ap;
5da784cc
CS		 1259 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1260 fc->max_pages);
ea9b9907 1261
68bfb7eb 1262 ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs);
f2ef459b 1263 if (!ap->folios) {
338f2e3f 1264 err = -ENOMEM;
ea9b9907
NP		 1265 break;
1266 }
1267
4f06dd92 1268 count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);
ea9b9907
NP		 1269 if (count <= 0) {
1270 err = count;
1271 } else {
338f2e3f
MS		 1272 err = fuse_send_write_pages(&ia, iocb, inode,
1273 pos, count);
ea9b9907 1274 if (!err) {
338f2e3f
MS		 1275 size_t num_written = ia.write.out.size;
1276
ea9b9907
NP		 1277 res += num_written;
1278 pos += num_written;
1279
1280 /* break out of the loop on short write */
1281 if (num_written != count)
1282 err = -EIO;
1283 }
1284 }
f2ef459b 1285 kfree(ap->folios);
ea9b9907
NP		 1286 } while (!err && iov_iter_count(ii));
1287
d347739a 1288 fuse_write_update_attr(inode, pos, res);
06a7c3c2 1289 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
ea9b9907 1290
70e986c3
CH		 1291 if (!res)
1292 return err;
1293 iocb->ki_pos += res;
1294 return res;
ea9b9907
NP		 1295}
1296
699cf824
BS		 1297static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter)
1298{
1299 struct inode *inode = file_inode(iocb->ki_filp);
1300
1301 return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode);
1302}
1303
1304/*
1305 * @return true if an exclusive lock for direct IO writes is needed
1306 */
1307static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from)
1308{
1309 struct file *file = iocb->ki_filp;
1310 struct fuse_file *ff = file->private_data;
1311 struct inode *inode = file_inode(iocb->ki_filp);
205c1d80 1312 struct fuse_inode *fi = get_fuse_inode(inode);
699cf824
BS		 1313
1314 /* Server side has to advise that it supports parallel dio writes. */
1315 if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES))
1316 return true;
1317
1318 /*
1319 * Append will need to know the eventual EOF - always needs an
1320 * exclusive lock.
1321 */
1322 if (iocb->ki_flags & IOCB_APPEND)
1323 return true;
1324
205c1d80
AG		 1325 /* shared locks are not allowed with parallel page cache IO */
1326 if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
2f3d8ff4 1327 return true;
699cf824
BS		 1328
1329 /* Parallel dio beyond EOF is not supported, at least for now. */
1330 if (fuse_io_past_eof(iocb, from))
1331 return true;
1332
1333 return false;
1334}
1335
9bbb6717
BS		 1336static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from,
1337 bool *exclusive)
1338{
1339 struct inode *inode = file_inode(iocb->ki_filp);
4864a6dd 1340 struct fuse_inode *fi = get_fuse_inode(inode);
9bbb6717
BS		 1341
1342 *exclusive = fuse_dio_wr_exclusive_lock(iocb, from);
1343 if (*exclusive) {
1344 inode_lock(inode);
1345 } else {
1346 inode_lock_shared(inode);
1347 /*
205c1d80
AG		 1348 * New parallal dio allowed only if inode is not in caching
1349 * mode and denies new opens in caching mode. This check
1350 * should be performed only after taking shared inode lock.
1351 * Previous past eof check was without inode lock and might
1352 * have raced, so check it again.
9bbb6717 1353 */
205c1d80 1354 if (fuse_io_past_eof(iocb, from) ||
4864a6dd 1355 fuse_inode_uncached_io_start(fi, NULL) != 0) {
9bbb6717
BS		 1356 inode_unlock_shared(inode);
1357 inode_lock(inode);
1358 *exclusive = true;
1359 }
1360 }
1361}
1362
205c1d80 1363static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive)
9bbb6717 1364{
205c1d80 1365 struct inode *inode = file_inode(iocb->ki_filp);
4864a6dd 1366 struct fuse_inode *fi = get_fuse_inode(inode);
205c1d80 1367
9bbb6717
BS		 1368 if (exclusive) {
1369 inode_unlock(inode);
1370 } else {
205c1d80 1371 /* Allow opens in caching mode after last parallel dio end */
4864a6dd 1372 fuse_inode_uncached_io_end(fi);
9bbb6717
BS		 1373 inode_unlock_shared(inode);
1374 }
1375}
1376
55752a3a 1377static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
ea9b9907
NP		 1378{
1379 struct file *file = iocb->ki_filp;
5b8ca5a5 1380 struct mnt_idmap *idmap = file_mnt_idmap(file);
ea9b9907 1381 struct address_space *mapping = file->f_mapping;
ea9b9907
NP		 1382 ssize_t written = 0;
1383 struct inode *inode = mapping->host;
2e3f7dd0 1384 ssize_t err, count;
8981bdfd 1385 struct fuse_conn *fc = get_fuse_conn(inode);
ea9b9907 1386
8981bdfd 1387 if (fc->writeback_cache) {
4d99ff8f 1388 /* Update size (EOF optimization) and mode (SUID clearing) */
c6c745b8
MS		 1389 err = fuse_update_attributes(mapping->host, file,
1390 STATX_SIZE | STATX_MODE);
4d99ff8f
PE		 1391 if (err)
1392 return err;
1393
8981bdfd 1394 if (fc->handle_killpriv_v2 &&
5b8ca5a5 1395 setattr_should_drop_suidgid(idmap,
9452e93e 1396 file_inode(file))) {
8981bdfd
VG		 1397 goto writethrough;
1398 }
1399
84c3d55c 1400 return generic_file_write_iter(iocb, from);
4d99ff8f
PE		 1401 }
1402
8981bdfd 1403writethrough:
5955102c 1404 inode_lock(inode);
ea9b9907 1405
2e3f7dd0 1406 err = count = generic_write_checks(iocb, from);
3309dd04 1407 if (err <= 0)
ea9b9907
NP		 1408 goto out;
1409
2e3f7dd0
ZJ		 1410 task_io_account_write(count);
1411
e6befec5 1412 err = kiocb_modified(iocb);
c3b2da31
JB		 1413 if (err)
1414 goto out;
ea9b9907 1415
2ba48ce5 1416 if (iocb->ki_flags & IOCB_DIRECT) {
1af5bb49 1417 written = generic_file_direct_write(iocb, from);
84c3d55c 1418 if (written < 0 || !iov_iter_count(from))
4273b793 1419 goto out;
64d1b4dd
CH		 1420 written = direct_write_fallback(iocb, from, written,
1421 fuse_perform_write(iocb, from));
4273b793 1422 } else {
596df33d 1423 written = fuse_perform_write(iocb, from);
4273b793 1424 }
ea9b9907 1425out:
5955102c 1426 inode_unlock(inode);
e1c0eecb
MS		 1427 if (written > 0)
1428 written = generic_write_sync(iocb, written);
ea9b9907
NP		 1429
1430 return written ? written : err;
1431}
1432
7c190c8b
MP		 1433static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1434{
de4f5fed 1435 return (unsigned long)iter_iov(ii)->iov_base + ii->iov_offset;
7c190c8b
MP		 1436}
1437
1438static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1439 size_t max_size)
1440{
1441 return min(iov_iter_single_seg_count(ii), max_size);
1442}
1443
45ac96ed
MS		 1444static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1445 size_t *nbytesp, int write,
41748675
HT		 1446 unsigned int max_pages,
1447 bool use_pages_for_kvec_io)
413ef8cb 1448{
41748675 1449 bool flush_or_invalidate = false;
3b97c365 1450 unsigned int nr_pages = 0;
7c190c8b 1451 size_t nbytes = 0; /* # bytes already packed in req */
742f9927 1452 ssize_t ret = 0;
b98d023a 1453
41748675
HT		 1454 /* Special case for kernel I/O: can copy directly into the buffer.
1455 * However if the implementation of fuse_conn requires pages instead of
1456 * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead.
1457 */
00e23707 1458 if (iov_iter_is_kvec(ii)) {
41748675 1459 void *user_addr = (void *)fuse_get_user_addr(ii);
7c190c8b 1460
41748675
HT		 1461 if (!use_pages_for_kvec_io) {
1462 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
f4975c67 1463
41748675
HT		 1464 if (write)
1465 ap->args.in_args[1].value = user_addr;
1466 else
1467 ap->args.out_args[0].value = user_addr;
1468
1469 iov_iter_advance(ii, frag_size);
1470 *nbytesp = frag_size;
1471 return 0;
1472 }
1473
1474 if (is_vmalloc_addr(user_addr)) {
1475 ap->args.vmap_base = user_addr;
1476 flush_or_invalidate = true;
1477 }
f4975c67 1478 }
413ef8cb 1479
3b97c365
JK		 1480 /*
1481 * Until there is support for iov_iter_extract_folios(), we have to
1482 * manually extract pages using iov_iter_extract_pages() and then
1483 * copy that to a folios array.
1484 */
1485 struct page **pages = kzalloc(max_pages * sizeof(struct page *),
1486 GFP_KERNEL);
78f2560f
BS		 1487 if (!pages) {
1488 ret = -ENOMEM;
1489 goto out;
1490 }
3b97c365
JK		 1491
1492 while (nbytes < *nbytesp && nr_pages < max_pages) {
1493 unsigned nfolios, i;
f67da30c 1494 size_t start;
738adade 1495
3b97c365 1496 ret = iov_iter_extract_pages(ii, &pages,
738adade 1497 *nbytesp - nbytes,
3b97c365 1498 max_pages - nr_pages,
738adade 1499 0, &start);
7c190c8b 1500 if (ret < 0)
742f9927 1501 break;
7c190c8b 1502
c9c37e2e 1503 nbytes += ret;
7c190c8b 1504
7a4f5418
JK		 1505 nfolios = DIV_ROUND_UP(ret + start, PAGE_SIZE);
1506
1507 for (i = 0; i < nfolios; i++) {
1508 struct folio *folio = page_folio(pages[i]);
1509 unsigned int offset = start +
1510 (folio_page_idx(folio, pages[i]) << PAGE_SHIFT);
1511 unsigned int len = min_t(unsigned int, ret, PAGE_SIZE - start);
1512
1513 ap->descs[ap->num_folios].offset = offset;
1514 ap->descs[ap->num_folios].length = len;
1515 ap->folios[ap->num_folios] = folio;
1516 start = 0;
1517 ret -= len;
1518 ap->num_folios++;
1519 }
1520
3b97c365 1521 nr_pages += nfolios;
7c190c8b 1522 }
3b97c365 1523 kfree(pages);
f4975c67 1524
41748675
HT		 1525 if (write && flush_or_invalidate)
1526 flush_kernel_vmap_range(ap->args.vmap_base, nbytes);
1527
1528 ap->args.invalidate_vmap = !write && flush_or_invalidate;
738adade 1529 ap->args.is_pinned = iov_iter_extract_will_pin(ii);
0c4bcfde 1530 ap->args.user_pages = true;
f4975c67 1531 if (write)
cabdb4fa 1532 ap->args.in_pages = true;
f4975c67 1533 else
cabdb4fa 1534 ap->args.out_pages = true;
f4975c67 1535
78f2560f 1536out:
7c190c8b 1537 *nbytesp = nbytes;
f4975c67 1538
2c932d4c 1539 return ret < 0 ? ret : 0;
413ef8cb
MS		 1540}
1541
d22a943f
AV		 1542ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1543 loff_t *ppos, int flags)
413ef8cb 1544{
ea8cd333
PE		 1545 int write = flags & FUSE_DIO_WRITE;
1546 int cuse = flags & FUSE_DIO_CUSE;
e1c0eecb 1547 struct file *file = io->iocb->ki_filp;
80e4f252
HX		 1548 struct address_space *mapping = file->f_mapping;
1549 struct inode *inode = mapping->host;
2106cb18 1550 struct fuse_file *ff = file->private_data;
fcee216b 1551 struct fuse_conn *fc = ff->fm->fc;
413ef8cb
MS		 1552 size_t nmax = write ? fc->max_write : fc->max_read;
1553 loff_t pos = *ppos;
d22a943f 1554 size_t count = iov_iter_count(iter);
09cbfeaf
KS		 1555 pgoff_t idx_from = pos >> PAGE_SHIFT;
1556 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
413ef8cb 1557 ssize_t res = 0;
742f9927 1558 int err = 0;
45ac96ed
MS		 1559 struct fuse_io_args *ia;
1560 unsigned int max_pages;
80e4f252 1561 bool fopen_direct_io = ff->open_flags & FOPEN_DIRECT_IO;
248d86e8 1562
45ac96ed 1563 max_pages = iov_iter_npages(iter, fc->max_pages);
68bfb7eb 1564 ia = fuse_io_alloc(io, max_pages);
45ac96ed
MS		 1565 if (!ia)
1566 return -ENOMEM;
413ef8cb 1567
c55e0a55 1568 if (fopen_direct_io && fc->direct_io_allow_mmap) {
b5a2a3a0
HX		 1569 res = filemap_write_and_wait_range(mapping, pos, pos + count - 1);
1570 if (res) {
68bfb7eb 1571 fuse_io_free(ia);
b5a2a3a0
HX		 1572 return res;
1573 }
1574 }
0c58a97f 1575 if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) {
ea8cd333 1576 if (!write)
5955102c 1577 inode_lock(inode);
ea8cd333
PE		 1578 fuse_sync_writes(inode);
1579 if (!write)
5955102c 1580 inode_unlock(inode);
ea8cd333
PE		 1581 }
1582
80e4f252
HX		 1583 if (fopen_direct_io && write) {
1584 res = invalidate_inode_pages2_range(mapping, idx_from, idx_to);
1585 if (res) {
68bfb7eb 1586 fuse_io_free(ia);
80e4f252
HX		 1587 return res;
1588 }
1589 }
1590
fcb14cb1 1591 io->should_dirty = !write && user_backed_iter(iter);
413ef8cb 1592 while (count) {
45ac96ed 1593 ssize_t nres;
2106cb18 1594 fl_owner_t owner = current->files;
f4975c67 1595 size_t nbytes = min(count, nmax);
45ac96ed
MS		 1596
1597 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
41748675 1598 max_pages, fc->use_pages_for_kvec_io);
742f9927 1599 if (err && !nbytes)
413ef8cb 1600 break;
f4975c67 1601
4a2abf99 1602 if (write) {
45ac96ed 1603 if (!capable(CAP_FSETID))
10c52c84 1604 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
4a2abf99 1605
45ac96ed 1606 nres = fuse_send_write(ia, pos, nbytes, owner);
4a2abf99 1607 } else {
45ac96ed 1608 nres = fuse_send_read(ia, pos, nbytes, owner);
4a2abf99 1609 }
2106cb18 1610
45ac96ed 1611 if (!io->async || nres < 0) {
41748675 1612 fuse_release_user_pages(&ia->ap, nres, io->should_dirty);
68bfb7eb 1613 fuse_io_free(ia);
45ac96ed
MS		 1614 }
1615 ia = NULL;
1616 if (nres < 0) {
f658adee 1617 iov_iter_revert(iter, nbytes);
45ac96ed 1618 err = nres;
413ef8cb
MS		 1619 break;
1620 }
45ac96ed
MS		 1621 WARN_ON(nres > nbytes);
1622
413ef8cb
MS		 1623 count -= nres;
1624 res += nres;
1625 pos += nres;
f658adee
MS		 1626 if (nres != nbytes) {
1627 iov_iter_revert(iter, nbytes - nres);
413ef8cb 1628 break;
f658adee 1629 }
56cf34ff 1630 if (count) {
45ac96ed 1631 max_pages = iov_iter_npages(iter, fc->max_pages);
68bfb7eb 1632 ia = fuse_io_alloc(io, max_pages);
45ac96ed 1633 if (!ia)
56cf34ff
MS		 1634 break;
1635 }
413ef8cb 1636 }
45ac96ed 1637 if (ia)
68bfb7eb 1638 fuse_io_free(ia);
d09cb9d7 1639 if (res > 0)
413ef8cb 1640 *ppos = pos;
413ef8cb 1641
742f9927 1642 return res > 0 ? res : err;
413ef8cb 1643}
08cbf542 1644EXPORT_SYMBOL_GPL(fuse_direct_io);
413ef8cb 1645
36cf66ed 1646static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
d22a943f
AV		 1647 struct iov_iter *iter,
1648 loff_t *ppos)
413ef8cb 1649{
d09cb9d7 1650 ssize_t res;
e1c0eecb 1651 struct inode *inode = file_inode(io->iocb->ki_filp);
d09cb9d7 1652
d22a943f 1653 res = fuse_direct_io(io, iter, ppos, 0);
d09cb9d7 1654
9a2eb24d 1655 fuse_invalidate_atime(inode);
d09cb9d7
MS		 1656
1657 return res;
413ef8cb
MS		 1658}
1659
23c94e1c
MR		 1660static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1661
15316263 1662static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
b98d023a 1663{
23c94e1c
MR		 1664 ssize_t res;
1665
cc23d537 1666 if (!is_sync_kiocb(iocb)) {
23c94e1c
MR		 1667 res = fuse_direct_IO(iocb, to);
1668 } else {
1669 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1670
1671 res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1672 }
1673
1674 return res;
b98d023a
MP		 1675}
1676
15316263 1677static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
4273b793 1678{
e1c0eecb 1679 struct inode *inode = file_inode(iocb->ki_filp);
15316263 1680 ssize_t res;
9bbb6717 1681 bool exclusive;
4273b793 1682
9bbb6717 1683 fuse_dio_lock(iocb, from, &exclusive);
3309dd04 1684 res = generic_write_checks(iocb, from);
23c94e1c 1685 if (res > 0) {
2e3f7dd0 1686 task_io_account_write(res);
cc23d537 1687 if (!is_sync_kiocb(iocb)) {
23c94e1c
MR		 1688 res = fuse_direct_IO(iocb, from);
1689 } else {
cc23d537 1690 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1691
23c94e1c
MR		 1692 res = fuse_direct_io(&io, from, &iocb->ki_pos,
1693 FUSE_DIO_WRITE);
d347739a 1694 fuse_write_update_attr(inode, iocb->ki_pos, res);
23c94e1c
MR		 1695 }
1696 }
205c1d80 1697 fuse_dio_unlock(iocb, exclusive);
4273b793
AA		 1698
1699 return res;
1700}
1701
55752a3a
MS		 1702static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1703{
2f7b6f5b
MS		 1704 struct file *file = iocb->ki_filp;
1705 struct fuse_file *ff = file->private_data;
c2d0ad00 1706 struct inode *inode = file_inode(file);
2f7b6f5b 1707
5d069dbe 1708 if (fuse_is_bad(inode))
2f7b6f5b 1709 return -EIO;
55752a3a 1710
c2d0ad00
VG		 1711 if (FUSE_IS_DAX(inode))
1712 return fuse_dax_read_iter(iocb, to);
1713
57e1176e
AG		 1714 /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1715 if (ff->open_flags & FOPEN_DIRECT_IO)
55752a3a 1716 return fuse_direct_read_iter(iocb, to);
57e1176e
AG		 1717 else if (fuse_file_passthrough(ff))
1718 return fuse_passthrough_read_iter(iocb, to);
1719 else
1720 return fuse_cache_read_iter(iocb, to);
55752a3a
MS		 1721}
1722
1723static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1724{
2f7b6f5b
MS		 1725 struct file *file = iocb->ki_filp;
1726 struct fuse_file *ff = file->private_data;
c2d0ad00 1727 struct inode *inode = file_inode(file);
2f7b6f5b 1728
5d069dbe 1729 if (fuse_is_bad(inode))
2f7b6f5b 1730 return -EIO;
55752a3a 1731
c2d0ad00
VG		 1732 if (FUSE_IS_DAX(inode))
1733 return fuse_dax_write_iter(iocb, from);
1734
57e1176e
AG		 1735 /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1736 if (ff->open_flags & FOPEN_DIRECT_IO)
55752a3a 1737 return fuse_direct_write_iter(iocb, from);
57e1176e
AG		 1738 else if (fuse_file_passthrough(ff))
1739 return fuse_passthrough_write_iter(iocb, from);
1740 else
55752a3a 1741 return fuse_cache_write_iter(iocb, from);
55752a3a
MS		 1742}
1743
5ca73468
AG		 1744static ssize_t fuse_splice_read(struct file *in, loff_t *ppos,
1745 struct pipe_inode_info *pipe, size_t len,
1746 unsigned int flags)
1747{
1748 struct fuse_file *ff = in->private_data;
1749
1750 /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1751 if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
1752 return fuse_passthrough_splice_read(in, ppos, pipe, len, flags);
55752a3a 1753 else
5ca73468
AG		 1754 return filemap_splice_read(in, ppos, pipe, len, flags);
1755}
1756
1757static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out,
1758 loff_t *ppos, size_t len, unsigned int flags)
1759{
1760 struct fuse_file *ff = out->private_data;
1761
1762 /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1763 if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
1764 return fuse_passthrough_splice_write(pipe, out, ppos, len, flags);
1765 else
1766 return iter_file_splice_write(pipe, out, ppos, len, flags);
55752a3a
MS		 1767}
1768
33826ebb 1769static void fuse_writepage_free(struct fuse_writepage_args *wpa)
b6aeaded 1770{
33826ebb 1771 struct fuse_args_pages *ap = &wpa->ia.ap;
385b1268 1772
660585b5
MS		 1773 if (wpa->bucket)
1774 fuse_sync_bucket_dec(wpa->bucket);
1775
4046d3ad 1776 fuse_file_put(wpa->ia.ff, false);
8b284dc4 1777
cbe9c115 1778 kfree(ap->folios);
33826ebb 1779 kfree(wpa);
3be5a52b
MS		 1780}
1781
509a6458 1782static void fuse_writepage_finish(struct fuse_writepage_args *wpa)
3be5a52b 1783{
33826ebb
MS		 1784 struct fuse_args_pages *ap = &wpa->ia.ap;
1785 struct inode *inode = wpa->inode;
3be5a52b 1786 struct fuse_inode *fi = get_fuse_inode(inode);
0c58a97f 1787 struct backing_dev_info *bdi = inode_to_bdi(inode);
385b1268 1788 int i;
3be5a52b 1789
0c58a97f
JK		 1790 for (i = 0; i < ap->num_folios; i++) {
1791 /*
1792 * Benchmarks showed that ending writeback within the
1793 * scope of the fi->lock alleviates xarray lock
1794 * contention and noticeably improves performance.
1795 */
1796 folio_end_writeback(ap->folios[i]);
1797 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1798 wb_writeout_inc(&bdi->wb);
1799 }
c04e3b21 1800
3be5a52b
MS		 1801 wake_up(&fi->page_waitq);
1802}
1803
f15ecfef 1804/* Called under fi->lock, may release and reacquire it */
fcee216b 1805static void fuse_send_writepage(struct fuse_mount *fm,
33826ebb 1806 struct fuse_writepage_args *wpa, loff_t size)
f15ecfef
KT		 1807__releases(fi->lock)
1808__acquires(fi->lock)
3be5a52b 1809{
33826ebb 1810 struct fuse_inode *fi = get_fuse_inode(wpa->inode);
ff7c3ee4 1811 struct fuse_args_pages *ap = &wpa->ia.ap;
33826ebb 1812 struct fuse_write_in *inarg = &wpa->ia.write.in;
ff7c3ee4
JK		 1813 struct fuse_args *args = &ap->args;
1814 __u64 data_size = 0;
1815 int err, i;
1816
1817 for (i = 0; i < ap->num_folios; i++)
1818 data_size += ap->descs[i].length;
3be5a52b 1819
33826ebb 1820 fi->writectr++;
385b1268
PE		 1821 if (inarg->offset + data_size <= size) {
1822 inarg->size = data_size;
3be5a52b 1823 } else if (inarg->offset < size) {
385b1268 1824 inarg->size = size - inarg->offset;
3be5a52b
MS		 1825 } else {
1826 /* Got truncated off completely */
1827 goto out_free;
b6aeaded 1828 }
3be5a52b 1829
33826ebb
MS		 1830 args->in_args[1].size = inarg->size;
1831 args->force = true;
1832 args->nocreds = true;
1833
fcee216b 1834 err = fuse_simple_background(fm, args, GFP_ATOMIC);
33826ebb
MS		 1835 if (err == -ENOMEM) {
1836 spin_unlock(&fi->lock);
fcee216b 1837 err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
33826ebb
MS		 1838 spin_lock(&fi->lock);
1839 }
1840
f15ecfef 1841 /* Fails on broken connection only */
33826ebb 1842 if (unlikely(err))
f15ecfef
KT		 1843 goto out_free;
1844
3be5a52b
MS		 1845 return;
1846
1847 out_free:
33826ebb 1848 fi->writectr--;
509a6458 1849 fuse_writepage_finish(wpa);
f15ecfef 1850 spin_unlock(&fi->lock);
33826ebb 1851 fuse_writepage_free(wpa);
f15ecfef 1852 spin_lock(&fi->lock);
b6aeaded
MS		 1853}
1854
3be5a52b
MS		 1855/*
1856 * If fi->writectr is positive (no truncate or fsync going on) send
1857 * all queued writepage requests.
1858 *
f15ecfef 1859 * Called with fi->lock
3be5a52b
MS		 1860 */
1861void fuse_flush_writepages(struct inode *inode)
f15ecfef
KT		 1862__releases(fi->lock)
1863__acquires(fi->lock)
b6aeaded 1864{
fcee216b 1865 struct fuse_mount *fm = get_fuse_mount(inode);
3be5a52b 1866 struct fuse_inode *fi = get_fuse_inode(inode);
9de5be06 1867 loff_t crop = i_size_read(inode);
33826ebb 1868 struct fuse_writepage_args *wpa;
3be5a52b
MS		 1869
1870 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
33826ebb
MS		 1871 wpa = list_entry(fi->queued_writes.next,
1872 struct fuse_writepage_args, queue_entry);
1873 list_del_init(&wpa->queue_entry);
fcee216b 1874 fuse_send_writepage(fm, wpa, crop);
3be5a52b
MS		 1875 }
1876}
1877
fcee216b 1878static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
33826ebb 1879 int error)
3be5a52b 1880{
33826ebb
MS		 1881 struct fuse_writepage_args *wpa =
1882 container_of(args, typeof(*wpa), ia.ap.args);
1883 struct inode *inode = wpa->inode;
3be5a52b 1884 struct fuse_inode *fi = get_fuse_inode(inode);
3466958b 1885 struct fuse_conn *fc = get_fuse_conn(inode);
3be5a52b 1886
33826ebb 1887 mapping_set_error(inode->i_mapping, error);
3466958b
VG		 1888 /*
1889 * A writeback finished and this might have updated mtime/ctime on
1890 * server making local mtime/ctime stale. Hence invalidate attrs.
1891 * Do this only if writeback_cache is not enabled. If writeback_cache
1892 * is enabled, we trust local ctime/mtime.
1893 */
1894 if (!fc->writeback_cache)
fa5eee57 1895 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY);
f15ecfef 1896 spin_lock(&fi->lock);
3be5a52b 1897 fi->writectr--;
509a6458 1898 fuse_writepage_finish(wpa);
f15ecfef 1899 spin_unlock(&fi->lock);
33826ebb 1900 fuse_writepage_free(wpa);
3be5a52b
MS		 1901}
1902
a9667ac8 1903static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi)
adcadfa8 1904{
84840efc 1905 struct fuse_file *ff;
adcadfa8 1906
f15ecfef 1907 spin_lock(&fi->lock);
84840efc
MS		 1908 ff = list_first_entry_or_null(&fi->write_files, struct fuse_file,
1909 write_entry);
1910 if (ff)
72523425 1911 fuse_file_get(ff);
f15ecfef 1912 spin_unlock(&fi->lock);
adcadfa8
PE		 1913
1914 return ff;
1915}
1916
a9667ac8 1917static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi)
1e18bda8 1918{
a9667ac8 1919 struct fuse_file *ff = __fuse_write_file_get(fi);
1e18bda8
MS		 1920 WARN_ON(!ff);
1921 return ff;
1922}
1923
1924int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1925{
1e18bda8
MS		 1926 struct fuse_inode *fi = get_fuse_inode(inode);
1927 struct fuse_file *ff;
1928 int err;
1929
5c791fe1
MS		 1930 /*
1931 * Inode is always written before the last reference is dropped and
1932 * hence this should not be reached from reclaim.
1933 *
1934 * Writing back the inode from reclaim can deadlock if the request
1935 * processing itself needs an allocation. Allocations triggering
1936 * reclaim while serving a request can't be prevented, because it can
1937 * involve any number of unrelated userspace processes.
1938 */
1939 WARN_ON(wbc->for_reclaim);
1940
a9667ac8 1941 ff = __fuse_write_file_get(fi);
ab9e13f7 1942 err = fuse_flush_times(inode, ff);
1e18bda8 1943 if (ff)
e26ee4ef 1944 fuse_file_put(ff, false);
1e18bda8
MS		 1945
1946 return err;
1947}
1948
33826ebb
MS		 1949static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
1950{
1951 struct fuse_writepage_args *wpa;
1952 struct fuse_args_pages *ap;
1953
1954 wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
1955 if (wpa) {
1956 ap = &wpa->ia.ap;
cbe9c115 1957 ap->num_folios = 0;
68bfb7eb 1958 ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs);
cbe9c115 1959 if (!ap->folios) {
33826ebb
MS		 1960 kfree(wpa);
1961 wpa = NULL;
1962 }
1963 }
1964 return wpa;
1965
1966}
1967
660585b5
MS		 1968static void fuse_writepage_add_to_bucket(struct fuse_conn *fc,
1969 struct fuse_writepage_args *wpa)
1970{
1971 if (!fc->sync_fs)
1972 return;
1973
1974 rcu_read_lock();
1975 /* Prevent resurrection of dead bucket in unlikely race with syncfs */
1976 do {
1977 wpa->bucket = rcu_dereference(fc->curr_bucket);
1978 } while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count)));
1979 rcu_read_unlock();
1980}
1981
0acad928 1982static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio,
0c58a97f 1983 uint32_t folio_index)
0acad928
JK		 1984{
1985 struct inode *inode = folio->mapping->host;
1986 struct fuse_args_pages *ap = &wpa->ia.ap;
1987
0c58a97f 1988 ap->folios[folio_index] = folio;
68bfb7eb 1989 ap->descs[folio_index].offset = 0;
f3cb8bd9 1990 ap->descs[folio_index].length = folio_size(folio);
0acad928
JK		 1991
1992 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
0acad928
JK		 1993}
1994
1995static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio,
1996 struct fuse_file *ff)
1997{
1998 struct inode *inode = folio->mapping->host;
1999 struct fuse_conn *fc = get_fuse_conn(inode);
2000 struct fuse_writepage_args *wpa;
2001 struct fuse_args_pages *ap;
2002
2003 wpa = fuse_writepage_args_alloc();
2004 if (!wpa)
2005 return NULL;
2006
2007 fuse_writepage_add_to_bucket(fc, wpa);
2008 fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio), 0);
2009 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2010 wpa->inode = inode;
2011 wpa->ia.ff = ff;
2012
2013 ap = &wpa->ia.ap;
2014 ap->args.in_pages = true;
2015 ap->args.end = fuse_writepage_end;
2016
2017 return wpa;
2018}
2019
e0887e09 2020static int fuse_writepage_locked(struct folio *folio)
3be5a52b 2021{
e0887e09 2022 struct address_space *mapping = folio->mapping;
3be5a52b 2023 struct inode *inode = mapping->host;
3be5a52b 2024 struct fuse_inode *fi = get_fuse_inode(inode);
33826ebb
MS		 2025 struct fuse_writepage_args *wpa;
2026 struct fuse_args_pages *ap;
0acad928 2027 struct fuse_file *ff;
0c58a97f 2028 int error = -EIO;
3be5a52b 2029
0acad928
JK		 2030 ff = fuse_write_file_get(fi);
2031 if (!ff)
0c58a97f 2032 goto err;
72523425 2033
0acad928
JK		 2034 wpa = fuse_writepage_args_setup(folio, ff);
2035 error = -ENOMEM;
2036 if (!wpa)
2037 goto err_writepage_args;
3be5a52b 2038
0acad928 2039 ap = &wpa->ia.ap;
cbe9c115 2040 ap->num_folios = 1;
3be5a52b 2041
0acad928 2042 folio_start_writeback(folio);
0c58a97f 2043 fuse_writepage_args_page_fill(wpa, folio, 0);
3be5a52b 2044
f15ecfef 2045 spin_lock(&fi->lock);
33826ebb 2046 list_add_tail(&wpa->queue_entry, &fi->queued_writes);
3be5a52b 2047 fuse_flush_writepages(inode);
f15ecfef 2048 spin_unlock(&fi->lock);
3be5a52b
MS		 2049
2050 return 0;
2051
0acad928
JK		 2052err_writepage_args:
2053 fuse_file_put(ff, false);
3be5a52b 2054err:
e0887e09 2055 mapping_set_error(folio->mapping, error);
72523425 2056 return error;
3be5a52b
MS		 2057}
2058
26d614df 2059struct fuse_fill_wb_data {
33826ebb 2060 struct fuse_writepage_args *wpa;
26d614df
PE		 2061 struct fuse_file *ff;
2062 struct inode *inode;
cbe9c115 2063 unsigned int max_folios;
f3cb8bd9 2064 unsigned int nr_pages;
26d614df
PE		 2065};
2066
33826ebb
MS		 2067static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
2068{
2069 struct fuse_args_pages *ap = &data->wpa->ia.ap;
2070 struct fuse_conn *fc = get_fuse_conn(data->inode);
cbe9c115
JK		 2071 struct folio **folios;
2072 struct fuse_folio_desc *descs;
2073 unsigned int nfolios = min_t(unsigned int,
2074 max_t(unsigned int, data->max_folios * 2,
2075 FUSE_DEFAULT_MAX_PAGES_PER_REQ),
33826ebb 2076 fc->max_pages);
cbe9c115 2077 WARN_ON(nfolios <= data->max_folios);
33826ebb 2078
cbe9c115
JK		 2079 folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs);
2080 if (!folios)
33826ebb
MS		 2081 return false;
2082
cbe9c115 2083 memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios);
68bfb7eb 2084 memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios);
cbe9c115
JK		 2085 kfree(ap->folios);
2086 ap->folios = folios;
68bfb7eb 2087 ap->descs = descs;
cbe9c115 2088 data->max_folios = nfolios;
33826ebb
MS		 2089
2090 return true;
2091}
2092
26d614df
PE		 2093static void fuse_writepages_send(struct fuse_fill_wb_data *data)
2094{
33826ebb 2095 struct fuse_writepage_args *wpa = data->wpa;
26d614df 2096 struct inode *inode = data->inode;
26d614df
PE		 2097 struct fuse_inode *fi = get_fuse_inode(inode);
2098
f15ecfef 2099 spin_lock(&fi->lock);
33826ebb 2100 list_add_tail(&wpa->queue_entry, &fi->queued_writes);
26d614df 2101 fuse_flush_writepages(inode);
f15ecfef 2102 spin_unlock(&fi->lock);
8b284dc4
MS		 2103}
2104
6930b8da 2105static bool fuse_writepage_need_send(struct fuse_conn *fc, struct folio *folio,
6ddf3af9
MS		 2106 struct fuse_args_pages *ap,
2107 struct fuse_fill_wb_data *data)
2108{
cbe9c115 2109 WARN_ON(!ap->num_folios);
6ddf3af9 2110
6ddf3af9 2111 /* Reached max pages */
f3cb8bd9 2112 if (data->nr_pages + folio_nr_pages(folio) > fc->max_pages)
6ddf3af9
MS		 2113 return true;
2114
2115 /* Reached max write bytes */
f3cb8bd9 2116 if ((data->nr_pages * PAGE_SIZE) + folio_size(folio) > fc->max_write)
6ddf3af9
MS		 2117 return true;
2118
2119 /* Discontinuity */
2619a6d4 2120 if (folio_next_index(ap->folios[ap->num_folios - 1]) != folio->index)
6ddf3af9
MS		 2121 return true;
2122
2123 /* Need to grow the pages array? If so, did the expansion fail? */
cbe9c115 2124 if (ap->num_folios == data->max_folios && !fuse_pages_realloc(data))
6ddf3af9
MS		 2125 return true;
2126
2127 return false;
2128}
2129
d585bdbe 2130static int fuse_writepages_fill(struct folio *folio,
26d614df
PE		 2131 struct writeback_control *wbc, void *_data)
2132{
2133 struct fuse_fill_wb_data *data = _data;
33826ebb
MS		 2134 struct fuse_writepage_args *wpa = data->wpa;
2135 struct fuse_args_pages *ap = &wpa->ia.ap;
26d614df 2136 struct inode *inode = data->inode;
f15ecfef 2137 struct fuse_inode *fi = get_fuse_inode(inode);
26d614df 2138 struct fuse_conn *fc = get_fuse_conn(inode);
26d614df
PE		 2139 int err;
2140
184429a1
MS		 2141 if (!data->ff) {
2142 err = -EIO;
2143 data->ff = fuse_write_file_get(fi);
2144 if (!data->ff)
2145 goto out_unlock;
2146 }
2147
6930b8da 2148 if (wpa && fuse_writepage_need_send(fc, folio, ap, data)) {
8b284dc4 2149 fuse_writepages_send(data);
33826ebb 2150 data->wpa = NULL;
f3cb8bd9 2151 data->nr_pages = 0;
26d614df 2152 }
e52a8250 2153
33826ebb 2154 if (data->wpa == NULL) {
26d614df 2155 err = -ENOMEM;
0acad928 2156 wpa = fuse_writepage_args_setup(folio, data->ff);
0c58a97f 2157 if (!wpa)
26d614df 2158 goto out_unlock;
0acad928 2159 fuse_file_get(wpa->ia.ff);
cbe9c115 2160 data->max_folios = 1;
33826ebb 2161 ap = &wpa->ia.ap;
26d614df 2162 }
d585bdbe 2163 folio_start_writeback(folio);
26d614df 2164
0c58a97f 2165 fuse_writepage_args_page_fill(wpa, folio, ap->num_folios);
f3cb8bd9 2166 data->nr_pages += folio_nr_pages(folio);
26d614df 2167
8b284dc4 2168 err = 0;
0c58a97f
JK		 2169 ap->num_folios++;
2170 if (!data->wpa)
c146024e 2171 data->wpa = wpa;
26d614df 2172out_unlock:
d585bdbe 2173 folio_unlock(folio);
26d614df
PE		 2174
2175 return err;
2176}
2177
2178static int fuse_writepages(struct address_space *mapping,
2179 struct writeback_control *wbc)
2180{
2181 struct inode *inode = mapping->host;
5da784cc 2182 struct fuse_conn *fc = get_fuse_conn(inode);
26d614df
PE		 2183 struct fuse_fill_wb_data data;
2184 int err;
2185
184429a1 2186 err = -EIO;
5d069dbe 2187 if (fuse_is_bad(inode))
184429a1 2188 goto out;
26d614df 2189
670d21c6
N		 2190 if (wbc->sync_mode == WB_SYNC_NONE &&
2191 fc->num_background >= fc->congestion_threshold)
2192 return 0;
2193
26d614df 2194 data.inode = inode;
33826ebb 2195 data.wpa = NULL;
184429a1 2196 data.ff = NULL;
f3cb8bd9 2197 data.nr_pages = 0;
2d033eaa 2198
26d614df 2199 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
33826ebb 2200 if (data.wpa) {
cbe9c115 2201 WARN_ON(!data.wpa->ia.ap.num_folios);
26d614df 2202 fuse_writepages_send(&data);
26d614df 2203 }
184429a1
MS		 2204 if (data.ff)
2205 fuse_file_put(data.ff, false);
2d033eaa 2206
26d614df
PE		 2207out:
2208 return err;
2209}
2210
6b12c1b3
PE		 2211/*
2212 * It's worthy to make sure that space is reserved on disk for the write,
2213 * but how to implement it without killing performance need more thinking.
2214 */
2215static int fuse_write_begin(struct file *file, struct address_space *mapping,
1da86618 2216 loff_t pos, unsigned len, struct folio **foliop, void **fsdata)
6b12c1b3 2217{
09cbfeaf 2218 pgoff_t index = pos >> PAGE_SHIFT;
a455589f 2219 struct fuse_conn *fc = get_fuse_conn(file_inode(file));
a060d835 2220 struct folio *folio;
6b12c1b3
PE		 2221 loff_t fsize;
2222 int err = -ENOMEM;
2223
2224 WARN_ON(!fc->writeback_cache);
2225
a060d835
MWO		 2226 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
2227 mapping_gfp_mask(mapping));
2228 if (IS_ERR(folio))
6b12c1b3
PE		 2229 goto error;
2230
a060d835 2231 if (folio_test_uptodate(folio) || len >= folio_size(folio))
6b12c1b3
PE		 2232 goto success;
2233 /*
a060d835
MWO		 2234 * Check if the start of this folio comes after the end of file,
2235 * in which case the readpage can be optimized away.
6b12c1b3
PE		 2236 */
2237 fsize = i_size_read(mapping->host);
a060d835
MWO		 2238 if (fsize <= folio_pos(folio)) {
2239 size_t off = offset_in_folio(folio, pos);
6b12c1b3 2240 if (off)
a060d835 2241 folio_zero_segment(folio, 0, off);
6b12c1b3
PE		 2242 goto success;
2243 }
65fe891d 2244 err = fuse_do_readfolio(file, folio);
6b12c1b3
PE		 2245 if (err)
2246 goto cleanup;
2247success:
1da86618 2248 *foliop = folio;
6b12c1b3
PE		 2249 return 0;
2250
2251cleanup:
a060d835
MWO		 2252 folio_unlock(folio);
2253 folio_put(folio);
6b12c1b3
PE		 2254error:
2255 return err;
2256}
2257
2258static int fuse_write_end(struct file *file, struct address_space *mapping,
2259 loff_t pos, unsigned len, unsigned copied,
a225800f 2260 struct folio *folio, void *fsdata)
6b12c1b3 2261{
556d0ac0 2262 struct inode *inode = folio->mapping->host;
6b12c1b3 2263
59c3b76c
MS		 2264 /* Haven't copied anything? Skip zeroing, size extending, dirtying. */
2265 if (!copied)
2266 goto unlock;
2267
8c56e03d 2268 pos += copied;
556d0ac0 2269 if (!folio_test_uptodate(folio)) {
6b12c1b3 2270 /* Zero any unwritten bytes at the end of the page */
8c56e03d 2271 size_t endoff = pos & ~PAGE_MASK;
6b12c1b3 2272 if (endoff)
556d0ac0
MWO		 2273 folio_zero_segment(folio, endoff, PAGE_SIZE);
2274 folio_mark_uptodate(folio);
6b12c1b3
PE		 2275 }
2276
8c56e03d
MS		 2277 if (pos > inode->i_size)
2278 i_size_write(inode, pos);
2279
556d0ac0 2280 folio_mark_dirty(folio);
59c3b76c
MS		 2281
2282unlock:
556d0ac0
MWO		 2283 folio_unlock(folio);
2284 folio_put(folio);
6b12c1b3
PE		 2285
2286 return copied;
2287}
2288
2bf06b8e 2289static int fuse_launder_folio(struct folio *folio)
3be5a52b
MS		 2290{
2291 int err = 0;
2bf06b8e 2292 if (folio_clear_dirty_for_io(folio)) {
e0887e09 2293 err = fuse_writepage_locked(folio);
3be5a52b 2294 if (!err)
0c58a97f 2295 folio_wait_writeback(folio);
3be5a52b
MS		 2296 }
2297 return err;
2298}
2299
2300/*
36ea2337
MS		 2301 * Write back dirty data/metadata now (there may not be any suitable
2302 * open files later for data)
3be5a52b
MS		 2303 */
2304static void fuse_vma_close(struct vm_area_struct *vma)
2305{
36ea2337
MS		 2306 int err;
2307
2308 err = write_inode_now(vma->vm_file->f_mapping->host, 1);
2309 mapping_set_error(vma->vm_file->f_mapping, err);
3be5a52b
MS		 2310}
2311
2312/*
2313 * Wait for writeback against this page to complete before allowing it
2314 * to be marked dirty again, and hence written back again, possibly
2315 * before the previous writepage completed.
2316 *
2317 * Block here, instead of in ->writepage(), so that the userspace fs
2318 * can only block processes actually operating on the filesystem.
2319 *
2320 * Otherwise unprivileged userspace fs would be able to block
2321 * unrelated:
2322 *
2323 * - page migration
2324 * - sync(2)
2325 * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
2326 */
46fb504a 2327static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
3be5a52b 2328{
184b6eb3 2329 struct folio *folio = page_folio(vmf->page);
11bac800 2330 struct inode *inode = file_inode(vmf->vma->vm_file);
cca24370 2331
11bac800 2332 file_update_time(vmf->vma->vm_file);
184b6eb3
JB		 2333 folio_lock(folio);
2334 if (folio->mapping != inode->i_mapping) {
2335 folio_unlock(folio);
cca24370
MS		 2336 return VM_FAULT_NOPAGE;
2337 }
3be5a52b 2338
0c58a97f 2339 folio_wait_writeback(folio);
cca24370 2340 return VM_FAULT_LOCKED;
3be5a52b
MS		 2341}
2342
f0f37e2f 2343static const struct vm_operations_struct fuse_file_vm_ops = {
3be5a52b
MS		 2344 .close = fuse_vma_close,
2345 .fault = filemap_fault,
f1820361 2346 .map_pages = filemap_map_pages,
3be5a52b
MS		 2347 .page_mkwrite = fuse_page_mkwrite,
2348};
2349
2350static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2351{
55752a3a 2352 struct fuse_file *ff = file->private_data;
e78662e8 2353 struct fuse_conn *fc = ff->fm->fc;
fda0b98e 2354 struct inode *inode = file_inode(file);
cb098dd2 2355 int rc;
55752a3a 2356
2a9a609a 2357 /* DAX mmap is superior to direct_io mmap */
fda0b98e 2358 if (FUSE_IS_DAX(inode))
2a9a609a
SH		 2359 return fuse_dax_mmap(file, vma);
2360
fda0b98e
AG		 2361 /*
2362 * If inode is in passthrough io mode, because it has some file open
2363 * in passthrough mode, either mmap to backing file or fail mmap,
2364 * because mixing cached mmap and passthrough io mode is not allowed.
2365 */
4a90451b 2366 if (fuse_file_passthrough(ff))
fda0b98e
AG		 2367 return fuse_passthrough_mmap(file, vma);
2368 else if (fuse_inode_backing(get_fuse_inode(inode)))
4a90451b
AG		 2369 return -ENODEV;
2370
205c1d80
AG		 2371 /*
2372 * FOPEN_DIRECT_IO handling is special compared to O_DIRECT,
2373 * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP.
2374 */
55752a3a 2375 if (ff->open_flags & FOPEN_DIRECT_IO) {
9511176b
BS		 2376 /*
2377 * Can't provide the coherency needed for MAP_SHARED
c55e0a55 2378 * if FUSE_DIRECT_IO_ALLOW_MMAP isn't set.
e78662e8 2379 */
c55e0a55 2380 if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_allow_mmap)
55752a3a
MS		 2381 return -ENODEV;
2382
2383 invalidate_inode_pages2(file->f_mapping);
2384
9511176b
BS		 2385 if (!(vma->vm_flags & VM_MAYSHARE)) {
2386 /* MAP_PRIVATE */
2387 return generic_file_mmap(file, vma);
2388 }
cb098dd2 2389
205c1d80
AG		 2390 /*
2391 * First mmap of direct_io file enters caching inode io mode.
2392 * Also waits for parallel dio writers to go into serial mode
2393 * (exclusive instead of shared lock).
4864a6dd
AG		 2394 * After first mmap, the inode stays in caching io mode until
2395 * the direct_io file release.
205c1d80 2396 */
4864a6dd 2397 rc = fuse_file_cached_io_open(inode, ff);
cb098dd2
AG		 2398 if (rc)
2399 return rc;
55752a3a
MS		 2400 }
2401
650b22b9
PE		 2402 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2403 fuse_link_write_file(file);
2404
3be5a52b
MS		 2405 file_accessed(file);
2406 vma->vm_ops = &fuse_file_vm_ops;
b6aeaded
MS		 2407 return 0;
2408}
2409
0b6e9ea0
SF		 2410static int convert_fuse_file_lock(struct fuse_conn *fc,
2411 const struct fuse_file_lock *ffl,
71421259
MS		 2412 struct file_lock *fl)
2413{
2414 switch (ffl->type) {
2415 case F_UNLCK:
2416 break;
2417
2418 case F_RDLCK:
2419 case F_WRLCK:
2420 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2421 ffl->end < ffl->start)
2422 return -EIO;
2423
2424 fl->fl_start = ffl->start;
2425 fl->fl_end = ffl->end;
0b6e9ea0
SF		 2426
2427 /*
9d5b86ac
BC		 2428 * Convert pid into init's pid namespace. The locks API will
2429 * translate it into the caller's pid namespace.
0b6e9ea0
SF		 2430 */
2431 rcu_read_lock();
9a7eec48 2432 fl->c.flc_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
0b6e9ea0 2433 rcu_read_unlock();
71421259
MS		 2434 break;
2435
2436 default:
2437 return -EIO;
2438 }
9a7eec48 2439 fl->c.flc_type = ffl->type;
71421259
MS		 2440 return 0;
2441}
2442
7078187a 2443static void fuse_lk_fill(struct fuse_args *args, struct file *file,
a9ff4f87 2444 const struct file_lock *fl, int opcode, pid_t pid,
7078187a 2445 int flock, struct fuse_lk_in *inarg)
71421259 2446{
6131ffaa 2447 struct inode *inode = file_inode(file);
9c8ef561 2448 struct fuse_conn *fc = get_fuse_conn(inode);
71421259 2449 struct fuse_file *ff = file->private_data;
7078187a
MS		 2450
2451 memset(inarg, 0, sizeof(*inarg));
2452 inarg->fh = ff->fh;
9a7eec48 2453 inarg->owner = fuse_lock_owner_id(fc, fl->c.flc_owner);
7078187a
MS		 2454 inarg->lk.start = fl->fl_start;
2455 inarg->lk.end = fl->fl_end;
9a7eec48 2456 inarg->lk.type = fl->c.flc_type;
7078187a 2457 inarg->lk.pid = pid;
a9ff4f87 2458 if (flock)
7078187a 2459 inarg->lk_flags |= FUSE_LK_FLOCK;
d5b48543
MS		 2460 args->opcode = opcode;
2461 args->nodeid = get_node_id(inode);
2462 args->in_numargs = 1;
2463 args->in_args[0].size = sizeof(*inarg);
2464 args->in_args[0].value = inarg;
71421259
MS		 2465}
2466
2467static int fuse_getlk(struct file *file, struct file_lock *fl)
2468{
6131ffaa 2469 struct inode *inode = file_inode(file);
fcee216b 2470 struct fuse_mount *fm = get_fuse_mount(inode);
7078187a
MS		 2471 FUSE_ARGS(args);
2472 struct fuse_lk_in inarg;
71421259
MS		 2473 struct fuse_lk_out outarg;
2474 int err;
2475
7078187a 2476 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
d5b48543
MS		 2477 args.out_numargs = 1;
2478 args.out_args[0].size = sizeof(outarg);
2479 args.out_args[0].value = &outarg;
fcee216b 2480 err = fuse_simple_request(fm, &args);
71421259 2481 if (!err)
fcee216b 2482 err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
71421259
MS		 2483
2484 return err;
2485}
2486
a9ff4f87 2487static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
71421259 2488{
6131ffaa 2489 struct inode *inode = file_inode(file);
fcee216b 2490 struct fuse_mount *fm = get_fuse_mount(inode);
7078187a
MS		 2491 FUSE_ARGS(args);
2492 struct fuse_lk_in inarg;
9a7eec48
JL		 2493 int opcode = (fl->c.flc_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2494 struct pid *pid = fl->c.flc_type != F_UNLCK ? task_tgid(current) : NULL;
fcee216b 2495 pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
71421259
MS		 2496 int err;
2497
8fb47a4f 2498 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
48e90761
MS		 2499 /* NLM needs asynchronous locks, which we don't support yet */
2500 return -ENOLCK;
2501 }
2502
0b6e9ea0 2503 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
fcee216b 2504 err = fuse_simple_request(fm, &args);
71421259 2505
a4d27e75
MS		 2506 /* locking is restartable */
2507 if (err == -EINTR)
2508 err = -ERESTARTSYS;
7078187a 2509
71421259
MS		 2510 return err;
2511}
2512
2513static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2514{
6131ffaa 2515 struct inode *inode = file_inode(file);
71421259
MS		 2516 struct fuse_conn *fc = get_fuse_conn(inode);
2517 int err;
2518
48e90761
MS		 2519 if (cmd == F_CANCELLK) {
2520 err = 0;
2521 } else if (cmd == F_GETLK) {
71421259 2522 if (fc->no_lock) {
9d6a8c5c 2523 posix_test_lock(file, fl);
71421259
MS		 2524 err = 0;
2525 } else
2526 err = fuse_getlk(file, fl);
2527 } else {
2528 if (fc->no_lock)
48e90761 2529 err = posix_lock_file(file, fl, NULL);
71421259 2530 else
a9ff4f87 2531 err = fuse_setlk(file, fl, 0);
71421259
MS		 2532 }
2533 return err;
2534}
2535
a9ff4f87
MS		 2536static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2537{
6131ffaa 2538 struct inode *inode = file_inode(file);
a9ff4f87
MS		 2539 struct fuse_conn *fc = get_fuse_conn(inode);
2540 int err;
2541
37fb3a30 2542 if (fc->no_flock) {
4f656367 2543 err = locks_lock_file_wait(file, fl);
a9ff4f87 2544 } else {
37fb3a30
MS		 2545 struct fuse_file *ff = file->private_data;
2546
a9ff4f87 2547 /* emulate flock with POSIX locks */
37fb3a30 2548 ff->flock = true;
a9ff4f87
MS		 2549 err = fuse_setlk(file, fl, 1);
2550 }
2551
2552 return err;
2553}
2554
b2d2272f
MS		 2555static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2556{
2557 struct inode *inode = mapping->host;
fcee216b 2558 struct fuse_mount *fm = get_fuse_mount(inode);
7078187a 2559 FUSE_ARGS(args);
b2d2272f
MS		 2560 struct fuse_bmap_in inarg;
2561 struct fuse_bmap_out outarg;
2562 int err;
2563
fcee216b 2564 if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
b2d2272f
MS		 2565 return 0;
2566
b2d2272f
MS		 2567 memset(&inarg, 0, sizeof(inarg));
2568 inarg.block = block;
2569 inarg.blocksize = inode->i_sb->s_blocksize;
d5b48543
MS		 2570 args.opcode = FUSE_BMAP;
2571 args.nodeid = get_node_id(inode);
2572 args.in_numargs = 1;
2573 args.in_args[0].size = sizeof(inarg);
2574 args.in_args[0].value = &inarg;
2575 args.out_numargs = 1;
2576 args.out_args[0].size = sizeof(outarg);
2577 args.out_args[0].value = &outarg;
fcee216b 2578 err = fuse_simple_request(fm, &args);
b2d2272f 2579 if (err == -ENOSYS)
fcee216b 2580 fm->fc->no_bmap = 1;
b2d2272f
MS		 2581
2582 return err ? 0 : outarg.block;
2583}
2584
0b5da8db
R		 2585static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2586{
2587 struct inode *inode = file->f_mapping->host;
fcee216b 2588 struct fuse_mount *fm = get_fuse_mount(inode);
0b5da8db
R		 2589 struct fuse_file *ff = file->private_data;
2590 FUSE_ARGS(args);
2591 struct fuse_lseek_in inarg = {
2592 .fh = ff->fh,
2593 .offset = offset,
2594 .whence = whence
2595 };
2596 struct fuse_lseek_out outarg;
2597 int err;
2598
fcee216b 2599 if (fm->fc->no_lseek)
0b5da8db
R		 2600 goto fallback;
2601
d5b48543
MS		 2602 args.opcode = FUSE_LSEEK;
2603 args.nodeid = ff->nodeid;
2604 args.in_numargs = 1;
2605 args.in_args[0].size = sizeof(inarg);
2606 args.in_args[0].value = &inarg;
2607 args.out_numargs = 1;
2608 args.out_args[0].size = sizeof(outarg);
2609 args.out_args[0].value = &outarg;
fcee216b 2610 err = fuse_simple_request(fm, &args);
0b5da8db
R		 2611 if (err) {
2612 if (err == -ENOSYS) {
fcee216b 2613 fm->fc->no_lseek = 1;
0b5da8db
R		 2614 goto fallback;
2615 }
2616 return err;
2617 }
2618
2619 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2620
2621fallback:
c6c745b8 2622 err = fuse_update_attributes(inode, file, STATX_SIZE);
0b5da8db
R		 2623 if (!err)
2624 return generic_file_llseek(file, offset, whence);
2625 else
2626 return err;
2627}
2628
965c8e59 2629static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
5559b8f4
MS		 2630{
2631 loff_t retval;
6131ffaa 2632 struct inode *inode = file_inode(file);
5559b8f4 2633
0b5da8db
R		 2634 switch (whence) {
2635 case SEEK_SET:
2636 case SEEK_CUR:
2637 /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
965c8e59 2638 retval = generic_file_llseek(file, offset, whence);
0b5da8db
R		 2639 break;
2640 case SEEK_END:
5955102c 2641 inode_lock(inode);
c6c745b8 2642 retval = fuse_update_attributes(inode, file, STATX_SIZE);
0b5da8db
R		 2643 if (!retval)
2644 retval = generic_file_llseek(file, offset, whence);
5955102c 2645 inode_unlock(inode);
0b5da8db
R		 2646 break;
2647 case SEEK_HOLE:
2648 case SEEK_DATA:
5955102c 2649 inode_lock(inode);
0b5da8db 2650 retval = fuse_lseek(file, offset, whence);
5955102c 2651 inode_unlock(inode);
0b5da8db
R		 2652 break;
2653 default:
2654 retval = -EINVAL;
2655 }
c07c3d19 2656
5559b8f4
MS		 2657 return retval;
2658}
2659
95668a69
TH		 2660/*
2661 * All files which have been polled are linked to RB tree
2662 * fuse_conn->polled_files which is indexed by kh. Walk the tree and
2663 * find the matching one.
2664 */
2665static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2666 struct rb_node **parent_out)
2667{
2668 struct rb_node **link = &fc->polled_files.rb_node;
2669 struct rb_node *last = NULL;
2670
2671 while (*link) {
2672 struct fuse_file *ff;
2673
2674 last = *link;
2675 ff = rb_entry(last, struct fuse_file, polled_node);
2676
2677 if (kh < ff->kh)
2678 link = &last->rb_left;
2679 else if (kh > ff->kh)
2680 link = &last->rb_right;
2681 else
2682 return link;
2683 }
2684
2685 if (parent_out)
2686 *parent_out = last;
2687 return link;
2688}
2689
2690/*
2691 * The file is about to be polled. Make sure it's on the polled_files
2692 * RB tree. Note that files once added to the polled_files tree are
2693 * not removed before the file is released. This is because a file
2694 * polled once is likely to be polled again.
2695 */
2696static void fuse_register_polled_file(struct fuse_conn *fc,
2697 struct fuse_file *ff)
2698{
2699 spin_lock(&fc->lock);
2700 if (RB_EMPTY_NODE(&ff->polled_node)) {
3f649ab7 2701 struct rb_node **link, *parent;
95668a69
TH		 2702
2703 link = fuse_find_polled_node(fc, ff->kh, &parent);
2704 BUG_ON(*link);
2705 rb_link_node(&ff->polled_node, parent, link);
2706 rb_insert_color(&ff->polled_node, &fc->polled_files);
2707 }
2708 spin_unlock(&fc->lock);
2709}
2710
076ccb76 2711__poll_t fuse_file_poll(struct file *file, poll_table *wait)
95668a69 2712{
95668a69 2713 struct fuse_file *ff = file->private_data;
fcee216b 2714 struct fuse_mount *fm = ff->fm;
95668a69
TH		 2715 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2716 struct fuse_poll_out outarg;
7078187a 2717 FUSE_ARGS(args);
95668a69
TH		 2718 int err;
2719
fcee216b 2720 if (fm->fc->no_poll)
95668a69
TH		 2721 return DEFAULT_POLLMASK;
2722
2723 poll_wait(file, &ff->poll_wait, wait);
c71d227f 2724 inarg.events = mangle_poll(poll_requested_events(wait));
95668a69
TH		 2725
2726 /*
2727 * Ask for notification iff there's someone waiting for it.
2728 * The client may ignore the flag and always notify.
2729 */
2730 if (waitqueue_active(&ff->poll_wait)) {
2731 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
fcee216b 2732 fuse_register_polled_file(fm->fc, ff);
95668a69
TH		 2733 }
2734
d5b48543
MS		 2735 args.opcode = FUSE_POLL;
2736 args.nodeid = ff->nodeid;
2737 args.in_numargs = 1;
2738 args.in_args[0].size = sizeof(inarg);
2739 args.in_args[0].value = &inarg;
2740 args.out_numargs = 1;
2741 args.out_args[0].size = sizeof(outarg);
2742 args.out_args[0].value = &outarg;
fcee216b 2743 err = fuse_simple_request(fm, &args);
95668a69
TH		 2744
2745 if (!err)
c71d227f 2746 return demangle_poll(outarg.revents);
95668a69 2747 if (err == -ENOSYS) {
fcee216b 2748 fm->fc->no_poll = 1;
95668a69
TH		 2749 return DEFAULT_POLLMASK;
2750 }
a9a08845 2751 return EPOLLERR;
95668a69 2752}
08cbf542 2753EXPORT_SYMBOL_GPL(fuse_file_poll);
95668a69
TH		 2754
2755/*
2756 * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
2757 * wakes up the poll waiters.
2758 */
2759int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2760 struct fuse_notify_poll_wakeup_out *outarg)
2761{
2762 u64 kh = outarg->kh;
2763 struct rb_node **link;
2764
2765 spin_lock(&fc->lock);
2766
2767 link = fuse_find_polled_node(fc, kh, NULL);
2768 if (*link) {
2769 struct fuse_file *ff;
2770
2771 ff = rb_entry(*link, struct fuse_file, polled_node);
2772 wake_up_interruptible_sync(&ff->poll_wait);
2773 }
2774
2775 spin_unlock(&fc->lock);
2776 return 0;
2777}
2778
efb9fa9e
MP		 2779static void fuse_do_truncate(struct file *file)
2780{
2781 struct inode *inode = file->f_mapping->host;
2782 struct iattr attr;
2783
2784 attr.ia_valid = ATTR_SIZE;
2785 attr.ia_size = i_size_read(inode);
2786
2787 attr.ia_file = file;
2788 attr.ia_valid |= ATTR_FILE;
2789
276a0256 2790 fuse_do_setattr(file_mnt_idmap(file), file_dentry(file), &attr, file);
efb9fa9e
MP		 2791}
2792
5da784cc 2793static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
e5c5f05d 2794{
5da784cc 2795 return round_up(off, fc->max_pages << PAGE_SHIFT);
e5c5f05d
MP		 2796}
2797
4273b793 2798static ssize_t
c8b8e32d 2799fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
4273b793 2800{
9d5722b7 2801 DECLARE_COMPLETION_ONSTACK(wait);
4273b793 2802 ssize_t ret = 0;
60b9df7a
MS		 2803 struct file *file = iocb->ki_filp;
2804 struct fuse_file *ff = file->private_data;
4273b793 2805 loff_t pos = 0;
bcba24cc
MP		 2806 struct inode *inode;
2807 loff_t i_size;
933a3752 2808 size_t count = iov_iter_count(iter), shortened = 0;
c8b8e32d 2809 loff_t offset = iocb->ki_pos;
36cf66ed 2810 struct fuse_io_priv *io;
4273b793 2811
4273b793 2812 pos = offset;
bcba24cc
MP		 2813 inode = file->f_mapping->host;
2814 i_size = i_size_read(inode);
4273b793 2815
933a3752 2816 if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
9fe55eea
SW		 2817 return 0;
2818
bcba24cc 2819 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
36cf66ed
MP		 2820 if (!io)
2821 return -ENOMEM;
bcba24cc 2822 spin_lock_init(&io->lock);
744742d6 2823 kref_init(&io->refcnt);
bcba24cc
MP		 2824 io->reqs = 1;
2825 io->bytes = -1;
2826 io->size = 0;
2827 io->offset = offset;
6f673763 2828 io->write = (iov_iter_rw(iter) == WRITE);
bcba24cc 2829 io->err = 0;
bcba24cc
MP		 2830 /*
2831 * By default, we want to optimize all I/Os with async request
60b9df7a 2832 * submission to the client filesystem if supported.
bcba24cc 2833 */
69456535 2834 io->async = ff->fm->fc->async_dio;
bcba24cc 2835 io->iocb = iocb;
7879c4e5 2836 io->blocking = is_sync_kiocb(iocb);
bcba24cc 2837
933a3752
AV		 2838 /* optimization for short read */
2839 if (io->async && !io->write && offset + count > i_size) {
69456535 2840 iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
933a3752
AV		 2841 shortened = count - iov_iter_count(iter);
2842 count -= shortened;
2843 }
2844
bcba24cc 2845 /*
7879c4e5
AS		 2846 * We cannot asynchronously extend the size of a file.
2847 * In such case the aio will behave exactly like sync io.
bcba24cc 2848 */
933a3752 2849 if ((offset + count > i_size) && io->write)
7879c4e5 2850 io->blocking = true;
4273b793 2851
7879c4e5 2852 if (io->async && io->blocking) {
744742d6
SF		 2853 /*
2854 * Additional reference to keep io around after
2855 * calling fuse_aio_complete()
2856 */
2857 kref_get(&io->refcnt);
9d5722b7 2858 io->done = &wait;
744742d6 2859 }
9d5722b7 2860
6f673763 2861 if (iov_iter_rw(iter) == WRITE) {
6b775b18 2862 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
fa5eee57 2863 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
812408fb 2864 } else {
d22a943f 2865 ret = __fuse_direct_read(io, iter, &pos);
812408fb 2866 }
933a3752 2867 iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
36cf66ed 2868
bcba24cc 2869 if (io->async) {
ebacb812
LC		 2870 bool blocking = io->blocking;
2871
bcba24cc
MP		 2872 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2873
2874 /* we have a non-extending, async request, so return */
ebacb812 2875 if (!blocking)
bcba24cc
MP		 2876 return -EIOCBQUEUED;
2877
9d5722b7
CH		 2878 wait_for_completion(&wait);
2879 ret = fuse_get_res_by_io(io);
bcba24cc
MP		 2880 }
2881
744742d6 2882 kref_put(&io->refcnt, fuse_io_release);
9d5722b7 2883
6f673763 2884 if (iov_iter_rw(iter) == WRITE) {
d347739a 2885 fuse_write_update_attr(inode, pos, ret);
15352405 2886 /* For extending writes we already hold exclusive lock */
d347739a 2887 if (ret < 0 && offset + count > i_size)
efb9fa9e
MP		 2888 fuse_do_truncate(file);
2889 }
4273b793
AA		 2890
2891 return ret;
2892}
2893
26eb3bae
MS		 2894static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
2895{
e388164e 2896 int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
26eb3bae
MS		 2897
2898 if (!err)
2899 fuse_sync_writes(inode);
2900
2901 return err;
2902}
2903
cdadb11c
MS		 2904static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
2905 loff_t length)
05ba1f08
AP		 2906{
2907 struct fuse_file *ff = file->private_data;
1c68271c 2908 struct inode *inode = file_inode(file);
0ab08f57 2909 struct fuse_inode *fi = get_fuse_inode(inode);
fcee216b 2910 struct fuse_mount *fm = ff->fm;
7078187a 2911 FUSE_ARGS(args);
05ba1f08
AP		 2912 struct fuse_fallocate_in inarg = {
2913 .fh = ff->fh,
2914 .offset = offset,
2915 .length = length,
2916 .mode = mode
2917 };
2918 int err;
44361e8c
MS		 2919 bool block_faults = FUSE_IS_DAX(inode) &&
2920 (!(mode & FALLOC_FL_KEEP_SIZE) ||
2921 (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)));
6ae330ca 2922
6b1bdb56
RJ		 2923 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
2924 FALLOC_FL_ZERO_RANGE))
4adb8302
MS		 2925 return -EOPNOTSUPP;
2926
fcee216b 2927 if (fm->fc->no_fallocate)
519c6040
MS		 2928 return -EOPNOTSUPP;
2929
44361e8c
MS		 2930 inode_lock(inode);
2931 if (block_faults) {
2932 filemap_invalidate_lock(inode->i_mapping);
7851bf64 2933 err = fuse_dax_break_layouts(inode, 0, -1);
44361e8c
MS		 2934 if (err)
2935 goto out;
2936 }
6ae330ca 2937
44361e8c
MS		 2938 if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) {
2939 loff_t endbyte = offset + length - 1;
26eb3bae 2940
44361e8c
MS		 2941 err = fuse_writeback_range(inode, offset, endbyte);
2942 if (err)
2943 goto out;
3634a632
BF		 2944 }
2945
0cbade02
LB		 2946 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
2947 offset + length > i_size_read(inode)) {
2948 err = inode_newsize_ok(inode, offset + length);
2949 if (err)
35d6fcbb 2950 goto out;
0cbade02
LB		 2951 }
2952
4a6f278d
MS		 2953 err = file_modified(file);
2954 if (err)
2955 goto out;
2956
0ab08f57
MP		 2957 if (!(mode & FALLOC_FL_KEEP_SIZE))
2958 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2959
d5b48543
MS		 2960 args.opcode = FUSE_FALLOCATE;
2961 args.nodeid = ff->nodeid;
2962 args.in_numargs = 1;
2963 args.in_args[0].size = sizeof(inarg);
2964 args.in_args[0].value = &inarg;
fcee216b 2965 err = fuse_simple_request(fm, &args);
519c6040 2966 if (err == -ENOSYS) {
fcee216b 2967 fm->fc->no_fallocate = 1;
519c6040
MS		 2968 err = -EOPNOTSUPP;
2969 }
bee6c307
BF		 2970 if (err)
2971 goto out;
2972
2973 /* we could have extended the file */
b0aa7606 2974 if (!(mode & FALLOC_FL_KEEP_SIZE)) {
20235b43 2975 if (fuse_write_update_attr(inode, offset + length, length))
93d2269d 2976 file_update_time(file);
b0aa7606 2977 }
bee6c307 2978
6b1bdb56 2979 if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))
bee6c307
BF		 2980 truncate_pagecache_range(inode, offset, offset + length - 1);
2981
fa5eee57 2982 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
bee6c307 2983
3634a632 2984out:
0ab08f57
MP		 2985 if (!(mode & FALLOC_FL_KEEP_SIZE))
2986 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2987
6ae330ca 2988 if (block_faults)
8bcbbe9c 2989 filemap_invalidate_unlock(inode->i_mapping);
6ae330ca 2990
44361e8c 2991 inode_unlock(inode);
3634a632 2992
5c791fe1
MS		 2993 fuse_flush_time_update(inode);
2994
05ba1f08
AP		 2995 return err;
2996}
05ba1f08 2997
64bf5ff5
DC		 2998static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
2999 struct file *file_out, loff_t pos_out,
3000 size_t len, unsigned int flags)
88bc7d50
NV		 3001{
3002 struct fuse_file *ff_in = file_in->private_data;
3003 struct fuse_file *ff_out = file_out->private_data;
a2bc9236 3004 struct inode *inode_in = file_inode(file_in);
88bc7d50
NV		 3005 struct inode *inode_out = file_inode(file_out);
3006 struct fuse_inode *fi_out = get_fuse_inode(inode_out);
fcee216b
MR		 3007 struct fuse_mount *fm = ff_in->fm;
3008 struct fuse_conn *fc = fm->fc;
88bc7d50
NV		 3009 FUSE_ARGS(args);
3010 struct fuse_copy_file_range_in inarg = {
3011 .fh_in = ff_in->fh,
3012 .off_in = pos_in,
3013 .nodeid_out = ff_out->nodeid,
3014 .fh_out = ff_out->fh,
3015 .off_out = pos_out,
3016 .len = len,
3017 .flags = flags
3018 };
3019 struct fuse_write_out outarg;
3020 ssize_t err;
3021 /* mark unstable when write-back is not used, and file_out gets
3022 * extended */
3023 bool is_unstable = (!fc->writeback_cache) &&
3024 ((pos_out + len) > inode_out->i_size);
3025
3026 if (fc->no_copy_file_range)
3027 return -EOPNOTSUPP;
3028
5dae222a
AG		 3029 if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3030 return -EXDEV;
3031
2c4656df
MS		 3032 inode_lock(inode_in);
3033 err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
3034 inode_unlock(inode_in);
3035 if (err)
3036 return err;
a2bc9236 3037
88bc7d50
NV		 3038 inode_lock(inode_out);
3039
fe0da9c0
AG		 3040 err = file_modified(file_out);
3041 if (err)
3042 goto out;
3043
9b46418c
MS		 3044 /*
3045 * Write out dirty pages in the destination file before sending the COPY
3046 * request to userspace. After the request is completed, truncate off
3047 * pages (including partial ones) from the cache that have been copied,
3048 * since these contain stale data at that point.
3049 *
3050 * This should be mostly correct, but if the COPY writes to partial
3051 * pages (at the start or end) and the parts not covered by the COPY are
3052 * written through a memory map after calling fuse_writeback_range(),
3053 * then these partial page modifications will be lost on truncation.
3054 *
3055 * It is unlikely that someone would rely on such mixed style
3056 * modifications. Yet this does give less guarantees than if the
3057 * copying was performed with write(2).
3058 *
8bcbbe9c 3059 * To fix this a mapping->invalidate_lock could be used to prevent new
9b46418c
MS		 3060 * faults while the copy is ongoing.
3061 */
2c4656df
MS		 3062 err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
3063 if (err)
3064 goto out;
88bc7d50
NV		 3065
3066 if (is_unstable)
3067 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3068
d5b48543
MS		 3069 args.opcode = FUSE_COPY_FILE_RANGE;
3070 args.nodeid = ff_in->nodeid;
3071 args.in_numargs = 1;
3072 args.in_args[0].size = sizeof(inarg);
3073 args.in_args[0].value = &inarg;
3074 args.out_numargs = 1;
3075 args.out_args[0].size = sizeof(outarg);
3076 args.out_args[0].value = &outarg;
fcee216b 3077 err = fuse_simple_request(fm, &args);
88bc7d50
NV		 3078 if (err == -ENOSYS) {
3079 fc->no_copy_file_range = 1;
3080 err = -EOPNOTSUPP;
3081 }
3082 if (err)
3083 goto out;
3084
9b46418c
MS		 3085 truncate_inode_pages_range(inode_out->i_mapping,
3086 ALIGN_DOWN(pos_out, PAGE_SIZE),
3087 ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1);
3088
20235b43
MS		 3089 file_update_time(file_out);
3090 fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size);
88bc7d50
NV		 3091
3092 err = outarg.size;
3093out:
3094 if (is_unstable)
3095 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3096
3097 inode_unlock(inode_out);
fe0da9c0 3098 file_accessed(file_in);
88bc7d50 3099
5c791fe1
MS		 3100 fuse_flush_time_update(inode_out);
3101
88bc7d50
NV		 3102 return err;
3103}
3104
64bf5ff5
DC		 3105static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3106 struct file *dst_file, loff_t dst_off,
3107 size_t len, unsigned int flags)
3108{
3109 ssize_t ret;
3110
3111 ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3112 len, flags);
3113
5dae222a 3114 if (ret == -EOPNOTSUPP || ret == -EXDEV)
705bcfcb
AG		 3115 ret = splice_copy_file_range(src_file, src_off, dst_file,
3116 dst_off, len);
64bf5ff5
DC		 3117 return ret;
3118}
3119
4b6f5d20 3120static const struct file_operations fuse_file_operations = {
5559b8f4 3121 .llseek = fuse_file_llseek,
37c20f16 3122 .read_iter = fuse_file_read_iter,
84c3d55c 3123 .write_iter = fuse_file_write_iter,
b6aeaded
MS		 3124 .mmap = fuse_file_mmap,
3125 .open = fuse_open,
3126 .flush = fuse_flush,
3127 .release = fuse_release,
3128 .fsync = fuse_fsync,
71421259 3129 .lock = fuse_file_lock,
2a9a609a 3130 .get_unmapped_area = thp_get_unmapped_area,
a9ff4f87 3131 .flock = fuse_file_flock,
5ca73468
AG		 3132 .splice_read = fuse_splice_read,
3133 .splice_write = fuse_splice_write,
59efec7b
TH		 3134 .unlocked_ioctl = fuse_file_ioctl,
3135 .compat_ioctl = fuse_file_compat_ioctl,
95668a69 3136 .poll = fuse_file_poll,
05ba1f08 3137 .fallocate = fuse_file_fallocate,
d4136d60 3138 .copy_file_range = fuse_copy_file_range,
413ef8cb
MS		 3139};
3140
f5e54d6e 3141static const struct address_space_operations fuse_file_aops = {
5efd00e4 3142 .read_folio = fuse_read_folio,
76a0294e 3143 .readahead = fuse_readahead,
26d614df 3144 .writepages = fuse_writepages,
2bf06b8e 3145 .launder_folio = fuse_launder_folio,
187c82cb 3146 .dirty_folio = filemap_dirty_folio,
e1c420ac 3147 .migrate_folio = filemap_migrate_folio,
b2d2272f 3148 .bmap = fuse_bmap,
4273b793 3149 .direct_IO = fuse_direct_IO,
6b12c1b3
PE		 3150 .write_begin = fuse_write_begin,
3151 .write_end = fuse_write_end,
b6aeaded
MS		 3152};
3153
93a497b9 3154void fuse_init_file_inode(struct inode *inode, unsigned int flags)
b6aeaded 3155{
ab2257e9 3156 struct fuse_inode *fi = get_fuse_inode(inode);
0c58a97f 3157 struct fuse_conn *fc = get_fuse_conn(inode);
ab2257e9 3158
45323fb7
MS		 3159 inode->i_fop = &fuse_file_operations;
3160 inode->i_data.a_ops = &fuse_file_aops;
0c58a97f
JK		 3161 if (fc->writeback_cache)
3162 mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data);
ab2257e9
MS		 3163
3164 INIT_LIST_HEAD(&fi->write_files);
3165 INIT_LIST_HEAD(&fi->queued_writes);
3166 fi->writectr = 0;
cb098dd2 3167 fi->iocachectr = 0;
ab2257e9 3168 init_waitqueue_head(&fi->page_waitq);
205c1d80 3169 init_waitqueue_head(&fi->direct_io_waitq);
c2d0ad00
VG		 3170
3171 if (IS_ENABLED(CONFIG_FUSE_DAX))
93a497b9 3172 fuse_dax_inode_init(inode, flags);
b6aeaded 3173}
Linux 6.x block layer and io_uring tree(s)