treewide: Add SPDX license identifier for missed files
[linux-block.git] / fs / nfs / direct.c
CommitLineData
457c8996 1// SPDX-License-Identifier: GPL-2.0-only
1da177e4
LT
2/*
3 * linux/fs/nfs/direct.c
4 *
5 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
6 *
7 * High-performance uncached I/O for the Linux NFS client
8 *
9 * There are important applications whose performance or correctness
10 * depends on uncached access to file data. Database clusters
88467055 11 * (multiple copies of the same instance running on separate hosts)
1da177e4 12 * implement their own cache coherency protocol that subsumes file
88467055
CL
13 * system cache protocols. Applications that process datasets
14 * considerably larger than the client's memory do not always benefit
15 * from a local cache. A streaming video server, for instance, has no
1da177e4
LT
16 * need to cache the contents of a file.
17 *
18 * When an application requests uncached I/O, all read and write requests
19 * are made directly to the server; data stored or fetched via these
20 * requests is not cached in the Linux page cache. The client does not
21 * correct unaligned requests from applications. All requested bytes are
22 * held on permanent storage before a direct write system call returns to
23 * an application.
24 *
25 * Solaris implements an uncached I/O facility called directio() that
26 * is used for backups and sequential I/O to very large files. Solaris
27 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
28 * an undocumented mount option.
29 *
30 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
31 * help from Andrew Morton.
32 *
33 * 18 Dec 2001 Initial implementation for 2.4 --cel
34 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
35 * 08 Jun 2003 Port to 2.5 APIs --cel
36 * 31 Mar 2004 Handle direct I/O without VFS support --cel
37 * 15 Sep 2004 Parallel async reads --cel
88467055 38 * 04 May 2005 support O_DIRECT with aio --cel
1da177e4
LT
39 *
40 */
41
1da177e4
LT
42#include <linux/errno.h>
43#include <linux/sched.h>
44#include <linux/kernel.h>
1da177e4
LT
45#include <linux/file.h>
46#include <linux/pagemap.h>
47#include <linux/kref.h>
5a0e3ad6 48#include <linux/slab.h>
7ec10f26 49#include <linux/task_io_accounting_ops.h>
6296556f 50#include <linux/module.h>
1da177e4
LT
51
52#include <linux/nfs_fs.h>
53#include <linux/nfs_page.h>
54#include <linux/sunrpc/clnt.h>
55
7c0f6ba6 56#include <linux/uaccess.h>
60063497 57#include <linux/atomic.h>
1da177e4 58
8d5658c9 59#include "internal.h"
91d5b470 60#include "iostat.h"
1763da12 61#include "pnfs.h"
91d5b470 62
1da177e4 63#define NFSDBG_FACILITY NFSDBG_VFS
1da177e4 64
e18b890b 65static struct kmem_cache *nfs_direct_cachep;
1da177e4
LT
66
67/*
68 * This represents a set of asynchronous requests that we're waiting on
69 */
0a00b77b
WAA
70struct nfs_direct_mirror {
71 ssize_t count;
72};
73
1da177e4
LT
74struct nfs_direct_req {
75 struct kref kref; /* release manager */
15ce4a0c
CL
76
77 /* I/O parameters */
a8881f5a 78 struct nfs_open_context *ctx; /* file open context info */
f11ac8db 79 struct nfs_lock_context *l_ctx; /* Lock context info */
99514f8f 80 struct kiocb * iocb; /* controlling i/o request */
88467055 81 struct inode * inode; /* target file of i/o */
15ce4a0c
CL
82
83 /* completion state */
607f31e8 84 atomic_t io_count; /* i/os we're waiting for */
15ce4a0c 85 spinlock_t lock; /* protect completion state */
0a00b77b
WAA
86
87 struct nfs_direct_mirror mirrors[NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX];
88 int mirror_count;
89
d9ee6553 90 loff_t io_start; /* Start offset for I/O */
15ce4a0c 91 ssize_t count, /* bytes actually processed */
ed3743a6 92 max_count, /* max expected count */
35754bc0 93 bytes_left, /* bytes left to be sent */
1da177e4 94 error; /* any reported error */
d72b7a6b 95 struct completion completion; /* wait for i/o completion */
fad61490
TM
96
97 /* commit state */
1763da12
FI
98 struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */
99 struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */
100 struct work_struct work;
fad61490 101 int flags;
ad3cba22 102 /* for write */
fad61490
TM
103#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
104#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
ad3cba22
DK
105 /* for read */
106#define NFS_ODIRECT_SHOULD_DIRTY (3) /* dirty user-space page after read */
fad61490 107 struct nfs_writeverf verf; /* unstable write verifier */
1da177e4
LT
108};
109
1763da12
FI
110static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
111static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
4d3b55d3 112static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
1763da12 113static void nfs_direct_write_schedule_work(struct work_struct *work);
607f31e8
TM
114
115static inline void get_dreq(struct nfs_direct_req *dreq)
116{
117 atomic_inc(&dreq->io_count);
118}
119
120static inline int put_dreq(struct nfs_direct_req *dreq)
121{
122 return atomic_dec_and_test(&dreq->io_count);
123}
124
0a00b77b
WAA
125static void
126nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)
127{
128 int i;
129 ssize_t count;
130
ed3743a6
WAA
131 WARN_ON_ONCE(dreq->count >= dreq->max_count);
132
1ccbad9f
PT
133 if (dreq->mirror_count == 1) {
134 dreq->mirrors[hdr->pgio_mirror_idx].count += hdr->good_bytes;
135 dreq->count += hdr->good_bytes;
136 } else {
137 /* mirrored writes */
138 count = dreq->mirrors[hdr->pgio_mirror_idx].count;
139 if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {
140 count = hdr->io_start + hdr->good_bytes - dreq->io_start;
141 dreq->mirrors[hdr->pgio_mirror_idx].count = count;
142 }
143 /* update the dreq->count by finding the minimum agreed count from all
144 * mirrors */
145 count = dreq->mirrors[0].count;
0a00b77b 146
1ccbad9f
PT
147 for (i = 1; i < dreq->mirror_count; i++)
148 count = min(count, dreq->mirrors[i].count);
0a00b77b 149
1ccbad9f 150 dreq->count = count;
5fadeb47 151 }
0a00b77b
WAA
152}
153
5002c586
WAA
154/*
155 * nfs_direct_select_verf - select the right verifier
156 * @dreq - direct request possibly spanning multiple servers
157 * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs
6cccbb6f 158 * @commit_idx - commit bucket index for the DS
5002c586
WAA
159 *
160 * returns the correct verifier to use given the role of the server
161 */
162static struct nfs_writeverf *
163nfs_direct_select_verf(struct nfs_direct_req *dreq,
164 struct nfs_client *ds_clp,
6cccbb6f 165 int commit_idx)
5002c586
WAA
166{
167 struct nfs_writeverf *verfp = &dreq->verf;
168
169#ifdef CONFIG_NFS_V4_1
834e465b
KM
170 /*
171 * pNFS is in use, use the DS verf except commit_through_mds is set
172 * for layout segment where nbuckets is zero.
173 */
174 if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {
6cccbb6f
WAA
175 if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)
176 verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;
5002c586
WAA
177 else
178 WARN_ON_ONCE(1);
179 }
180#endif
181 return verfp;
182}
183
184
185/*
186 * nfs_direct_set_hdr_verf - set the write/commit verifier
187 * @dreq - direct request possibly spanning multiple servers
188 * @hdr - pageio header to validate against previously seen verfs
189 *
190 * Set the server's (MDS or DS) "seen" verifier
191 */
192static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,
193 struct nfs_pgio_header *hdr)
194{
195 struct nfs_writeverf *verfp;
196
6cccbb6f 197 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
5002c586
WAA
198 WARN_ON_ONCE(verfp->committed >= 0);
199 memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
200 WARN_ON_ONCE(verfp->committed < 0);
201}
202
8fc3c386
TM
203static int nfs_direct_cmp_verf(const struct nfs_writeverf *v1,
204 const struct nfs_writeverf *v2)
205{
206 return nfs_write_verifier_cmp(&v1->verifier, &v2->verifier);
207}
208
5002c586
WAA
209/*
210 * nfs_direct_cmp_hdr_verf - compare verifier for pgio header
211 * @dreq - direct request possibly spanning multiple servers
212 * @hdr - pageio header to validate against previously seen verf
213 *
214 * set the server's "seen" verf if not initialized.
215 * returns result of comparison between @hdr->verf and the "seen"
216 * verf of the server used by @hdr (DS or MDS)
217 */
218static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,
219 struct nfs_pgio_header *hdr)
220{
221 struct nfs_writeverf *verfp;
222
6cccbb6f 223 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
5002c586
WAA
224 if (verfp->committed < 0) {
225 nfs_direct_set_hdr_verf(dreq, hdr);
226 return 0;
227 }
8fc3c386 228 return nfs_direct_cmp_verf(verfp, &hdr->verf);
5002c586
WAA
229}
230
5002c586
WAA
231/*
232 * nfs_direct_cmp_commit_data_verf - compare verifier for commit data
233 * @dreq - direct request possibly spanning multiple servers
234 * @data - commit data to validate against previously seen verf
235 *
236 * returns result of comparison between @data->verf and the verf of
237 * the server used by @data (DS or MDS)
238 */
239static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,
240 struct nfs_commit_data *data)
241{
242 struct nfs_writeverf *verfp;
243
244 verfp = nfs_direct_select_verf(dreq, data->ds_clp,
245 data->ds_commit_index);
80c76fe3
WAA
246
247 /* verifier not set so always fail */
248 if (verfp->committed < 0)
249 return 1;
250
8fc3c386 251 return nfs_direct_cmp_verf(verfp, &data->verf);
5002c586 252}
5002c586 253
1da177e4 254/**
b8a32e2b 255 * nfs_direct_IO - NFS address space operation for direct I/O
b8a32e2b 256 * @iocb: target I/O control block
90090ae6 257 * @iter: I/O buffer
b8a32e2b
CL
258 *
259 * The presence of this routine in the address space ops vector means
a564b8f0
MG
260 * the NFS client supports direct I/O. However, for most direct IO, we
261 * shunt off direct read and write requests before the VFS gets them,
262 * so this method is only ever called for swap.
1da177e4 263 */
c8b8e32d 264ssize_t nfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
b8a32e2b 265{
ee8a1a8b
PT
266 struct inode *inode = iocb->ki_filp->f_mapping->host;
267
268 /* we only support swap file calling nfs_direct_IO */
269 if (!IS_SWAPFILE(inode))
270 return 0;
271
66ee59af 272 VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
a564b8f0 273
6f673763 274 if (iov_iter_rw(iter) == READ)
c8b8e32d 275 return nfs_file_direct_read(iocb, iter);
65a4a1ca 276 return nfs_file_direct_write(iocb, iter);
b8a32e2b
CL
277}
278
749e146e 279static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
9c93ab7d 280{
749e146e 281 unsigned int i;
607f31e8 282 for (i = 0; i < npages; i++)
09cbfeaf 283 put_page(pages[i]);
6b45d858
TM
284}
285
1763da12
FI
286void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
287 struct nfs_direct_req *dreq)
288{
fe238e60 289 cinfo->inode = dreq->inode;
1763da12
FI
290 cinfo->mds = &dreq->mds_cinfo;
291 cinfo->ds = &dreq->ds_cinfo;
292 cinfo->dreq = dreq;
293 cinfo->completion_ops = &nfs_direct_commit_completion_ops;
294}
295
0a00b77b
WAA
296static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,
297 struct nfs_pageio_descriptor *pgio,
298 struct nfs_page *req)
299{
300 int mirror_count = 1;
301
302 if (pgio->pg_ops->pg_get_mirror_count)
303 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
304
305 dreq->mirror_count = mirror_count;
306}
307
93619e59 308static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
1da177e4 309{
93619e59
CL
310 struct nfs_direct_req *dreq;
311
292f3eee 312 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
93619e59
CL
313 if (!dreq)
314 return NULL;
315
316 kref_init(&dreq->kref);
607f31e8 317 kref_get(&dreq->kref);
d72b7a6b 318 init_completion(&dreq->completion);
1763da12 319 INIT_LIST_HEAD(&dreq->mds_cinfo.list);
5002c586 320 dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */
1763da12 321 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
0a00b77b 322 dreq->mirror_count = 1;
15ce4a0c 323 spin_lock_init(&dreq->lock);
93619e59
CL
324
325 return dreq;
1da177e4
LT
326}
327
b4946ffb 328static void nfs_direct_req_free(struct kref *kref)
1da177e4
LT
329{
330 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
a8881f5a 331
8c393f9a 332 nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);
f11ac8db
TM
333 if (dreq->l_ctx != NULL)
334 nfs_put_lock_context(dreq->l_ctx);
a8881f5a
TM
335 if (dreq->ctx != NULL)
336 put_nfs_open_context(dreq->ctx);
1da177e4
LT
337 kmem_cache_free(nfs_direct_cachep, dreq);
338}
339
b4946ffb
TM
340static void nfs_direct_req_release(struct nfs_direct_req *dreq)
341{
342 kref_put(&dreq->kref, nfs_direct_req_free);
343}
344
6296556f
PT
345ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
346{
347 return dreq->bytes_left;
348}
349EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
350
bc0fb201
CL
351/*
352 * Collects and returns the final error value/byte-count.
353 */
354static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
355{
15ce4a0c 356 ssize_t result = -EIOCBQUEUED;
bc0fb201
CL
357
358 /* Async requests don't wait here */
359 if (dreq->iocb)
360 goto out;
361
150030b7 362 result = wait_for_completion_killable(&dreq->completion);
bc0fb201 363
d2a7de0b
TM
364 if (!result) {
365 result = dreq->count;
366 WARN_ON_ONCE(dreq->count < 0);
367 }
bc0fb201 368 if (!result)
15ce4a0c 369 result = dreq->error;
bc0fb201
CL
370
371out:
bc0fb201
CL
372 return (ssize_t) result;
373}
374
63ab46ab 375/*
607f31e8
TM
376 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
377 * the iocb is still valid here if this is a synchronous request.
63ab46ab 378 */
f7b5c340 379static void nfs_direct_complete(struct nfs_direct_req *dreq)
63ab46ab 380{
9811cd57
CH
381 struct inode *inode = dreq->inode;
382
fe0f07d0 383 inode_dio_end(inode);
2a009ec9
CH
384
385 if (dreq->iocb) {
15ce4a0c 386 long res = (long) dreq->error;
d2a7de0b 387 if (dreq->count != 0) {
15ce4a0c 388 res = (long) dreq->count;
d2a7de0b
TM
389 WARN_ON_ONCE(dreq->count < 0);
390 }
04b2fa9f 391 dreq->iocb->ki_complete(dreq->iocb, res, 0);
d72b7a6b 392 }
2a009ec9 393
024de8f1 394 complete(&dreq->completion);
63ab46ab 395
b4946ffb 396 nfs_direct_req_release(dreq);
63ab46ab
CL
397}
398
584aa810 399static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
fdd1e74c 400{
584aa810
FI
401 unsigned long bytes = 0;
402 struct nfs_direct_req *dreq = hdr->dreq;
fdd1e74c 403
584aa810
FI
404 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
405 goto out_put;
15ce4a0c
CL
406
407 spin_lock(&dreq->lock);
584aa810
FI
408 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
409 dreq->error = hdr->error;
410 else
0a00b77b
WAA
411 nfs_direct_good_bytes(dreq, hdr);
412
584aa810
FI
413 spin_unlock(&dreq->lock);
414
4bd8b010
TM
415 while (!list_empty(&hdr->pages)) {
416 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
417 struct page *page = req->wb_page;
418
ad3cba22
DK
419 if (!PageCompound(page) && bytes < hdr->good_bytes &&
420 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))
be7e9858 421 set_page_dirty(page);
4bd8b010
TM
422 bytes += req->wb_bytes;
423 nfs_list_remove_request(req);
beeb5338 424 nfs_release_request(req);
d4a8f367 425 }
584aa810 426out_put:
607f31e8 427 if (put_dreq(dreq))
f7b5c340 428 nfs_direct_complete(dreq);
584aa810 429 hdr->release(hdr);
1da177e4
LT
430}
431
df3accb8 432static void nfs_read_sync_pgio_error(struct list_head *head, int error)
cd841605 433{
584aa810 434 struct nfs_page *req;
cd841605 435
584aa810
FI
436 while (!list_empty(head)) {
437 req = nfs_list_entry(head->next);
438 nfs_list_remove_request(req);
439 nfs_release_request(req);
440 }
cd841605
FI
441}
442
584aa810
FI
443static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
444{
445 get_dreq(hdr->dreq);
446}
447
448static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
3e9e0ca3 449 .error_cleanup = nfs_read_sync_pgio_error,
584aa810
FI
450 .init_hdr = nfs_direct_pgio_init,
451 .completion = nfs_direct_read_completion,
452};
453
d4cc948b 454/*
607f31e8
TM
455 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
456 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
457 * bail and stop sending more reads. Read length accounting is
458 * handled automatically by nfs_direct_read_result(). Otherwise, if
459 * no requests have been sent, just return an error.
1da177e4 460 */
1da177e4 461
91f79c43
AV
462static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
463 struct iov_iter *iter,
464 loff_t pos)
465{
466 struct nfs_pageio_descriptor desc;
467 struct inode *inode = dreq->inode;
468 ssize_t result = -EINVAL;
469 size_t requested_bytes = 0;
470 size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
e9f7bee1 471
16b90578 472 nfs_pageio_init_read(&desc, dreq->inode, false,
91f79c43
AV
473 &nfs_direct_read_completion_ops);
474 get_dreq(dreq);
475 desc.pg_dreq = dreq;
fe0f07d0 476 inode_dio_begin(inode);
a564b8f0 477
91f79c43
AV
478 while (iov_iter_count(iter)) {
479 struct page **pagevec;
480 size_t bytes;
481 size_t pgbase;
482 unsigned npages, i;
607f31e8 483
91f79c43
AV
484 result = iov_iter_get_pages_alloc(iter, &pagevec,
485 rsize, &pgbase);
486 if (result < 0)
487 break;
488
489 bytes = result;
490 iov_iter_advance(iter, bytes);
491 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
584aa810
FI
492 for (i = 0; i < npages; i++) {
493 struct nfs_page *req;
bf5fc402 494 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
584aa810 495 /* XXX do we need to do the eof zeroing found in async_filler? */
28b1d3f5 496 req = nfs_create_request(dreq->ctx, pagevec[i],
584aa810
FI
497 pgbase, req_len);
498 if (IS_ERR(req)) {
584aa810
FI
499 result = PTR_ERR(req);
500 break;
501 }
502 req->wb_index = pos >> PAGE_SHIFT;
503 req->wb_offset = pos & ~PAGE_MASK;
91f79c43
AV
504 if (!nfs_pageio_add_request(&desc, req)) {
505 result = desc.pg_error;
584aa810 506 nfs_release_request(req);
584aa810
FI
507 break;
508 }
509 pgbase = 0;
510 bytes -= req_len;
91f79c43 511 requested_bytes += req_len;
584aa810 512 pos += req_len;
35754bc0 513 dreq->bytes_left -= req_len;
584aa810 514 }
6d74743b 515 nfs_direct_release_pages(pagevec, npages);
91f79c43 516 kvfree(pagevec);
19f73787
CL
517 if (result < 0)
518 break;
19f73787
CL
519 }
520
584aa810
FI
521 nfs_pageio_complete(&desc);
522
839f7ad6
CL
523 /*
524 * If no bytes were started, return the error, and let the
525 * generic layer handle the completion.
526 */
527 if (requested_bytes == 0) {
fe0f07d0 528 inode_dio_end(inode);
839f7ad6
CL
529 nfs_direct_req_release(dreq);
530 return result < 0 ? result : -EIO;
531 }
532
19f73787 533 if (put_dreq(dreq))
f7b5c340 534 nfs_direct_complete(dreq);
85128b2b 535 return requested_bytes;
19f73787
CL
536}
537
14a3ec79
CH
538/**
539 * nfs_file_direct_read - file direct read operation for NFS files
540 * @iocb: target I/O control block
619d30b4 541 * @iter: vector of user buffers into which to read data
14a3ec79
CH
542 *
543 * We use this function for direct reads instead of calling
544 * generic_file_aio_read() in order to avoid gfar's check to see if
545 * the request starts before the end of the file. For that check
546 * to work, we must generate a GETATTR before each direct read, and
547 * even then there is a window between the GETATTR and the subsequent
548 * READ where the file size could change. Our preference is simply
549 * to do all reads the application wants, and the server will take
550 * care of managing the end of file boundary.
551 *
552 * This function also eliminates unnecessarily updating the file's
553 * atime locally, as the NFS server sets the file's atime, and this
554 * client must read the updated atime from the server back into its
555 * cache.
556 */
c8b8e32d 557ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
1da177e4 558{
14a3ec79
CH
559 struct file *file = iocb->ki_filp;
560 struct address_space *mapping = file->f_mapping;
561 struct inode *inode = mapping->host;
1da177e4 562 struct nfs_direct_req *dreq;
b3c54de6 563 struct nfs_lock_context *l_ctx;
85128b2b 564 ssize_t result = -EINVAL, requested;
a6cbcd4a 565 size_t count = iov_iter_count(iter);
14a3ec79
CH
566 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
567
568 dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
c8b8e32d 569 file, count, (long long) iocb->ki_pos);
14a3ec79
CH
570
571 result = 0;
572 if (!count)
573 goto out;
574
14a3ec79
CH
575 task_io_account_read(count);
576
577 result = -ENOMEM;
607f31e8 578 dreq = nfs_direct_req_alloc();
f11ac8db 579 if (dreq == NULL)
a5864c99 580 goto out;
1da177e4 581
91d5b470 582 dreq->inode = inode;
ed3743a6 583 dreq->bytes_left = dreq->max_count = count;
c8b8e32d 584 dreq->io_start = iocb->ki_pos;
cd3758e3 585 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
b3c54de6
TM
586 l_ctx = nfs_get_lock_context(dreq->ctx);
587 if (IS_ERR(l_ctx)) {
588 result = PTR_ERR(l_ctx);
f11ac8db 589 goto out_release;
b3c54de6
TM
590 }
591 dreq->l_ctx = l_ctx;
487b8372
CL
592 if (!is_sync_kiocb(iocb))
593 dreq->iocb = iocb;
1da177e4 594
ad3cba22
DK
595 if (iter_is_iovec(iter))
596 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
597
a5864c99
TM
598 nfs_start_io_direct(inode);
599
619d30b4 600 NFS_I(inode)->read_io += count;
85128b2b 601 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
d0b9875d 602
a5864c99 603 nfs_end_io_direct(inode);
d0b9875d 604
85128b2b 605 if (requested > 0) {
607f31e8 606 result = nfs_direct_wait(dreq);
85128b2b
AV
607 if (result > 0) {
608 requested -= result;
c8b8e32d 609 iocb->ki_pos += result;
85128b2b
AV
610 }
611 iov_iter_revert(iter, requested);
612 } else {
613 result = requested;
14a3ec79 614 }
d0b9875d 615
f11ac8db 616out_release:
b4946ffb 617 nfs_direct_req_release(dreq);
f11ac8db 618out:
1da177e4
LT
619 return result;
620}
621
085d1e33
TH
622static void
623nfs_direct_write_scan_commit_list(struct inode *inode,
624 struct list_head *list,
625 struct nfs_commit_info *cinfo)
626{
e824f99a 627 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
085d1e33
TH
628#ifdef CONFIG_NFS_V4_1
629 if (cinfo->ds != NULL && cinfo->ds->nwritten != 0)
630 NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
631#endif
632 nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
e824f99a 633 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
085d1e33
TH
634}
635
fad61490
TM
636static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
637{
1763da12
FI
638 struct nfs_pageio_descriptor desc;
639 struct nfs_page *req, *tmp;
640 LIST_HEAD(reqs);
641 struct nfs_commit_info cinfo;
642 LIST_HEAD(failed);
0a00b77b 643 int i;
1763da12
FI
644
645 nfs_init_cinfo_from_dreq(&cinfo, dreq);
085d1e33 646 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
1da177e4 647
fad61490 648 dreq->count = 0;
a5314a74
TM
649 dreq->verf.committed = NFS_INVALID_STABLE_HOW;
650 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
0a00b77b
WAA
651 for (i = 0; i < dreq->mirror_count; i++)
652 dreq->mirrors[i].count = 0;
607f31e8
TM
653 get_dreq(dreq);
654
a20c93e3 655 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
1763da12
FI
656 &nfs_direct_write_completion_ops);
657 desc.pg_dreq = dreq;
fedb595c 658
0a00b77b
WAA
659 req = nfs_list_entry(reqs.next);
660 nfs_direct_setup_mirroring(dreq, &desc, req);
d600ad1f
PT
661 if (desc.pg_error < 0) {
662 list_splice_init(&reqs, &failed);
663 goto out_failed;
664 }
0a00b77b 665
1763da12 666 list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
33344e0f
TM
667 /* Bump the transmission count */
668 req->wb_nio++;
1763da12 669 if (!nfs_pageio_add_request(&desc, req)) {
078b5fd9 670 nfs_list_move_request(req, &failed);
fe238e60 671 spin_lock(&cinfo.inode->i_lock);
1763da12 672 dreq->flags = 0;
d600ad1f
PT
673 if (desc.pg_error < 0)
674 dreq->error = desc.pg_error;
675 else
676 dreq->error = -EIO;
fe238e60 677 spin_unlock(&cinfo.inode->i_lock);
1763da12 678 }
5a695da2 679 nfs_release_request(req);
1763da12
FI
680 }
681 nfs_pageio_complete(&desc);
fad61490 682
d600ad1f 683out_failed:
4035c248
TM
684 while (!list_empty(&failed)) {
685 req = nfs_list_entry(failed.next);
686 nfs_list_remove_request(req);
1d1afcbc 687 nfs_unlock_and_release_request(req);
4035c248 688 }
fad61490 689
1763da12 690 if (put_dreq(dreq))
4d3b55d3 691 nfs_direct_write_complete(dreq);
c9d8f89d
TM
692}
693
1763da12 694static void nfs_direct_commit_complete(struct nfs_commit_data *data)
c9d8f89d 695{
0b7c0153 696 struct nfs_direct_req *dreq = data->dreq;
1763da12
FI
697 struct nfs_commit_info cinfo;
698 struct nfs_page *req;
c9d8f89d
TM
699 int status = data->task.tk_status;
700
1763da12 701 nfs_init_cinfo_from_dreq(&cinfo, dreq);
fe4f844d 702 if (status < 0 || nfs_direct_cmp_commit_data_verf(dreq, data))
fad61490 703 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
1da177e4 704
1763da12
FI
705 while (!list_empty(&data->pages)) {
706 req = nfs_list_entry(data->pages.next);
707 nfs_list_remove_request(req);
708 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
33344e0f
TM
709 /*
710 * Despite the reboot, the write was successful,
711 * so reset wb_nio.
712 */
713 req->wb_nio = 0;
1763da12 714 /* Note the rewrite will go through mds */
b57ff130 715 nfs_mark_request_commit(req, NULL, &cinfo, 0);
906369e4
FI
716 } else
717 nfs_release_request(req);
1d1afcbc 718 nfs_unlock_and_release_request(req);
1763da12
FI
719 }
720
721 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
4d3b55d3 722 nfs_direct_write_complete(dreq);
1da177e4
LT
723}
724
b20135d0
TM
725static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
726 struct nfs_page *req)
1763da12 727{
b20135d0
TM
728 struct nfs_direct_req *dreq = cinfo->dreq;
729
730 spin_lock(&dreq->lock);
731 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
732 spin_unlock(&dreq->lock);
733 nfs_mark_request_commit(req, NULL, cinfo, 0);
1763da12
FI
734}
735
736static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
737 .completion = nfs_direct_commit_complete,
b20135d0 738 .resched_write = nfs_direct_resched_write,
fad61490
TM
739};
740
741static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
1da177e4 742{
1763da12
FI
743 int res;
744 struct nfs_commit_info cinfo;
745 LIST_HEAD(mds_list);
746
747 nfs_init_cinfo_from_dreq(&cinfo, dreq);
748 nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
749 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
750 if (res < 0) /* res == -ENOMEM */
751 nfs_direct_write_reschedule(dreq);
fad61490 752}
1da177e4 753
1763da12 754static void nfs_direct_write_schedule_work(struct work_struct *work)
fad61490 755{
1763da12 756 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
fad61490 757 int flags = dreq->flags;
1da177e4 758
fad61490
TM
759 dreq->flags = 0;
760 switch (flags) {
761 case NFS_ODIRECT_DO_COMMIT:
762 nfs_direct_commit_schedule(dreq);
1da177e4 763 break;
fad61490
TM
764 case NFS_ODIRECT_RESCHED_WRITES:
765 nfs_direct_write_reschedule(dreq);
766 break;
767 default:
f7b5c340
TM
768 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
769 nfs_direct_complete(dreq);
fad61490
TM
770 }
771}
1da177e4 772
4d3b55d3 773static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
fad61490 774{
46483c2e 775 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */
fad61490 776}
1763da12 777
1763da12
FI
778static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
779{
780 struct nfs_direct_req *dreq = hdr->dreq;
781 struct nfs_commit_info cinfo;
c65e6254 782 bool request_commit = false;
1763da12
FI
783 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
784
785 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
786 goto out_put;
787
788 nfs_init_cinfo_from_dreq(&cinfo, dreq);
789
790 spin_lock(&dreq->lock);
791
1b8d97b0 792 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags))
1763da12 793 dreq->error = hdr->error;
c65e6254 794 if (dreq->error == 0) {
0a00b77b 795 nfs_direct_good_bytes(dreq, hdr);
c65e6254 796 if (nfs_write_need_commit(hdr)) {
1763da12 797 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
c65e6254 798 request_commit = true;
1763da12 799 else if (dreq->flags == 0) {
5002c586 800 nfs_direct_set_hdr_verf(dreq, hdr);
c65e6254 801 request_commit = true;
1763da12
FI
802 dreq->flags = NFS_ODIRECT_DO_COMMIT;
803 } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
c65e6254
WAA
804 request_commit = true;
805 if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))
5002c586
WAA
806 dreq->flags =
807 NFS_ODIRECT_RESCHED_WRITES;
1763da12
FI
808 }
809 }
810 }
811 spin_unlock(&dreq->lock);
812
813 while (!list_empty(&hdr->pages)) {
2bfc6e56 814
1763da12
FI
815 req = nfs_list_entry(hdr->pages.next);
816 nfs_list_remove_request(req);
c65e6254 817 if (request_commit) {
04277086 818 kref_get(&req->wb_kref);
b57ff130
WAA
819 nfs_mark_request_commit(req, hdr->lseg, &cinfo,
820 hdr->ds_commit_idx);
1763da12 821 }
1d1afcbc 822 nfs_unlock_and_release_request(req);
1763da12
FI
823 }
824
825out_put:
826 if (put_dreq(dreq))
4d3b55d3 827 nfs_direct_write_complete(dreq);
1763da12
FI
828 hdr->release(hdr);
829}
830
df3accb8 831static void nfs_write_sync_pgio_error(struct list_head *head, int error)
3e9e0ca3
TM
832{
833 struct nfs_page *req;
834
835 while (!list_empty(head)) {
836 req = nfs_list_entry(head->next);
837 nfs_list_remove_request(req);
1d1afcbc 838 nfs_unlock_and_release_request(req);
3e9e0ca3
TM
839 }
840}
841
dc602dd7
TM
842static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
843{
844 struct nfs_direct_req *dreq = hdr->dreq;
845
846 spin_lock(&dreq->lock);
847 if (dreq->error == 0) {
848 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
849 /* fake unstable write to let common nfs resend pages */
850 hdr->verf.committed = NFS_UNSTABLE;
851 hdr->good_bytes = hdr->args.count;
852 }
853 spin_unlock(&dreq->lock);
854}
855
1763da12 856static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
3e9e0ca3 857 .error_cleanup = nfs_write_sync_pgio_error,
1763da12
FI
858 .init_hdr = nfs_direct_pgio_init,
859 .completion = nfs_direct_write_completion,
dc602dd7 860 .reschedule_io = nfs_direct_write_reschedule_io,
1763da12
FI
861};
862
91f79c43
AV
863
864/*
865 * NB: Return the value of the first error return code. Subsequent
866 * errors after the first one are ignored.
867 */
868/*
869 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
870 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
871 * bail and stop sending more writes. Write length accounting is
872 * handled automatically by nfs_direct_write_result(). Otherwise, if
873 * no requests have been sent, just return an error.
874 */
19f73787 875static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
619d30b4 876 struct iov_iter *iter,
91f79c43 877 loff_t pos)
19f73787 878{
1763da12 879 struct nfs_pageio_descriptor desc;
1d59d61f 880 struct inode *inode = dreq->inode;
19f73787
CL
881 ssize_t result = 0;
882 size_t requested_bytes = 0;
91f79c43 883 size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
19f73787 884
a20c93e3 885 nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
1763da12
FI
886 &nfs_direct_write_completion_ops);
887 desc.pg_dreq = dreq;
19f73787 888 get_dreq(dreq);
fe0f07d0 889 inode_dio_begin(inode);
19f73787 890
91f79c43
AV
891 NFS_I(inode)->write_io += iov_iter_count(iter);
892 while (iov_iter_count(iter)) {
893 struct page **pagevec;
894 size_t bytes;
895 size_t pgbase;
896 unsigned npages, i;
897
898 result = iov_iter_get_pages_alloc(iter, &pagevec,
899 wsize, &pgbase);
19f73787
CL
900 if (result < 0)
901 break;
91f79c43
AV
902
903 bytes = result;
904 iov_iter_advance(iter, bytes);
905 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
906 for (i = 0; i < npages; i++) {
907 struct nfs_page *req;
908 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
909
28b1d3f5 910 req = nfs_create_request(dreq->ctx, pagevec[i],
91f79c43
AV
911 pgbase, req_len);
912 if (IS_ERR(req)) {
913 result = PTR_ERR(req);
914 break;
915 }
0a00b77b
WAA
916
917 nfs_direct_setup_mirroring(dreq, &desc, req);
d600ad1f
PT
918 if (desc.pg_error < 0) {
919 nfs_free_request(req);
920 result = desc.pg_error;
921 break;
922 }
0a00b77b 923
91f79c43
AV
924 nfs_lock_request(req);
925 req->wb_index = pos >> PAGE_SHIFT;
926 req->wb_offset = pos & ~PAGE_MASK;
927 if (!nfs_pageio_add_request(&desc, req)) {
928 result = desc.pg_error;
929 nfs_unlock_and_release_request(req);
930 break;
931 }
932 pgbase = 0;
933 bytes -= req_len;
934 requested_bytes += req_len;
935 pos += req_len;
936 dreq->bytes_left -= req_len;
937 }
938 nfs_direct_release_pages(pagevec, npages);
939 kvfree(pagevec);
940 if (result < 0)
19f73787 941 break;
19f73787 942 }
1763da12 943 nfs_pageio_complete(&desc);
19f73787 944
839f7ad6
CL
945 /*
946 * If no bytes were started, return the error, and let the
947 * generic layer handle the completion.
948 */
949 if (requested_bytes == 0) {
fe0f07d0 950 inode_dio_end(inode);
839f7ad6
CL
951 nfs_direct_req_release(dreq);
952 return result < 0 ? result : -EIO;
953 }
954
19f73787 955 if (put_dreq(dreq))
4d3b55d3 956 nfs_direct_write_complete(dreq);
85128b2b 957 return requested_bytes;
19f73787
CL
958}
959
1da177e4
LT
960/**
961 * nfs_file_direct_write - file direct write operation for NFS files
962 * @iocb: target I/O control block
619d30b4 963 * @iter: vector of user buffers from which to write data
1da177e4
LT
964 *
965 * We use this function for direct writes instead of calling
966 * generic_file_aio_write() in order to avoid taking the inode
967 * semaphore and updating the i_size. The NFS server will set
968 * the new i_size and this client must read the updated size
969 * back into its cache. We let the server do generic write
970 * parameter checking and report problems.
971 *
1da177e4
LT
972 * We eliminate local atime updates, see direct read above.
973 *
974 * We avoid unnecessary page cache invalidations for normal cached
975 * readers of this file.
976 *
977 * Note that O_APPEND is not supported for NFS direct writes, as there
978 * is no atomic O_APPEND write facility in the NFS protocol.
979 */
65a4a1ca 980ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
1da177e4 981{
85128b2b 982 ssize_t result = -EINVAL, requested;
89698b24 983 size_t count;
1da177e4 984 struct file *file = iocb->ki_filp;
1da177e4 985 struct address_space *mapping = file->f_mapping;
22cd1bf1
CH
986 struct inode *inode = mapping->host;
987 struct nfs_direct_req *dreq;
988 struct nfs_lock_context *l_ctx;
65a4a1ca 989 loff_t pos, end;
c216fd70 990
6de1472f 991 dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
3309dd04 992 file, iov_iter_count(iter), (long long) iocb->ki_pos);
027445c3 993
89698b24
TM
994 result = generic_write_checks(iocb, iter);
995 if (result <= 0)
996 return result;
997 count = result;
998 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
3309dd04
AV
999
1000 pos = iocb->ki_pos;
09cbfeaf 1001 end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
ce1a8e67 1002
89698b24 1003 task_io_account_write(count);
7ec10f26 1004
22cd1bf1
CH
1005 result = -ENOMEM;
1006 dreq = nfs_direct_req_alloc();
1007 if (!dreq)
a5864c99 1008 goto out;
9eafa8cc 1009
22cd1bf1 1010 dreq->inode = inode;
89698b24 1011 dreq->bytes_left = dreq->max_count = count;
5fadeb47 1012 dreq->io_start = pos;
22cd1bf1
CH
1013 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
1014 l_ctx = nfs_get_lock_context(dreq->ctx);
1015 if (IS_ERR(l_ctx)) {
1016 result = PTR_ERR(l_ctx);
1017 goto out_release;
1018 }
1019 dreq->l_ctx = l_ctx;
1020 if (!is_sync_kiocb(iocb))
1021 dreq->iocb = iocb;
1022
a5864c99
TM
1023 nfs_start_io_direct(inode);
1024
85128b2b 1025 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos);
a9ab5e84
CH
1026
1027 if (mapping->nrpages) {
1028 invalidate_inode_pages2_range(mapping,
09cbfeaf 1029 pos >> PAGE_SHIFT, end);
a9ab5e84
CH
1030 }
1031
a5864c99 1032 nfs_end_io_direct(inode);
a9ab5e84 1033
85128b2b 1034 if (requested > 0) {
22cd1bf1
CH
1035 result = nfs_direct_wait(dreq);
1036 if (result > 0) {
85128b2b 1037 requested -= result;
22cd1bf1 1038 iocb->ki_pos = pos + result;
e2592217
CH
1039 /* XXX: should check the generic_write_sync retval */
1040 generic_write_sync(iocb, result);
22cd1bf1 1041 }
85128b2b
AV
1042 iov_iter_revert(iter, requested);
1043 } else {
1044 result = requested;
1763da12 1045 }
22cd1bf1
CH
1046out_release:
1047 nfs_direct_req_release(dreq);
a5864c99 1048out:
22cd1bf1 1049 return result;
1da177e4
LT
1050}
1051
88467055
CL
1052/**
1053 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1054 *
1055 */
f7b422b1 1056int __init nfs_init_directcache(void)
1da177e4
LT
1057{
1058 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1059 sizeof(struct nfs_direct_req),
fffb60f9
PJ
1060 0, (SLAB_RECLAIM_ACCOUNT|
1061 SLAB_MEM_SPREAD),
20c2df83 1062 NULL);
1da177e4
LT
1063 if (nfs_direct_cachep == NULL)
1064 return -ENOMEM;
1065
1066 return 0;
1067}
1068
88467055 1069/**
f7b422b1 1070 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
88467055
CL
1071 *
1072 */
266bee88 1073void nfs_destroy_directcache(void)
1da177e4 1074{
1a1d92c1 1075 kmem_cache_destroy(nfs_direct_cachep);
1da177e4 1076}