projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
media: staging: media: use relevant lock
[linux-2.6-block.git] / fs / f2fs / data.c
CommitLineData
0a8165d7 1/*
eb47b800
JK		 2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
8f46dcae 17#include <linux/pagevec.h>
eb47b800
JK		 18#include <linux/blkdev.h>
19#include <linux/bio.h>
690e4a3e 20#include <linux/prefetch.h>
e2e40f2c 21#include <linux/uio.h>
fe76b796
JK		 22#include <linux/mm.h>
23#include <linux/memcontrol.h>
f1e88660 24#include <linux/cleancache.h>
174cd4b1 25#include <linux/sched/signal.h>
eb47b800
JK		 26
27#include "f2fs.h"
28#include "node.h"
29#include "segment.h"
db9f7c1a 30#include "trace.h"
848753aa 31#include <trace/events/f2fs.h>
eb47b800 32
36951b38
CY		 33static bool __is_cp_guaranteed(struct page *page)
34{
35 struct address_space *mapping = page->mapping;
36 struct inode *inode;
37 struct f2fs_sb_info *sbi;
38
39 if (!mapping)
40 return false;
41
42 inode = mapping->host;
43 sbi = F2FS_I_SB(inode);
44
45 if (inode->i_ino == F2FS_META_INO(sbi) ||
46 inode->i_ino == F2FS_NODE_INO(sbi) ||
47 S_ISDIR(inode->i_mode) ||
48 is_cold_data(page))
49 return true;
50 return false;
51}
52
4246a0b6 53static void f2fs_read_end_io(struct bio *bio)
93dfe2ac 54{
f568849e
LT		 55 struct bio_vec *bvec;
56 int i;
93dfe2ac 57
8b038c70 58#ifdef CONFIG_F2FS_FAULT_INJECTION
263663cd 59 if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)), FAULT_IO)) {
55523519 60 f2fs_show_injection_info(FAULT_IO);
4e4cbee9 61 bio->bi_status = BLK_STS_IOERR;
55523519 62 }
8b038c70
CY		 63#endif
64
4375a336 65 if (f2fs_bio_encrypted(bio)) {
4e4cbee9 66 if (bio->bi_status) {
0b81d077 67 fscrypt_release_ctx(bio->bi_private);
4375a336 68 } else {
0b81d077 69 fscrypt_decrypt_bio_pages(bio->bi_private, bio);
4375a336
JK		 70 return;
71 }
72 }
73
12377024
CY		 74 bio_for_each_segment_all(bvec, bio, i) {
75 struct page *page = bvec->bv_page;
f1e88660 76
4e4cbee9 77 if (!bio->bi_status) {
237c0790
JK		 78 if (!PageUptodate(page))
79 SetPageUptodate(page);
f1e88660
JK		 80 } else {
81 ClearPageUptodate(page);
82 SetPageError(page);
83 }
84 unlock_page(page);
85 }
f1e88660
JK		 86 bio_put(bio);
87}
88
4246a0b6 89static void f2fs_write_end_io(struct bio *bio)
93dfe2ac 90{
1b1f559f 91 struct f2fs_sb_info *sbi = bio->bi_private;
f568849e
LT		 92 struct bio_vec *bvec;
93 int i;
93dfe2ac 94
f568849e 95 bio_for_each_segment_all(bvec, bio, i) {
93dfe2ac 96 struct page *page = bvec->bv_page;
36951b38 97 enum count_type type = WB_DATA_TYPE(page);
93dfe2ac 98
0a595eba
JK		 99 if (IS_DUMMY_WRITTEN_PAGE(page)) {
100 set_page_private(page, (unsigned long)NULL);
101 ClearPagePrivate(page);
102 unlock_page(page);
103 mempool_free(page, sbi->write_io_dummy);
104
4e4cbee9 105 if (unlikely(bio->bi_status))
0a595eba
JK		 106 f2fs_stop_checkpoint(sbi, true);
107 continue;
108 }
109
0b81d077 110 fscrypt_pullback_bio_page(&page, true);
4375a336 111
4e4cbee9 112 if (unlikely(bio->bi_status)) {
5114a97a 113 mapping_set_error(page->mapping, -EIO);
b1ca321d
JK		 114 if (type == F2FS_WB_CP_DATA)
115 f2fs_stop_checkpoint(sbi, true);
93dfe2ac 116 }
7dff55d2
YH		 117
118 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
119 page->index != nid_of_node(page));
120
36951b38
CY		 121 dec_page_count(sbi, type);
122 clear_cold_data(page);
93dfe2ac 123 end_page_writeback(page);
f568849e 124 }
36951b38 125 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
f5730184 126 wq_has_sleeper(&sbi->cp_wait))
93dfe2ac
JK		 127 wake_up(&sbi->cp_wait);
128
129 bio_put(bio);
130}
131
3c62be17
JK		 132/*
133 * Return true, if pre_bio's bdev is same as its target device.
134 */
135struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
136 block_t blk_addr, struct bio *bio)
137{
138 struct block_device *bdev = sbi->sb->s_bdev;
139 int i;
140
141 for (i = 0; i < sbi->s_ndevs; i++) {
142 if (FDEV(i).start_blk <= blk_addr &&
143 FDEV(i).end_blk >= blk_addr) {
144 blk_addr -= FDEV(i).start_blk;
145 bdev = FDEV(i).bdev;
146 break;
147 }
148 }
149 if (bio) {
74d46992 150 bio_set_dev(bio, bdev);
3c62be17
JK		 151 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
152 }
153 return bdev;
154}
155
156int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
157{
158 int i;
159
160 for (i = 0; i < sbi->s_ndevs; i++)
161 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
162 return i;
163 return 0;
164}
165
166static bool __same_bdev(struct f2fs_sb_info *sbi,
167 block_t blk_addr, struct bio *bio)
168{
74d46992
CH		 169 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
170 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
3c62be17
JK		 171}
172
940a6d34
GZ		 173/*
174 * Low-level block read/write IO operations.
175 */
176static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
578c6478 177 struct writeback_control *wbc,
0cdd3195
HL		 178 int npages, bool is_read,
179 enum page_type type, enum temp_type temp)
940a6d34
GZ		 180{
181 struct bio *bio;
182
d62fe971 183 bio = f2fs_bio_alloc(sbi, npages, true);
940a6d34 184
3c62be17 185 f2fs_target_device(sbi, blk_addr, bio);
0cdd3195
HL		 186 if (is_read) {
187 bio->bi_end_io = f2fs_read_end_io;
188 bio->bi_private = NULL;
189 } else {
190 bio->bi_end_io = f2fs_write_end_io;
191 bio->bi_private = sbi;
192 bio->bi_write_hint = io_type_to_rw_hint(sbi, type, temp);
193 }
578c6478
YY		 194 if (wbc)
195 wbc_init_bio(wbc, bio);
940a6d34
GZ		 196
197 return bio;
198}
199
4fc29c1a
LT		 200static inline void __submit_bio(struct f2fs_sb_info *sbi,
201 struct bio *bio, enum page_type type)
f5730184 202{
4fc29c1a 203 if (!is_read_io(bio_op(bio))) {
0a595eba
JK		 204 unsigned int start;
205
0a595eba
JK		 206 if (type != DATA && type != NODE)
207 goto submit_io;
208
ccd31cb2 209 if (f2fs_sb_has_blkzoned(sbi->sb) && current->plug)
3bb09a0e
TY		 210 blk_finish_plug(current->plug);
211
0a595eba
JK		 212 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
213 start %= F2FS_IO_SIZE(sbi);
214
215 if (start == 0)
216 goto submit_io;
217
218 /* fill dummy pages */
219 for (; start < F2FS_IO_SIZE(sbi); start++) {
220 struct page *page =
221 mempool_alloc(sbi->write_io_dummy,
222 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
223 f2fs_bug_on(sbi, !page);
224
225 SetPagePrivate(page);
226 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
227 lock_page(page);
228 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
229 f2fs_bug_on(sbi, 1);
230 }
231 /*
232 * In the NODE case, we lose next block address chain. So, we
233 * need to do checkpoint in f2fs_sync_file.
234 */
235 if (type == NODE)
236 set_sbi_flag(sbi, SBI_NEED_CP);
19a5f5e2 237 }
0a595eba 238submit_io:
554b5125
JK		 239 if (is_read_io(bio_op(bio)))
240 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
241 else
242 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
4e49ea4a 243 submit_bio(bio);
f5730184
JK		 244}
245
458e6197 246static void __submit_merged_bio(struct f2fs_bio_info *io)
93dfe2ac 247{
458e6197 248 struct f2fs_io_info *fio = &io->fio;
93dfe2ac
JK		 249
250 if (!io->bio)
251 return;
252
554b5125
JK		 253 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
254
04d328de 255 if (is_read_io(fio->op))
554b5125 256 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
6a8f8ca5 257 else
554b5125 258 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
04d328de 259
4fc29c1a 260 __submit_bio(io->sbi, io->bio, fio->type);
93dfe2ac
JK		 261 io->bio = NULL;
262}
263
942fd319
JK		 264static bool __has_merged_page(struct f2fs_bio_info *io,
265 struct inode *inode, nid_t ino, pgoff_t idx)
0fd785eb 266{
0fd785eb
CY		 267 struct bio_vec *bvec;
268 struct page *target;
269 int i;
270
0c3a5797 271 if (!io->bio)
0fd785eb 272 return false;
0c3a5797 273
942fd319 274 if (!inode && !ino)
0c3a5797 275 return true;
0fd785eb
CY		 276
277 bio_for_each_segment_all(bvec, io->bio, i) {
278
0b81d077 279 if (bvec->bv_page->mapping)
0fd785eb 280 target = bvec->bv_page;
0b81d077
JK		 281 else
282 target = fscrypt_control_page(bvec->bv_page);
0fd785eb 283
942fd319
JK		 284 if (idx != target->index)
285 continue;
286
0c3a5797
CY		 287 if (inode && inode == target->mapping->host)
288 return true;
0c3a5797 289 if (ino && ino == ino_of_node(target))
0fd785eb 290 return true;
0fd785eb
CY		 291 }
292
0fd785eb
CY		 293 return false;
294}
295
0c3a5797 296static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
942fd319 297 nid_t ino, pgoff_t idx, enum page_type type)
0c3a5797
CY		 298{
299 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d
JK		 300 enum temp_type temp;
301 struct f2fs_bio_info *io;
302 bool ret = false;
0c3a5797 303
a912b54d
JK		 304 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
305 io = sbi->write_io[btype] + temp;
306
307 down_read(&io->io_rwsem);
308 ret = __has_merged_page(io, inode, ino, idx);
309 up_read(&io->io_rwsem);
0c3a5797 310
a912b54d
JK		 311 /* TODO: use HOT temp only for meta pages now. */
312 if (ret || btype == META)
313 break;
314 }
0c3a5797
CY		 315 return ret;
316}
317
b9109b0e 318static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
a912b54d 319 enum page_type type, enum temp_type temp)
93dfe2ac
JK		 320{
321 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d 322 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
93dfe2ac 323
df0f8dc0 324 down_write(&io->io_rwsem);
458e6197
JK		 325
326 /* change META to META_FLUSH in the checkpoint procedure */
327 if (type >= META_FLUSH) {
328 io->fio.type = META_FLUSH;
04d328de 329 io->fio.op = REQ_OP_WRITE;
3adc5fcb 330 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
70fd7614 331 if (!test_opt(sbi, NOBARRIER))
7f54f51f 332 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
458e6197
JK		 333 }
334 __submit_merged_bio(io);
df0f8dc0 335 up_write(&io->io_rwsem);
93dfe2ac
JK		 336}
337
a912b54d
JK		 338static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
339 struct inode *inode, nid_t ino, pgoff_t idx,
340 enum page_type type, bool force)
0c3a5797 341{
a912b54d
JK		 342 enum temp_type temp;
343
344 if (!force && !has_merged_page(sbi, inode, ino, idx, type))
345 return;
346
347 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
348
349 __f2fs_submit_merged_write(sbi, type, temp);
350
351 /* TODO: use HOT temp only for meta pages now. */
352 if (type >= META)
353 break;
354 }
0c3a5797
CY		 355}
356
b9109b0e 357void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
0c3a5797 358{
a912b54d 359 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
0c3a5797
CY		 360}
361
b9109b0e 362void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
942fd319 363 struct inode *inode, nid_t ino, pgoff_t idx,
b9109b0e 364 enum page_type type)
0c3a5797 365{
a912b54d 366 __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
0c3a5797
CY		 367}
368
b9109b0e 369void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
406657dd 370{
b9109b0e
JK		 371 f2fs_submit_merged_write(sbi, DATA);
372 f2fs_submit_merged_write(sbi, NODE);
373 f2fs_submit_merged_write(sbi, META);
406657dd
CY		 374}
375
93dfe2ac
JK		 376/*
377 * Fill the locked page with data located in the block address.
771a9a71 378 * A caller needs to unlock the page on failure.
93dfe2ac 379 */
05ca3632 380int f2fs_submit_page_bio(struct f2fs_io_info *fio)
93dfe2ac 381{
93dfe2ac 382 struct bio *bio;
0b81d077
JK		 383 struct page *page = fio->encrypted_page ?
384 fio->encrypted_page : fio->page;
93dfe2ac 385
0833721e 386 verify_block_addr(fio, fio->new_blkaddr);
2ace38e0 387 trace_f2fs_submit_page_bio(page, fio);
05ca3632 388 f2fs_trace_ios(fio, 0);
93dfe2ac
JK		 389
390 /* Allocate a new bio */
578c6478 391 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
0cdd3195 392 1, is_read_io(fio->op), fio->type, fio->temp);
93dfe2ac 393
09cbfeaf 394 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
93dfe2ac 395 bio_put(bio);
93dfe2ac
JK		 396 return -EFAULT;
397 }
04d328de 398 bio_set_op_attrs(bio, fio->op, fio->op_flags);
93dfe2ac 399
4fc29c1a 400 __submit_bio(fio->sbi, bio, fio->type);
d1b3e72d
JK		 401
402 if (!is_read_io(fio->op))
403 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
93dfe2ac
JK		 404 return 0;
405}
406
b9109b0e 407int f2fs_submit_page_write(struct f2fs_io_info *fio)
93dfe2ac 408{
05ca3632 409 struct f2fs_sb_info *sbi = fio->sbi;
458e6197 410 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
a912b54d 411 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
4375a336 412 struct page *bio_page;
0a595eba 413 int err = 0;
93dfe2ac 414
b9109b0e 415 f2fs_bug_on(sbi, is_read_io(fio->op));
93dfe2ac 416
fb830fc5
CY		 417 down_write(&io->io_rwsem);
418next:
419 if (fio->in_list) {
420 spin_lock(&io->io_lock);
421 if (list_empty(&io->io_list)) {
422 spin_unlock(&io->io_lock);
423 goto out_fail;
424 }
425 fio = list_first_entry(&io->io_list,
426 struct f2fs_io_info, list);
427 list_del(&fio->list);
428 spin_unlock(&io->io_lock);
429 }
93dfe2ac 430
7a9d7548 431 if (fio->old_blkaddr != NEW_ADDR)
0833721e
YH		 432 verify_block_addr(fio, fio->old_blkaddr);
433 verify_block_addr(fio, fio->new_blkaddr);
93dfe2ac 434
36951b38
CY		 435 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
436
ebf7c522
TM		 437 /* set submitted = true as a return value */
438 fio->submitted = true;
d68f735b 439
b9109b0e 440 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
93dfe2ac 441
7a9d7548 442 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
3c62be17
JK		 443 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
444 !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
458e6197 445 __submit_merged_bio(io);
93dfe2ac
JK		 446alloc_new:
447 if (io->bio == NULL) {
0a595eba
JK		 448 if ((fio->type == DATA || fio->type == NODE) &&
449 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
450 err = -EAGAIN;
b9109b0e 451 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
0a595eba
JK		 452 goto out_fail;
453 }
578c6478 454 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
0cdd3195
HL		 455 BIO_MAX_PAGES, false,
456 fio->type, fio->temp);
458e6197 457 io->fio = *fio;
93dfe2ac
JK		 458 }
459
a912b54d 460 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
458e6197 461 __submit_merged_bio(io);
93dfe2ac
JK		 462 goto alloc_new;
463 }
464
578c6478
YY		 465 if (fio->io_wbc)
466 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
467
7a9d7548 468 io->last_block_in_bio = fio->new_blkaddr;
05ca3632 469 f2fs_trace_ios(fio, 0);
fb830fc5
CY		 470
471 trace_f2fs_submit_page_write(fio->page, fio);
472
473 if (fio->in_list)
474 goto next;
0a595eba 475out_fail:
df0f8dc0 476 up_write(&io->io_rwsem);
0a595eba 477 return err;
93dfe2ac
JK		 478}
479
13ba41e3
JK		 480static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
481 unsigned nr_pages)
482{
483 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
484 struct fscrypt_ctx *ctx = NULL;
485 struct bio *bio;
486
487 if (f2fs_encrypted_file(inode)) {
488 ctx = fscrypt_get_ctx(inode, GFP_NOFS);
489 if (IS_ERR(ctx))
490 return ERR_CAST(ctx);
491
492 /* wait the page to be moved by cleaning */
493 f2fs_wait_on_block_writeback(sbi, blkaddr);
494 }
495
d62fe971 496 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
13ba41e3
JK		 497 if (!bio) {
498 if (ctx)
499 fscrypt_release_ctx(ctx);
500 return ERR_PTR(-ENOMEM);
501 }
502 f2fs_target_device(sbi, blkaddr, bio);
503 bio->bi_end_io = f2fs_read_end_io;
504 bio->bi_private = ctx;
505 bio_set_op_attrs(bio, REQ_OP_READ, 0);
506
507 return bio;
508}
509
510/* This can handle encryption stuffs */
511static int f2fs_submit_page_read(struct inode *inode, struct page *page,
512 block_t blkaddr)
513{
514 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1);
515
516 if (IS_ERR(bio))
517 return PTR_ERR(bio);
518
519 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
520 bio_put(bio);
521 return -EFAULT;
522 }
523 __submit_bio(F2FS_I_SB(inode), bio, DATA);
524 return 0;
525}
526
46008c6d
CY		 527static void __set_data_blkaddr(struct dnode_of_data *dn)
528{
529 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
530 __le32 *addr_array;
7a2af766
CY		 531 int base = 0;
532
533 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
534 base = get_extra_isize(dn->inode);
46008c6d
CY		 535
536 /* Get physical address of data block */
537 addr_array = blkaddr_in_node(rn);
7a2af766 538 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
46008c6d
CY		 539}
540
0a8165d7 541/*
eb47b800
JK		 542 * Lock ordering for the change of data block address:
543 * ->data_page
544 * ->node_page
545 * update block addresses in the node page
546 */
216a620a 547void set_data_blkaddr(struct dnode_of_data *dn)
eb47b800 548{
46008c6d
CY		 549 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
550 __set_data_blkaddr(dn);
551 if (set_page_dirty(dn->node_page))
12719ae1 552 dn->node_changed = true;
eb47b800
JK		 553}
554
f28b3434
CY		 555void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
556{
557 dn->data_blkaddr = blkaddr;
558 set_data_blkaddr(dn);
559 f2fs_update_extent_cache(dn);
560}
561
46008c6d
CY		 562/* dn->ofs_in_node will be returned with up-to-date last block pointer */
563int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
eb47b800 564{
4081363f 565 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
0abd675e 566 int err;
eb47b800 567
46008c6d
CY		 568 if (!count)
569 return 0;
570
91942321 571 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
eb47b800 572 return -EPERM;
0abd675e
CY		 573 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
574 return err;
eb47b800 575
46008c6d
CY		 576 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
577 dn->ofs_in_node, count);
578
579 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
580
581 for (; count > 0; dn->ofs_in_node++) {
7a2af766
CY		 582 block_t blkaddr = datablock_addr(dn->inode,
583 dn->node_page, dn->ofs_in_node);
46008c6d
CY		 584 if (blkaddr == NULL_ADDR) {
585 dn->data_blkaddr = NEW_ADDR;
586 __set_data_blkaddr(dn);
587 count--;
588 }
589 }
590
591 if (set_page_dirty(dn->node_page))
592 dn->node_changed = true;
eb47b800
JK		 593 return 0;
594}
595
46008c6d
CY		 596/* Should keep dn->ofs_in_node unchanged */
597int reserve_new_block(struct dnode_of_data *dn)
598{
599 unsigned int ofs_in_node = dn->ofs_in_node;
600 int ret;
601
602 ret = reserve_new_blocks(dn, 1);
603 dn->ofs_in_node = ofs_in_node;
604 return ret;
605}
606
b600965c
HL		 607int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
608{
609 bool need_put = dn->inode_page ? false : true;
610 int err;
611
612 err = get_dnode_of_data(dn, index, ALLOC_NODE);
613 if (err)
614 return err;
a8865372 615
b600965c
HL		 616 if (dn->data_blkaddr == NULL_ADDR)
617 err = reserve_new_block(dn);
a8865372 618 if (err || need_put)
b600965c
HL		 619 f2fs_put_dnode(dn);
620 return err;
621}
622
759af1c9 623int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
eb47b800 624{
e15882b6 625 struct extent_info ei = {0,0,0};
759af1c9 626 struct inode *inode = dn->inode;
028a41e8 627
759af1c9
FL		 628 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
629 dn->data_blkaddr = ei.blk + index - ei.fofs;
630 return 0;
429511cd 631 }
028a41e8 632
759af1c9 633 return f2fs_reserve_block(dn, index);
eb47b800
JK		 634}
635
a56c7c6f 636struct page *get_read_data_page(struct inode *inode, pgoff_t index,
04d328de 637 int op_flags, bool for_write)
eb47b800 638{
eb47b800
JK		 639 struct address_space *mapping = inode->i_mapping;
640 struct dnode_of_data dn;
641 struct page *page;
e15882b6 642 struct extent_info ei = {0,0,0};
eb47b800 643 int err;
4375a336 644
a56c7c6f 645 page = f2fs_grab_cache_page(mapping, index, for_write);
650495de
JK		 646 if (!page)
647 return ERR_PTR(-ENOMEM);
648
cb3bc9ee
CY		 649 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
650 dn.data_blkaddr = ei.blk + index - ei.fofs;
651 goto got_it;
652 }
653
eb47b800 654 set_new_dnode(&dn, inode, NULL, NULL, 0);
266e97a8 655 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
86531d6b
JK		 656 if (err)
657 goto put_err;
eb47b800
JK		 658 f2fs_put_dnode(&dn);
659
6bacf52f 660 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
86531d6b
JK		 661 err = -ENOENT;
662 goto put_err;
650495de 663 }
cb3bc9ee 664got_it:
43f3eae1
JK		 665 if (PageUptodate(page)) {
666 unlock_page(page);
eb47b800 667 return page;
43f3eae1 668 }
eb47b800 669
d59ff4df
JK		 670 /*
671 * A new dentry page is allocated but not able to be written, since its
672 * new inode page couldn't be allocated due to -ENOSPC.
673 * In such the case, its blkaddr can be remained as NEW_ADDR.
674 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
675 */
676 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 677 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 678 if (!PageUptodate(page))
679 SetPageUptodate(page);
43f3eae1 680 unlock_page(page);
d59ff4df
JK		 681 return page;
682 }
eb47b800 683
13ba41e3 684 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
393ff91f 685 if (err)
86531d6b 686 goto put_err;
43f3eae1 687 return page;
86531d6b
JK		 688
689put_err:
690 f2fs_put_page(page, 1);
691 return ERR_PTR(err);
43f3eae1
JK		 692}
693
694struct page *find_data_page(struct inode *inode, pgoff_t index)
695{
696 struct address_space *mapping = inode->i_mapping;
697 struct page *page;
698
699 page = find_get_page(mapping, index);
700 if (page && PageUptodate(page))
701 return page;
702 f2fs_put_page(page, 0);
703
70fd7614 704 page = get_read_data_page(inode, index, 0, false);
43f3eae1
JK		 705 if (IS_ERR(page))
706 return page;
707
708 if (PageUptodate(page))
709 return page;
710
711 wait_on_page_locked(page);
712 if (unlikely(!PageUptodate(page))) {
713 f2fs_put_page(page, 0);
714 return ERR_PTR(-EIO);
715 }
716 return page;
717}
718
719/*
720 * If it tries to access a hole, return an error.
721 * Because, the callers, functions in dir.c and GC, should be able to know
722 * whether this page exists or not.
723 */
a56c7c6f
JK		 724struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
725 bool for_write)
43f3eae1
JK		 726{
727 struct address_space *mapping = inode->i_mapping;
728 struct page *page;
729repeat:
70fd7614 730 page = get_read_data_page(inode, index, 0, for_write);
43f3eae1
JK		 731 if (IS_ERR(page))
732 return page;
393ff91f 733
43f3eae1 734 /* wait for read completion */
393ff91f 735 lock_page(page);
6bacf52f 736 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 737 f2fs_put_page(page, 1);
738 goto repeat;
eb47b800 739 }
1563ac75
CY		 740 if (unlikely(!PageUptodate(page))) {
741 f2fs_put_page(page, 1);
742 return ERR_PTR(-EIO);
743 }
eb47b800
JK		 744 return page;
745}
746
0a8165d7 747/*
eb47b800
JK		 748 * Caller ensures that this data page is never allocated.
749 * A new zero-filled data page is allocated in the page cache.
39936837 750 *
4f4124d0
CY		 751 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
752 * f2fs_unlock_op().
470f00e9
CY		 753 * Note that, ipage is set only by make_empty_dir, and if any error occur,
754 * ipage should be released by this function.
eb47b800 755 */
64aa7ed9 756struct page *get_new_data_page(struct inode *inode,
a8865372 757 struct page *ipage, pgoff_t index, bool new_i_size)
eb47b800 758{
eb47b800
JK		 759 struct address_space *mapping = inode->i_mapping;
760 struct page *page;
761 struct dnode_of_data dn;
762 int err;
7612118a 763
a56c7c6f 764 page = f2fs_grab_cache_page(mapping, index, true);
470f00e9
CY		 765 if (!page) {
766 /*
767 * before exiting, we should make sure ipage will be released
768 * if any error occur.
769 */
770 f2fs_put_page(ipage, 1);
01f28610 771 return ERR_PTR(-ENOMEM);
470f00e9 772 }
eb47b800 773
a8865372 774 set_new_dnode(&dn, inode, ipage, NULL, 0);
b600965c 775 err = f2fs_reserve_block(&dn, index);
01f28610
JK		 776 if (err) {
777 f2fs_put_page(page, 1);
eb47b800 778 return ERR_PTR(err);
a8865372 779 }
01f28610
JK		 780 if (!ipage)
781 f2fs_put_dnode(&dn);
eb47b800
JK		 782
783 if (PageUptodate(page))
01f28610 784 goto got_it;
eb47b800
JK		 785
786 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 787 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 788 if (!PageUptodate(page))
789 SetPageUptodate(page);
eb47b800 790 } else {
4375a336 791 f2fs_put_page(page, 1);
a8865372 792
7612118a
JK		 793 /* if ipage exists, blkaddr should be NEW_ADDR */
794 f2fs_bug_on(F2FS_I_SB(inode), ipage);
795 page = get_lock_data_page(inode, index, true);
4375a336 796 if (IS_ERR(page))
7612118a 797 return page;
eb47b800 798 }
01f28610 799got_it:
9edcdabf 800 if (new_i_size && i_size_read(inode) <
ee6d182f 801 ((loff_t)(index + 1) << PAGE_SHIFT))
fc9581c8 802 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
eb47b800
JK		 803 return page;
804}
805
d5097be5 806static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
bfad7c2d 807{
4081363f 808 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
bfad7c2d 809 struct f2fs_summary sum;
bfad7c2d 810 struct node_info ni;
976e4c50 811 pgoff_t fofs;
46008c6d 812 blkcnt_t count = 1;
0abd675e 813 int err;
bfad7c2d 814
91942321 815 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
bfad7c2d 816 return -EPERM;
df6136ef 817
7a2af766
CY		 818 dn->data_blkaddr = datablock_addr(dn->inode,
819 dn->node_page, dn->ofs_in_node);
df6136ef
CY		 820 if (dn->data_blkaddr == NEW_ADDR)
821 goto alloc;
822
0abd675e
CY		 823 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
824 return err;
bfad7c2d 825
df6136ef 826alloc:
bfad7c2d
JK		 827 get_node_info(sbi, dn->nid, &ni);
828 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
829
df6136ef 830 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
d5097be5 831 &sum, seg_type, NULL, false);
216a620a 832 set_data_blkaddr(dn);
bfad7c2d 833
976e4c50 834 /* update i_size */
81ca7350 835 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
976e4c50 836 dn->ofs_in_node;
09cbfeaf 837 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
fc9581c8 838 f2fs_i_size_write(dn->inode,
09cbfeaf 839 ((loff_t)(fofs + 1) << PAGE_SHIFT));
bfad7c2d
JK		 840 return 0;
841}
842
a7de6086 843int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
59b802e5 844{
b439b103 845 struct inode *inode = file_inode(iocb->ki_filp);
5b8db7fa 846 struct f2fs_map_blocks map;
d6d478a1 847 int flag;
a7de6086 848 int err = 0;
d6d478a1 849 bool direct_io = iocb->ki_flags & IOCB_DIRECT;
59b802e5 850
71ad682c 851 /* convert inline data for Direct I/O*/
d6d478a1 852 if (direct_io) {
71ad682c
WG		 853 err = f2fs_convert_inline_inode(inode);
854 if (err)
855 return err;
856 }
857
dc91de78
JK		 858 if (is_inode_flag_set(inode, FI_NO_PREALLOC))
859 return 0;
860
0080c507 861 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
dfd02e4d
CY		 862 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
863 if (map.m_len > map.m_lblk)
864 map.m_len -= map.m_lblk;
865 else
866 map.m_len = 0;
867
da85985c 868 map.m_next_pgofs = NULL;
c4020b2d 869 map.m_next_extent = NULL;
d5097be5 870 map.m_seg_type = NO_CHECK_TYPE;
2a340760 871
d6d478a1 872 if (direct_io) {
d5097be5 873 map.m_seg_type = rw_hint_to_seg_type(iocb->ki_hint);
b91050a8 874 flag = f2fs_force_buffered_io(inode, WRITE) ?
d6d478a1
CY		 875 F2FS_GET_BLOCK_PRE_AIO :
876 F2FS_GET_BLOCK_PRE_DIO;
877 goto map_blocks;
d5097be5 878 }
f2470371 879 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
a7de6086
JK		 880 err = f2fs_convert_inline_inode(inode);
881 if (err)
882 return err;
b439b103 883 }
d6d478a1 884 if (f2fs_has_inline_data(inode))
25006645 885 return err;
d6d478a1
CY		 886
887 flag = F2FS_GET_BLOCK_PRE_AIO;
888
889map_blocks:
890 err = f2fs_map_blocks(inode, &map, 1, flag);
891 if (map.m_len > 0 && err == -ENOSPC) {
892 if (!direct_io)
893 set_inode_flag(inode, FI_NO_PREALLOC);
894 err = 0;
25006645 895 }
a7de6086 896 return err;
59b802e5
JK		 897}
898
59c9081b
YH		 899static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
900{
901 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
902 if (lock)
903 down_read(&sbi->node_change);
904 else
905 up_read(&sbi->node_change);
906 } else {
907 if (lock)
908 f2fs_lock_op(sbi);
909 else
910 f2fs_unlock_op(sbi);
911 }
912}
913
0a8165d7 914/*
003a3e1d
JK		 915 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
916 * f2fs_map_blocks structure.
4f4124d0
CY		 917 * If original data blocks are allocated, then give them to blockdev.
918 * Otherwise,
919 * a. preallocate requested block addresses
920 * b. do not use extent cache for better performance
921 * c. give the block addresses to blockdev
eb47b800 922 */
d323d005 923int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
e2b4e2bc 924 int create, int flag)
eb47b800 925{
003a3e1d 926 unsigned int maxblocks = map->m_len;
eb47b800 927 struct dnode_of_data dn;
f9811703 928 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
ac6f1999 929 int mode = create ? ALLOC_NODE : LOOKUP_NODE;
46008c6d 930 pgoff_t pgofs, end_offset, end;
bfad7c2d 931 int err = 0, ofs = 1;
46008c6d
CY		 932 unsigned int ofs_in_node, last_ofs_in_node;
933 blkcnt_t prealloc;
e15882b6 934 struct extent_info ei = {0,0,0};
7df3a431 935 block_t blkaddr;
c4020b2d 936 unsigned int start_pgofs;
eb47b800 937
dfd02e4d
CY		 938 if (!maxblocks)
939 return 0;
940
003a3e1d
JK		 941 map->m_len = 0;
942 map->m_flags = 0;
943
944 /* it only supports block size == page size */
945 pgofs = (pgoff_t)map->m_lblk;
46008c6d 946 end = pgofs + maxblocks;
eb47b800 947
24b84912 948 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
003a3e1d
JK		 949 map->m_pblk = ei.blk + pgofs - ei.fofs;
950 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
951 map->m_flags = F2FS_MAP_MAPPED;
c4020b2d
CY		 952 if (map->m_next_extent)
953 *map->m_next_extent = pgofs + map->m_len;
bfad7c2d 954 goto out;
a2e7d1bf 955 }
bfad7c2d 956
4fe71e88 957next_dnode:
59b802e5 958 if (create)
59c9081b 959 __do_map_lock(sbi, flag, true);
eb47b800
JK		 960
961 /* When reading holes, we need its node page */
962 set_new_dnode(&dn, inode, NULL, NULL, 0);
bfad7c2d 963 err = get_dnode_of_data(&dn, pgofs, mode);
1ec79083 964 if (err) {
43473f96
CY		 965 if (flag == F2FS_GET_BLOCK_BMAP)
966 map->m_pblk = 0;
da85985c 967 if (err == -ENOENT) {
bfad7c2d 968 err = 0;
da85985c
CY		 969 if (map->m_next_pgofs)
970 *map->m_next_pgofs =
971 get_next_page_offset(&dn, pgofs);
c4020b2d
CY		 972 if (map->m_next_extent)
973 *map->m_next_extent =
974 get_next_page_offset(&dn, pgofs);
da85985c 975 }
bfad7c2d 976 goto unlock_out;
848753aa 977 }
973163fc 978
c4020b2d 979 start_pgofs = pgofs;
46008c6d 980 prealloc = 0;
230436b3 981 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
81ca7350 982 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
4fe71e88
CY		 983
984next_block:
7a2af766 985 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
4fe71e88
CY		 986
987 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
973163fc 988 if (create) {
f9811703
CY		 989 if (unlikely(f2fs_cp_error(sbi))) {
990 err = -EIO;
4fe71e88 991 goto sync_out;
f9811703 992 }
24b84912 993 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
46008c6d
CY		 994 if (blkaddr == NULL_ADDR) {
995 prealloc++;
996 last_ofs_in_node = dn.ofs_in_node;
997 }
24b84912 998 } else {
d5097be5
HL		 999 err = __allocate_data_block(&dn,
1000 map->m_seg_type);
6f2d8ed6 1001 if (!err)
91942321 1002 set_inode_flag(inode, FI_APPEND_WRITE);
24b84912 1003 }
973163fc 1004 if (err)
4fe71e88 1005 goto sync_out;
3f2be043 1006 map->m_flags |= F2FS_MAP_NEW;
4fe71e88 1007 blkaddr = dn.data_blkaddr;
973163fc 1008 } else {
43473f96
CY		 1009 if (flag == F2FS_GET_BLOCK_BMAP) {
1010 map->m_pblk = 0;
1011 goto sync_out;
1012 }
c4020b2d
CY		 1013 if (flag == F2FS_GET_BLOCK_PRECACHE)
1014 goto sync_out;
da85985c
CY		 1015 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1016 blkaddr == NULL_ADDR) {
1017 if (map->m_next_pgofs)
1018 *map->m_next_pgofs = pgofs + 1;
4c2ac6a8 1019 goto sync_out;
da85985c 1020 }
f3d98e74
CY		 1021 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1022 /* for defragment case */
1023 if (map->m_next_pgofs)
1024 *map->m_next_pgofs = pgofs + 1;
4fe71e88 1025 goto sync_out;
f3d98e74 1026 }
e2b4e2bc 1027 }
e2b4e2bc 1028 }
eb47b800 1029
46008c6d
CY		 1030 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1031 goto skip;
1032
4fe71e88
CY		 1033 if (map->m_len == 0) {
1034 /* preallocated unwritten block should be mapped for fiemap. */
1035 if (blkaddr == NEW_ADDR)
1036 map->m_flags |= F2FS_MAP_UNWRITTEN;
1037 map->m_flags |= F2FS_MAP_MAPPED;
1038
1039 map->m_pblk = blkaddr;
1040 map->m_len = 1;
1041 } else if ((map->m_pblk != NEW_ADDR &&
1042 blkaddr == (map->m_pblk + ofs)) ||
b439b103 1043 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
46008c6d 1044 flag == F2FS_GET_BLOCK_PRE_DIO) {
4fe71e88
CY		 1045 ofs++;
1046 map->m_len++;
1047 } else {
1048 goto sync_out;
1049 }
bfad7c2d 1050
46008c6d 1051skip:
bfad7c2d
JK		 1052 dn.ofs_in_node++;
1053 pgofs++;
1054
46008c6d
CY		 1055 /* preallocate blocks in batch for one dnode page */
1056 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1057 (pgofs == end || dn.ofs_in_node == end_offset)) {
7df3a431 1058
46008c6d
CY		 1059 dn.ofs_in_node = ofs_in_node;
1060 err = reserve_new_blocks(&dn, prealloc);
1061 if (err)
1062 goto sync_out;
bfad7c2d 1063
46008c6d
CY		 1064 map->m_len += dn.ofs_in_node - ofs_in_node;
1065 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1066 err = -ENOSPC;
1067 goto sync_out;
3104af35 1068 }
46008c6d
CY		 1069 dn.ofs_in_node = end_offset;
1070 }
1071
1072 if (pgofs >= end)
1073 goto sync_out;
1074 else if (dn.ofs_in_node < end_offset)
1075 goto next_block;
1076
c4020b2d
CY		 1077 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1078 if (map->m_flags & F2FS_MAP_MAPPED) {
1079 unsigned int ofs = start_pgofs - map->m_lblk;
1080
1081 f2fs_update_extent_cache_range(&dn,
1082 start_pgofs, map->m_pblk + ofs,
1083 map->m_len - ofs);
1084 }
1085 }
1086
46008c6d
CY		 1087 f2fs_put_dnode(&dn);
1088
1089 if (create) {
59c9081b 1090 __do_map_lock(sbi, flag, false);
6f2d8ed6 1091 f2fs_balance_fs(sbi, dn.node_changed);
eb47b800 1092 }
46008c6d 1093 goto next_dnode;
7df3a431 1094
bfad7c2d 1095sync_out:
c4020b2d
CY		 1096 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1097 if (map->m_flags & F2FS_MAP_MAPPED) {
1098 unsigned int ofs = start_pgofs - map->m_lblk;
1099
1100 f2fs_update_extent_cache_range(&dn,
1101 start_pgofs, map->m_pblk + ofs,
1102 map->m_len - ofs);
1103 }
1104 if (map->m_next_extent)
1105 *map->m_next_extent = pgofs + 1;
1106 }
eb47b800 1107 f2fs_put_dnode(&dn);
bfad7c2d 1108unlock_out:
2a340760 1109 if (create) {
59c9081b 1110 __do_map_lock(sbi, flag, false);
6f2d8ed6 1111 f2fs_balance_fs(sbi, dn.node_changed);
2a340760 1112 }
bfad7c2d 1113out:
003a3e1d 1114 trace_f2fs_map_blocks(inode, map, err);
bfad7c2d 1115 return err;
eb47b800
JK		 1116}
1117
b91050a8
HL		 1118bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1119{
1120 struct f2fs_map_blocks map;
1121 block_t last_lblk;
1122 int err;
1123
1124 if (pos + len > i_size_read(inode))
1125 return false;
1126
1127 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1128 map.m_next_pgofs = NULL;
1129 map.m_next_extent = NULL;
1130 map.m_seg_type = NO_CHECK_TYPE;
1131 last_lblk = F2FS_BLK_ALIGN(pos + len);
1132
1133 while (map.m_lblk < last_lblk) {
1134 map.m_len = last_lblk - map.m_lblk;
1135 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1136 if (err || map.m_len == 0)
1137 return false;
1138 map.m_lblk += map.m_len;
1139 }
1140 return true;
1141}
1142
003a3e1d 1143static int __get_data_block(struct inode *inode, sector_t iblock,
da85985c 1144 struct buffer_head *bh, int create, int flag,
d5097be5 1145 pgoff_t *next_pgofs, int seg_type)
003a3e1d
JK		 1146{
1147 struct f2fs_map_blocks map;
a7de6086 1148 int err;
003a3e1d
JK		 1149
1150 map.m_lblk = iblock;
1151 map.m_len = bh->b_size >> inode->i_blkbits;
da85985c 1152 map.m_next_pgofs = next_pgofs;
c4020b2d 1153 map.m_next_extent = NULL;
d5097be5 1154 map.m_seg_type = seg_type;
003a3e1d 1155
a7de6086
JK		 1156 err = f2fs_map_blocks(inode, &map, create, flag);
1157 if (!err) {
003a3e1d
JK		 1158 map_bh(bh, inode->i_sb, map.m_pblk);
1159 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
b86e3307 1160 bh->b_size = (u64)map.m_len << inode->i_blkbits;
003a3e1d 1161 }
a7de6086 1162 return err;
003a3e1d
JK		 1163}
1164
ccfb3000 1165static int get_data_block(struct inode *inode, sector_t iblock,
da85985c
CY		 1166 struct buffer_head *bh_result, int create, int flag,
1167 pgoff_t *next_pgofs)
e2b4e2bc 1168{
da85985c 1169 return __get_data_block(inode, iblock, bh_result, create,
d5097be5
HL		 1170 flag, next_pgofs,
1171 NO_CHECK_TYPE);
e2b4e2bc
CY		 1172}
1173
1174static int get_data_block_dio(struct inode *inode, sector_t iblock,
ccfb3000
JK		 1175 struct buffer_head *bh_result, int create)
1176{
e2b4e2bc 1177 return __get_data_block(inode, iblock, bh_result, create,
d5097be5
HL		 1178 F2FS_GET_BLOCK_DEFAULT, NULL,
1179 rw_hint_to_seg_type(
1180 inode->i_write_hint));
ccfb3000
JK		 1181}
1182
e2b4e2bc 1183static int get_data_block_bmap(struct inode *inode, sector_t iblock,
ccfb3000
JK		 1184 struct buffer_head *bh_result, int create)
1185{
179448bf 1186 /* Block number less than F2FS MAX BLOCKS */
e0afc4d6 1187 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
179448bf
YH		 1188 return -EFBIG;
1189
e2b4e2bc 1190 return __get_data_block(inode, iblock, bh_result, create,
d5097be5
HL		 1191 F2FS_GET_BLOCK_BMAP, NULL,
1192 NO_CHECK_TYPE);
ccfb3000
JK		 1193}
1194
7f63eb77
JK		 1195static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1196{
1197 return (offset >> inode->i_blkbits);
1198}
1199
1200static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1201{
1202 return (blk << inode->i_blkbits);
1203}
1204
442a9dbd
CY		 1205static int f2fs_xattr_fiemap(struct inode *inode,
1206 struct fiemap_extent_info *fieinfo)
1207{
1208 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1209 struct page *page;
1210 struct node_info ni;
1211 __u64 phys = 0, len;
1212 __u32 flags;
1213 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1214 int err = 0;
1215
1216 if (f2fs_has_inline_xattr(inode)) {
1217 int offset;
1218
1219 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1220 inode->i_ino, false);
1221 if (!page)
1222 return -ENOMEM;
1223
1224 get_node_info(sbi, inode->i_ino, &ni);
1225
1226 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1227 offset = offsetof(struct f2fs_inode, i_addr) +
1228 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
b323fd28 1229 get_inline_xattr_addrs(inode));
442a9dbd
CY		 1230
1231 phys += offset;
1232 len = inline_xattr_size(inode);
1233
1234 f2fs_put_page(page, 1);
1235
1236 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1237
1238 if (!xnid)
1239 flags |= FIEMAP_EXTENT_LAST;
1240
1241 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1242 if (err || err == 1)
1243 return err;
1244 }
1245
1246 if (xnid) {
1247 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1248 if (!page)
1249 return -ENOMEM;
1250
1251 get_node_info(sbi, xnid, &ni);
1252
1253 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1254 len = inode->i_sb->s_blocksize;
1255
1256 f2fs_put_page(page, 1);
1257
1258 flags = FIEMAP_EXTENT_LAST;
1259 }
1260
1261 if (phys)
1262 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1263
1264 return (err < 0 ? err : 0);
1265}
1266
9ab70134
JK		 1267int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1268 u64 start, u64 len)
1269{
7f63eb77
JK		 1270 struct buffer_head map_bh;
1271 sector_t start_blk, last_blk;
da85985c 1272 pgoff_t next_pgofs;
7f63eb77
JK		 1273 u64 logical = 0, phys = 0, size = 0;
1274 u32 flags = 0;
7f63eb77
JK		 1275 int ret = 0;
1276
c4020b2d
CY		 1277 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1278 ret = f2fs_precache_extents(inode);
1279 if (ret)
1280 return ret;
1281 }
1282
442a9dbd 1283 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
7f63eb77
JK		 1284 if (ret)
1285 return ret;
1286
f1b43d4c
CY		 1287 inode_lock(inode);
1288
442a9dbd
CY		 1289 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1290 ret = f2fs_xattr_fiemap(inode, fieinfo);
1291 goto out;
1292 }
1293
67f8cf3c
JK		 1294 if (f2fs_has_inline_data(inode)) {
1295 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1296 if (ret != -EAGAIN)
f1b43d4c 1297 goto out;
67f8cf3c
JK		 1298 }
1299
7f63eb77
JK		 1300 if (logical_to_blk(inode, len) == 0)
1301 len = blk_to_logical(inode, 1);
1302
1303 start_blk = logical_to_blk(inode, start);
1304 last_blk = logical_to_blk(inode, start + len - 1);
9a950d52 1305
7f63eb77
JK		 1306next:
1307 memset(&map_bh, 0, sizeof(struct buffer_head));
1308 map_bh.b_size = len;
1309
e2b4e2bc 1310 ret = get_data_block(inode, start_blk, &map_bh, 0,
da85985c 1311 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
7f63eb77
JK		 1312 if (ret)
1313 goto out;
1314
1315 /* HOLE */
1316 if (!buffer_mapped(&map_bh)) {
da85985c 1317 start_blk = next_pgofs;
58736fa6
CY		 1318
1319 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1320 F2FS_I_SB(inode)->max_file_blocks))
9a950d52 1321 goto prep_next;
58736fa6 1322
9a950d52
FL		 1323 flags |= FIEMAP_EXTENT_LAST;
1324 }
7f63eb77 1325
da5af127
CY		 1326 if (size) {
1327 if (f2fs_encrypted_inode(inode))
1328 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1329
9a950d52
FL		 1330 ret = fiemap_fill_next_extent(fieinfo, logical,
1331 phys, size, flags);
da5af127 1332 }
7f63eb77 1333
9a950d52
FL		 1334 if (start_blk > last_blk || ret)
1335 goto out;
7f63eb77 1336
9a950d52
FL		 1337 logical = blk_to_logical(inode, start_blk);
1338 phys = blk_to_logical(inode, map_bh.b_blocknr);
1339 size = map_bh.b_size;
1340 flags = 0;
1341 if (buffer_unwritten(&map_bh))
1342 flags = FIEMAP_EXTENT_UNWRITTEN;
7f63eb77 1343
9a950d52 1344 start_blk += logical_to_blk(inode, size);
7f63eb77 1345
9a950d52 1346prep_next:
7f63eb77
JK		 1347 cond_resched();
1348 if (fatal_signal_pending(current))
1349 ret = -EINTR;
1350 else
1351 goto next;
1352out:
1353 if (ret == 1)
1354 ret = 0;
1355
5955102c 1356 inode_unlock(inode);
7f63eb77 1357 return ret;
9ab70134
JK		 1358}
1359
f1e88660
JK		 1360/*
1361 * This function was originally taken from fs/mpage.c, and customized for f2fs.
1362 * Major change was from block_size == page_size in f2fs by default.
1363 */
1364static int f2fs_mpage_readpages(struct address_space *mapping,
1365 struct list_head *pages, struct page *page,
1366 unsigned nr_pages)
1367{
1368 struct bio *bio = NULL;
f1e88660
JK		 1369 sector_t last_block_in_bio = 0;
1370 struct inode *inode = mapping->host;
1371 const unsigned blkbits = inode->i_blkbits;
1372 const unsigned blocksize = 1 << blkbits;
1373 sector_t block_in_file;
1374 sector_t last_block;
1375 sector_t last_block_in_file;
1376 sector_t block_nr;
f1e88660
JK		 1377 struct f2fs_map_blocks map;
1378
1379 map.m_pblk = 0;
1380 map.m_lblk = 0;
1381 map.m_len = 0;
1382 map.m_flags = 0;
da85985c 1383 map.m_next_pgofs = NULL;
c4020b2d 1384 map.m_next_extent = NULL;
d5097be5 1385 map.m_seg_type = NO_CHECK_TYPE;
f1e88660 1386
736c0a74 1387 for (; nr_pages; nr_pages--) {
f1e88660 1388 if (pages) {
939afa94 1389 page = list_last_entry(pages, struct page, lru);
a83d50bc
KM		 1390
1391 prefetchw(&page->flags);
f1e88660
JK		 1392 list_del(&page->lru);
1393 if (add_to_page_cache_lru(page, mapping,
8a5c743e
MH		 1394 page->index,
1395 readahead_gfp_mask(mapping)))
f1e88660
JK		 1396 goto next_page;
1397 }
1398
1399 block_in_file = (sector_t)page->index;
1400 last_block = block_in_file + nr_pages;
1401 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1402 blkbits;
1403 if (last_block > last_block_in_file)
1404 last_block = last_block_in_file;
1405
1406 /*
1407 * Map blocks using the previous result first.
1408 */
1409 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1410 block_in_file > map.m_lblk &&
1411 block_in_file < (map.m_lblk + map.m_len))
1412 goto got_it;
1413
1414 /*
1415 * Then do more f2fs_map_blocks() calls until we are
1416 * done with this page.
1417 */
1418 map.m_flags = 0;
1419
1420 if (block_in_file < last_block) {
1421 map.m_lblk = block_in_file;
1422 map.m_len = last_block - block_in_file;
1423
46c9e141 1424 if (f2fs_map_blocks(inode, &map, 0,
f2220c7f 1425 F2FS_GET_BLOCK_DEFAULT))
f1e88660
JK		 1426 goto set_error_page;
1427 }
1428got_it:
1429 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1430 block_nr = map.m_pblk + block_in_file - map.m_lblk;
1431 SetPageMappedToDisk(page);
1432
1433 if (!PageUptodate(page) && !cleancache_get_page(page)) {
1434 SetPageUptodate(page);
1435 goto confused;
1436 }
1437 } else {
09cbfeaf 1438 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 1439 if (!PageUptodate(page))
1440 SetPageUptodate(page);
f1e88660
JK		 1441 unlock_page(page);
1442 goto next_page;
1443 }
1444
1445 /*
1446 * This page will go to BIO. Do we need to send this
1447 * BIO off first?
1448 */
3c62be17
JK		 1449 if (bio && (last_block_in_bio != block_nr - 1 ||
1450 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
f1e88660 1451submit_and_realloc:
4fc29c1a 1452 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1453 bio = NULL;
1454 }
1455 if (bio == NULL) {
13ba41e3 1456 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages);
1d353eb7
JK		 1457 if (IS_ERR(bio)) {
1458 bio = NULL;
f1e88660 1459 goto set_error_page;
4375a336 1460 }
f1e88660
JK		 1461 }
1462
1463 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1464 goto submit_and_realloc;
1465
1466 last_block_in_bio = block_nr;
1467 goto next_page;
1468set_error_page:
1469 SetPageError(page);
09cbfeaf 1470 zero_user_segment(page, 0, PAGE_SIZE);
f1e88660
JK		 1471 unlock_page(page);
1472 goto next_page;
1473confused:
1474 if (bio) {
4fc29c1a 1475 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1476 bio = NULL;
1477 }
1478 unlock_page(page);
1479next_page:
1480 if (pages)
09cbfeaf 1481 put_page(page);
f1e88660
JK		 1482 }
1483 BUG_ON(pages && !list_empty(pages));
1484 if (bio)
4fc29c1a 1485 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1486 return 0;
1487}
1488
eb47b800
JK		 1489static int f2fs_read_data_page(struct file *file, struct page *page)
1490{
9ffe0fb5 1491 struct inode *inode = page->mapping->host;
b3d208f9 1492 int ret = -EAGAIN;
9ffe0fb5 1493
c20e89cd
CY		 1494 trace_f2fs_readpage(page, DATA);
1495
e1c42045 1496 /* If the file has inline data, try to read it directly */
9ffe0fb5
HL		 1497 if (f2fs_has_inline_data(inode))
1498 ret = f2fs_read_inline_data(inode, page);
b3d208f9 1499 if (ret == -EAGAIN)
f1e88660 1500 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
9ffe0fb5 1501 return ret;
eb47b800
JK		 1502}
1503
1504static int f2fs_read_data_pages(struct file *file,
1505 struct address_space *mapping,
1506 struct list_head *pages, unsigned nr_pages)
1507{
71cb4aff 1508 struct inode *inode = mapping->host;
939afa94 1509 struct page *page = list_last_entry(pages, struct page, lru);
b8c29400
CY		 1510
1511 trace_f2fs_readpages(inode, page, nr_pages);
9ffe0fb5
HL		 1512
1513 /* If the file has inline data, skip readpages */
1514 if (f2fs_has_inline_data(inode))
1515 return 0;
1516
f1e88660 1517 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
eb47b800
JK		 1518}
1519
7eab0c0d
HP		 1520static int encrypt_one_page(struct f2fs_io_info *fio)
1521{
1522 struct inode *inode = fio->page->mapping->host;
1523 gfp_t gfp_flags = GFP_NOFS;
1524
1958593e 1525 if (!f2fs_encrypted_file(inode))
7eab0c0d
HP		 1526 return 0;
1527
1528 /* wait for GCed encrypted page writeback */
d4c759ee 1529 f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
7eab0c0d
HP		 1530
1531retry_encrypt:
1532 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1533 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1534 if (!IS_ERR(fio->encrypted_page))
1535 return 0;
1536
1537 /* flush pending IOs and wait for a while in the ENOMEM case */
1538 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
b9109b0e 1539 f2fs_flush_merged_writes(fio->sbi);
7eab0c0d
HP		 1540 congestion_wait(BLK_RW_ASYNC, HZ/50);
1541 gfp_flags |= __GFP_NOFAIL;
1542 goto retry_encrypt;
1543 }
1544 return PTR_ERR(fio->encrypted_page);
1545}
1546
bb9e3bb8
CY		 1547static inline bool check_inplace_update_policy(struct inode *inode,
1548 struct f2fs_io_info *fio)
7eab0c0d 1549{
bb9e3bb8
CY		 1550 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1551 unsigned int policy = SM_I(sbi)->ipu_policy;
7eab0c0d 1552
bb9e3bb8
CY		 1553 if (policy & (0x1 << F2FS_IPU_FORCE))
1554 return true;
1555 if (policy & (0x1 << F2FS_IPU_SSR) && need_SSR(sbi))
1556 return true;
1557 if (policy & (0x1 << F2FS_IPU_UTIL) &&
1558 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1559 return true;
1560 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && need_SSR(sbi) &&
1561 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1562 return true;
1563
1564 /*
1565 * IPU for rewrite async pages
1566 */
1567 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
1568 fio && fio->op == REQ_OP_WRITE &&
1569 !(fio->op_flags & REQ_SYNC) &&
1570 !f2fs_encrypted_inode(inode))
1571 return true;
1572
1573 /* this is only set during fdatasync */
1574 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
1575 is_inode_flag_set(inode, FI_NEED_IPU))
1576 return true;
1577
1578 return false;
1579}
1580
1581bool should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
1582{
1ad71a27
JK		 1583 if (f2fs_is_pinned_file(inode))
1584 return true;
bb9e3bb8
CY		 1585
1586 /* if this is cold file, we should overwrite to avoid fragmentation */
1587 if (file_is_cold(inode))
1588 return true;
1589
1590 return check_inplace_update_policy(inode, fio);
1591}
1592
1593bool should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
1594{
1595 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1596
1597 if (test_opt(sbi, LFS))
1598 return true;
1599 if (S_ISDIR(inode->i_mode))
1600 return true;
1601 if (f2fs_is_atomic_file(inode))
1602 return true;
1603 if (fio) {
1604 if (is_cold_data(fio->page))
1605 return true;
1606 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1607 return true;
1608 }
1609 return false;
1610}
1611
7eab0c0d
HP		 1612static inline bool need_inplace_update(struct f2fs_io_info *fio)
1613{
1614 struct inode *inode = fio->page->mapping->host;
1615
bb9e3bb8 1616 if (should_update_outplace(inode, fio))
7eab0c0d
HP		 1617 return false;
1618
bb9e3bb8 1619 return should_update_inplace(inode, fio);
7eab0c0d
HP		 1620}
1621
a817737e
JK		 1622static inline bool valid_ipu_blkaddr(struct f2fs_io_info *fio)
1623{
1624 if (fio->old_blkaddr == NEW_ADDR)
1625 return false;
1626 if (fio->old_blkaddr == NULL_ADDR)
1627 return false;
1628 return true;
1629}
1630
05ca3632 1631int do_write_data_page(struct f2fs_io_info *fio)
eb47b800 1632{
05ca3632 1633 struct page *page = fio->page;
eb47b800 1634 struct inode *inode = page->mapping->host;
eb47b800 1635 struct dnode_of_data dn;
e959c8f5
HP		 1636 struct extent_info ei = {0,0,0};
1637 bool ipu_force = false;
eb47b800
JK		 1638 int err = 0;
1639
1640 set_new_dnode(&dn, inode, NULL, NULL, 0);
e959c8f5
HP		 1641 if (need_inplace_update(fio) &&
1642 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1643 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
a817737e
JK		 1644
1645 if (valid_ipu_blkaddr(fio)) {
e959c8f5 1646 ipu_force = true;
cc15620b 1647 fio->need_lock = LOCK_DONE;
e959c8f5
HP		 1648 goto got_it;
1649 }
1650 }
279d6df2 1651
d29460e5
JK		 1652 /* Deadlock due to between page->lock and f2fs_lock_op */
1653 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1654 return -EAGAIN;
279d6df2 1655
266e97a8 1656 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
eb47b800 1657 if (err)
279d6df2 1658 goto out;
eb47b800 1659
28bc106b 1660 fio->old_blkaddr = dn.data_blkaddr;
eb47b800
JK		 1661
1662 /* This page is already truncated */
7a9d7548 1663 if (fio->old_blkaddr == NULL_ADDR) {
2bca1e23 1664 ClearPageUptodate(page);
eb47b800 1665 goto out_writepage;
2bca1e23 1666 }
e959c8f5 1667got_it:
eb47b800
JK		 1668 /*
1669 * If current allocation needs SSR,
1670 * it had better in-place writes for updated data.
1671 */
a817737e 1672 if (ipu_force || (valid_ipu_blkaddr(fio) && need_inplace_update(fio))) {
cc15620b
JK		 1673 err = encrypt_one_page(fio);
1674 if (err)
1675 goto out_writepage;
1676
1677 set_page_writeback(page);
279d6df2 1678 f2fs_put_dnode(&dn);
cc15620b 1679 if (fio->need_lock == LOCK_REQ)
279d6df2 1680 f2fs_unlock_op(fio->sbi);
d1b3e72d 1681 err = rewrite_data_page(fio);
7eab0c0d 1682 trace_f2fs_do_write_data_page(fio->page, IPU);
91942321 1683 set_inode_flag(inode, FI_UPDATE_WRITE);
279d6df2 1684 return err;
eb47b800 1685 }
279d6df2 1686
cc15620b
JK		 1687 if (fio->need_lock == LOCK_RETRY) {
1688 if (!f2fs_trylock_op(fio->sbi)) {
1689 err = -EAGAIN;
1690 goto out_writepage;
1691 }
1692 fio->need_lock = LOCK_REQ;
1693 }
1694
1695 err = encrypt_one_page(fio);
1696 if (err)
1697 goto out_writepage;
1698
1699 set_page_writeback(page);
1700
279d6df2
HP		 1701 /* LFS mode write path */
1702 write_data_page(&dn, fio);
1703 trace_f2fs_do_write_data_page(page, OPU);
1704 set_inode_flag(inode, FI_APPEND_WRITE);
1705 if (page->index == 0)
1706 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
eb47b800
JK		 1707out_writepage:
1708 f2fs_put_dnode(&dn);
279d6df2 1709out:
cc15620b 1710 if (fio->need_lock == LOCK_REQ)
279d6df2 1711 f2fs_unlock_op(fio->sbi);
eb47b800
JK		 1712 return err;
1713}
1714
d68f735b 1715static int __write_data_page(struct page *page, bool *submitted,
b0af6d49
CY		 1716 struct writeback_control *wbc,
1717 enum iostat_type io_type)
eb47b800
JK		 1718{
1719 struct inode *inode = page->mapping->host;
4081363f 1720 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
eb47b800
JK		 1721 loff_t i_size = i_size_read(inode);
1722 const pgoff_t end_index = ((unsigned long long) i_size)
09cbfeaf 1723 >> PAGE_SHIFT;
26de9b11 1724 loff_t psize = (page->index + 1) << PAGE_SHIFT;
9ffe0fb5 1725 unsigned offset = 0;
39936837 1726 bool need_balance_fs = false;
eb47b800 1727 int err = 0;
458e6197 1728 struct f2fs_io_info fio = {
05ca3632 1729 .sbi = sbi,
39d787be 1730 .ino = inode->i_ino,
458e6197 1731 .type = DATA,
04d328de 1732 .op = REQ_OP_WRITE,
7637241e 1733 .op_flags = wbc_to_write_flags(wbc),
e959c8f5 1734 .old_blkaddr = NULL_ADDR,
05ca3632 1735 .page = page,
4375a336 1736 .encrypted_page = NULL,
d68f735b 1737 .submitted = false,
cc15620b 1738 .need_lock = LOCK_RETRY,
b0af6d49 1739 .io_type = io_type,
578c6478 1740 .io_wbc = wbc,
458e6197 1741 };
eb47b800 1742
ecda0de3
CY		 1743 trace_f2fs_writepage(page, DATA);
1744
db198ae0
CY		 1745 /* we should bypass data pages to proceed the kworkder jobs */
1746 if (unlikely(f2fs_cp_error(sbi))) {
1747 mapping_set_error(page->mapping, -EIO);
1748 goto out;
1749 }
1750
0771fcc7
CY		 1751 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1752 goto redirty_out;
1753
eb47b800 1754 if (page->index < end_index)
39936837 1755 goto write;
eb47b800
JK		 1756
1757 /*
1758 * If the offset is out-of-range of file size,
1759 * this page does not have to be written to disk.
1760 */
09cbfeaf 1761 offset = i_size & (PAGE_SIZE - 1);
76f60268 1762 if ((page->index >= end_index + 1) || !offset)
39936837 1763 goto out;
eb47b800 1764
09cbfeaf 1765 zero_user_segment(page, offset, PAGE_SIZE);
39936837 1766write:
1e84371f
JK		 1767 if (f2fs_is_drop_cache(inode))
1768 goto out;
e6e5f561
JK		 1769 /* we should not write 0'th page having journal header */
1770 if (f2fs_is_volatile_file(inode) && (!page->index ||
1771 (!wbc->for_reclaim &&
1772 available_free_memory(sbi, BASE_CHECK))))
1e84371f 1773 goto redirty_out;
eb47b800 1774
39936837 1775 /* Dentry blocks are controlled by checkpoint */
eb47b800 1776 if (S_ISDIR(inode->i_mode)) {
cc15620b 1777 fio.need_lock = LOCK_DONE;
05ca3632 1778 err = do_write_data_page(&fio);
8618b881
JK		 1779 goto done;
1780 }
9ffe0fb5 1781
8618b881 1782 if (!wbc->for_reclaim)
39936837 1783 need_balance_fs = true;
7f3037a5 1784 else if (has_not_enough_free_secs(sbi, 0, 0))
39936837 1785 goto redirty_out;
ef095d19
JK		 1786 else
1787 set_inode_flag(inode, FI_HOT_DATA);
eb47b800 1788
b3d208f9 1789 err = -EAGAIN;
dd7b2333 1790 if (f2fs_has_inline_data(inode)) {
b3d208f9 1791 err = f2fs_write_inline_data(inode, page);
dd7b2333
YH		 1792 if (!err)
1793 goto out;
1794 }
279d6df2 1795
cc15620b 1796 if (err == -EAGAIN) {
05ca3632 1797 err = do_write_data_page(&fio);
cc15620b
JK		 1798 if (err == -EAGAIN) {
1799 fio.need_lock = LOCK_REQ;
1800 err = do_write_data_page(&fio);
1801 }
1802 }
a0d00fad 1803
eb449797
CY		 1804 if (err) {
1805 file_set_keep_isize(inode);
1806 } else {
1807 down_write(&F2FS_I(inode)->i_sem);
1808 if (F2FS_I(inode)->last_disk_size < psize)
1809 F2FS_I(inode)->last_disk_size = psize;
1810 up_write(&F2FS_I(inode)->i_sem);
1811 }
279d6df2 1812
8618b881
JK		 1813done:
1814 if (err && err != -ENOENT)
1815 goto redirty_out;
eb47b800 1816
39936837 1817out:
a7ffdbe2 1818 inode_dec_dirty_pages(inode);
2bca1e23
JK		 1819 if (err)
1820 ClearPageUptodate(page);
0c3a5797
CY		 1821
1822 if (wbc->for_reclaim) {
b9109b0e 1823 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
ef095d19 1824 clear_inode_flag(inode, FI_HOT_DATA);
0c3a5797 1825 remove_dirty_inode(inode);
d68f735b 1826 submitted = NULL;
0c3a5797
CY		 1827 }
1828
eb47b800 1829 unlock_page(page);
a7881893
JK		 1830 if (!S_ISDIR(inode->i_mode))
1831 f2fs_balance_fs(sbi, need_balance_fs);
0c3a5797 1832
d68f735b 1833 if (unlikely(f2fs_cp_error(sbi))) {
b9109b0e 1834 f2fs_submit_merged_write(sbi, DATA);
d68f735b
JK		 1835 submitted = NULL;
1836 }
1837
1838 if (submitted)
1839 *submitted = fio.submitted;
0c3a5797 1840
eb47b800
JK		 1841 return 0;
1842
eb47b800 1843redirty_out:
76f60268 1844 redirty_page_for_writepage(wbc, page);
0002b61b
CY		 1845 if (!err)
1846 return AOP_WRITEPAGE_ACTIVATE;
b230e6ca
JK		 1847 unlock_page(page);
1848 return err;
fa9150a8
NJ		 1849}
1850
f566bae8
JK		 1851static int f2fs_write_data_page(struct page *page,
1852 struct writeback_control *wbc)
1853{
b0af6d49 1854 return __write_data_page(page, NULL, wbc, FS_DATA_IO);
f566bae8
JK		 1855}
1856
8f46dcae
CY		 1857/*
1858 * This function was copied from write_cche_pages from mm/page-writeback.c.
1859 * The major change is making write step of cold data page separately from
1860 * warm/hot data page.
1861 */
1862static int f2fs_write_cache_pages(struct address_space *mapping,
b0af6d49
CY		 1863 struct writeback_control *wbc,
1864 enum iostat_type io_type)
8f46dcae
CY		 1865{
1866 int ret = 0;
1867 int done = 0;
1868 struct pagevec pvec;
1869 int nr_pages;
1870 pgoff_t uninitialized_var(writeback_index);
1871 pgoff_t index;
1872 pgoff_t end; /* Inclusive */
1873 pgoff_t done_index;
942fd319 1874 pgoff_t last_idx = ULONG_MAX;
8f46dcae
CY		 1875 int cycled;
1876 int range_whole = 0;
1877 int tag;
8f46dcae 1878
86679820 1879 pagevec_init(&pvec);
46ae957f 1880
ef095d19
JK		 1881 if (get_dirty_pages(mapping->host) <=
1882 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
1883 set_inode_flag(mapping->host, FI_HOT_DATA);
1884 else
1885 clear_inode_flag(mapping->host, FI_HOT_DATA);
1886
8f46dcae
CY		 1887 if (wbc->range_cyclic) {
1888 writeback_index = mapping->writeback_index; /* prev offset */
1889 index = writeback_index;
1890 if (index == 0)
1891 cycled = 1;
1892 else
1893 cycled = 0;
1894 end = -1;
1895 } else {
09cbfeaf
KS		 1896 index = wbc->range_start >> PAGE_SHIFT;
1897 end = wbc->range_end >> PAGE_SHIFT;
8f46dcae
CY		 1898 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1899 range_whole = 1;
1900 cycled = 1; /* ignore range_cyclic tests */
1901 }
1902 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1903 tag = PAGECACHE_TAG_TOWRITE;
1904 else
1905 tag = PAGECACHE_TAG_DIRTY;
1906retry:
1907 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1908 tag_pages_for_writeback(mapping, index, end);
1909 done_index = index;
1910 while (!done && (index <= end)) {
1911 int i;
1912
69c4f35d 1913 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
67fd707f 1914 tag);
8f46dcae
CY		 1915 if (nr_pages == 0)
1916 break;
1917
1918 for (i = 0; i < nr_pages; i++) {
1919 struct page *page = pvec.pages[i];
d68f735b 1920 bool submitted = false;
8f46dcae 1921
8f46dcae 1922 done_index = page->index;
d29460e5 1923retry_write:
8f46dcae
CY		 1924 lock_page(page);
1925
1926 if (unlikely(page->mapping != mapping)) {
1927continue_unlock:
1928 unlock_page(page);
1929 continue;
1930 }
1931
1932 if (!PageDirty(page)) {
1933 /* someone wrote it for us */
1934 goto continue_unlock;
1935 }
1936
8f46dcae
CY		 1937 if (PageWriteback(page)) {
1938 if (wbc->sync_mode != WB_SYNC_NONE)
fec1d657
JK		 1939 f2fs_wait_on_page_writeback(page,
1940 DATA, true);
8f46dcae
CY		 1941 else
1942 goto continue_unlock;
1943 }
1944
1945 BUG_ON(PageWriteback(page));
1946 if (!clear_page_dirty_for_io(page))
1947 goto continue_unlock;
1948
b0af6d49 1949 ret = __write_data_page(page, &submitted, wbc, io_type);
8f46dcae 1950 if (unlikely(ret)) {
0002b61b
CY		 1951 /*
1952 * keep nr_to_write, since vfs uses this to
1953 * get # of written pages.
1954 */
1955 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1956 unlock_page(page);
1957 ret = 0;
1958 continue;
d29460e5
JK		 1959 } else if (ret == -EAGAIN) {
1960 ret = 0;
1961 if (wbc->sync_mode == WB_SYNC_ALL) {
1962 cond_resched();
1963 congestion_wait(BLK_RW_ASYNC,
1964 HZ/50);
1965 goto retry_write;
1966 }
1967 continue;
0002b61b 1968 }
b230e6ca
JK		 1969 done_index = page->index + 1;
1970 done = 1;
1971 break;
d68f735b 1972 } else if (submitted) {
942fd319 1973 last_idx = page->index;
8f46dcae
CY		 1974 }
1975
687de7f1
JK		 1976 /* give a priority to WB_SYNC threads */
1977 if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
1978 --wbc->nr_to_write <= 0) &&
1979 wbc->sync_mode == WB_SYNC_NONE) {
8f46dcae
CY		 1980 done = 1;
1981 break;
1982 }
1983 }
1984 pagevec_release(&pvec);
1985 cond_resched();
1986 }
1987
8f46dcae
CY		 1988 if (!cycled && !done) {
1989 cycled = 1;
1990 index = 0;
1991 end = writeback_index - 1;
1992 goto retry;
1993 }
1994 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1995 mapping->writeback_index = done_index;
1996
942fd319 1997 if (last_idx != ULONG_MAX)
b9109b0e
JK		 1998 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
1999 0, last_idx, DATA);
6ca56ca4 2000
8f46dcae
CY		 2001 return ret;
2002}
2003
b0af6d49
CY		 2004int __f2fs_write_data_pages(struct address_space *mapping,
2005 struct writeback_control *wbc,
2006 enum iostat_type io_type)
eb47b800
JK		 2007{
2008 struct inode *inode = mapping->host;
4081363f 2009 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
9dfa1baf 2010 struct blk_plug plug;
eb47b800 2011 int ret;
eb47b800 2012
cfb185a1 2013 /* deal with chardevs and other special file */
2014 if (!mapping->a_ops->writepage)
2015 return 0;
2016
6a290544
CY		 2017 /* skip writing if there is no dirty page in this inode */
2018 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
2019 return 0;
2020
0771fcc7
CY		 2021 /* during POR, we don't need to trigger writepage at all. */
2022 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2023 goto skip_write;
2024
a1257023
JK		 2025 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
2026 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
2027 available_free_memory(sbi, DIRTY_DENTS))
2028 goto skip_write;
2029
d323d005 2030 /* skip writing during file defragment */
91942321 2031 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
d323d005
CY		 2032 goto skip_write;
2033
d31c7c3f
YH		 2034 trace_f2fs_writepages(mapping->host, wbc, DATA);
2035
687de7f1
JK		 2036 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
2037 if (wbc->sync_mode == WB_SYNC_ALL)
2038 atomic_inc(&sbi->wb_sync_req);
2039 else if (atomic_read(&sbi->wb_sync_req))
2040 goto skip_write;
2041
9dfa1baf 2042 blk_start_plug(&plug);
b0af6d49 2043 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
9dfa1baf 2044 blk_finish_plug(&plug);
687de7f1
JK		 2045
2046 if (wbc->sync_mode == WB_SYNC_ALL)
2047 atomic_dec(&sbi->wb_sync_req);
28ea6162
JK		 2048 /*
2049 * if some pages were truncated, we cannot guarantee its mapping->host
2050 * to detect pending bios.
2051 */
458e6197 2052
c227f912 2053 remove_dirty_inode(inode);
eb47b800 2054 return ret;
d3baf95d
JK		 2055
2056skip_write:
a7ffdbe2 2057 wbc->pages_skipped += get_dirty_pages(inode);
d31c7c3f 2058 trace_f2fs_writepages(mapping->host, wbc, DATA);
d3baf95d 2059 return 0;
eb47b800
JK		 2060}
2061
b0af6d49
CY		 2062static int f2fs_write_data_pages(struct address_space *mapping,
2063 struct writeback_control *wbc)
2064{
2065 struct inode *inode = mapping->host;
2066
2067 return __f2fs_write_data_pages(mapping, wbc,
2068 F2FS_I(inode)->cp_task == current ?
2069 FS_CP_DATA_IO : FS_DATA_IO);
2070}
2071
3aab8f82
CY		 2072static void f2fs_write_failed(struct address_space *mapping, loff_t to)
2073{
2074 struct inode *inode = mapping->host;
819d9153 2075 loff_t i_size = i_size_read(inode);
3aab8f82 2076
819d9153 2077 if (to > i_size) {
5a3a2d83 2078 down_write(&F2FS_I(inode)->i_mmap_sem);
819d9153
JK		 2079 truncate_pagecache(inode, i_size);
2080 truncate_blocks(inode, i_size, true);
5a3a2d83 2081 up_write(&F2FS_I(inode)->i_mmap_sem);
3aab8f82
CY		 2082 }
2083}
2084
2aadac08
JK		 2085static int prepare_write_begin(struct f2fs_sb_info *sbi,
2086 struct page *page, loff_t pos, unsigned len,
2087 block_t *blk_addr, bool *node_changed)
2088{
2089 struct inode *inode = page->mapping->host;
2090 pgoff_t index = page->index;
2091 struct dnode_of_data dn;
2092 struct page *ipage;
b4d07a3e 2093 bool locked = false;
e15882b6 2094 struct extent_info ei = {0,0,0};
2aadac08
JK		 2095 int err = 0;
2096
24b84912
JK		 2097 /*
2098 * we already allocated all the blocks, so we don't need to get
2099 * the block addresses when there is no need to fill the page.
2100 */
dc91de78
JK		 2101 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
2102 !is_inode_flag_set(inode, FI_NO_PREALLOC))
24b84912
JK		 2103 return 0;
2104
b4d07a3e 2105 if (f2fs_has_inline_data(inode) ||
09cbfeaf 2106 (pos & PAGE_MASK) >= i_size_read(inode)) {
59c9081b 2107 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
b4d07a3e
JK		 2108 locked = true;
2109 }
2110restart:
2aadac08
JK		 2111 /* check inline_data */
2112 ipage = get_node_page(sbi, inode->i_ino);
2113 if (IS_ERR(ipage)) {
2114 err = PTR_ERR(ipage);
2115 goto unlock_out;
2116 }
2117
2118 set_new_dnode(&dn, inode, ipage, ipage, 0);
2119
2120 if (f2fs_has_inline_data(inode)) {
f2470371 2121 if (pos + len <= MAX_INLINE_DATA(inode)) {
2aadac08 2122 read_inline_data(page, ipage);
91942321 2123 set_inode_flag(inode, FI_DATA_EXIST);
ab47036d
CY		 2124 if (inode->i_nlink)
2125 set_inline_node(ipage);
2aadac08
JK		 2126 } else {
2127 err = f2fs_convert_inline_page(&dn, page);
2128 if (err)
b4d07a3e
JK		 2129 goto out;
2130 if (dn.data_blkaddr == NULL_ADDR)
2131 err = f2fs_get_block(&dn, index);
2132 }
2133 } else if (locked) {
2134 err = f2fs_get_block(&dn, index);
2135 } else {
2136 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
2137 dn.data_blkaddr = ei.blk + index - ei.fofs;
2138 } else {
b4d07a3e
JK		 2139 /* hole case */
2140 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
4da7bf5a 2141 if (err || dn.data_blkaddr == NULL_ADDR) {
b4d07a3e 2142 f2fs_put_dnode(&dn);
59c9081b
YH		 2143 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
2144 true);
b4d07a3e
JK		 2145 locked = true;
2146 goto restart;
2147 }
2aadac08
JK		 2148 }
2149 }
b4d07a3e 2150
2aadac08
JK		 2151 /* convert_inline_page can make node_changed */
2152 *blk_addr = dn.data_blkaddr;
2153 *node_changed = dn.node_changed;
b4d07a3e 2154out:
2aadac08
JK		 2155 f2fs_put_dnode(&dn);
2156unlock_out:
b4d07a3e 2157 if (locked)
59c9081b 2158 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
2aadac08
JK		 2159 return err;
2160}
2161
eb47b800
JK		 2162static int f2fs_write_begin(struct file *file, struct address_space *mapping,
2163 loff_t pos, unsigned len, unsigned flags,
2164 struct page **pagep, void **fsdata)
2165{
2166 struct inode *inode = mapping->host;
4081363f 2167 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
86531d6b 2168 struct page *page = NULL;
09cbfeaf 2169 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
a2e2e76b 2170 bool need_balance = false, drop_atomic = false;
2aadac08 2171 block_t blkaddr = NULL_ADDR;
eb47b800
JK		 2172 int err = 0;
2173
62aed044
CY		 2174 trace_f2fs_write_begin(inode, pos, len, flags);
2175
57864ae5
JK		 2176 if (f2fs_is_atomic_file(inode) &&
2177 !available_free_memory(sbi, INMEM_PAGES)) {
2178 err = -ENOMEM;
a2e2e76b 2179 drop_atomic = true;
57864ae5
JK		 2180 goto fail;
2181 }
2182
5f727395
JK		 2183 /*
2184 * We should check this at this moment to avoid deadlock on inode page
2185 * and #0 page. The locking rule for inline_data conversion should be:
2186 * lock_page(page #0) -> lock_page(inode_page)
2187 */
2188 if (index != 0) {
2189 err = f2fs_convert_inline_inode(inode);
2190 if (err)
2191 goto fail;
2192 }
afcb7ca0 2193repeat:
86d54795
JK		 2194 /*
2195 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
2196 * wait_for_stable_page. Will wait that below with our IO control.
2197 */
01eccef7 2198 page = f2fs_pagecache_get_page(mapping, index,
86d54795 2199 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3aab8f82
CY		 2200 if (!page) {
2201 err = -ENOMEM;
2202 goto fail;
2203 }
d5f66990 2204
eb47b800
JK		 2205 *pagep = page;
2206
2aadac08
JK		 2207 err = prepare_write_begin(sbi, page, pos, len,
2208 &blkaddr, &need_balance);
9ba69cf9 2209 if (err)
2aadac08 2210 goto fail;
9ba69cf9 2211
7f3037a5 2212 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
2a340760 2213 unlock_page(page);
2c4db1a6 2214 f2fs_balance_fs(sbi, true);
2a340760
JK		 2215 lock_page(page);
2216 if (page->mapping != mapping) {
2217 /* The page got truncated from under us */
2218 f2fs_put_page(page, 1);
2219 goto repeat;
2220 }
2221 }
2222
fec1d657 2223 f2fs_wait_on_page_writeback(page, DATA, false);
b3d208f9 2224
08b39fbd 2225 /* wait for GCed encrypted page writeback */
1958593e 2226 if (f2fs_encrypted_file(inode))
d4c759ee 2227 f2fs_wait_on_block_writeback(sbi, blkaddr);
08b39fbd 2228
649d7df2
JK		 2229 if (len == PAGE_SIZE || PageUptodate(page))
2230 return 0;
eb47b800 2231
746e2403
YH		 2232 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2233 zero_user_segment(page, len, PAGE_SIZE);
2234 return 0;
2235 }
2236
2aadac08 2237 if (blkaddr == NEW_ADDR) {
09cbfeaf 2238 zero_user_segment(page, 0, PAGE_SIZE);
649d7df2 2239 SetPageUptodate(page);
eb47b800 2240 } else {
13ba41e3
JK		 2241 err = f2fs_submit_page_read(inode, page, blkaddr);
2242 if (err)
78682f79 2243 goto fail;
d54c795b 2244
393ff91f 2245 lock_page(page);
6bacf52f 2246 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 2247 f2fs_put_page(page, 1);
2248 goto repeat;
eb47b800 2249 }
1563ac75
CY		 2250 if (unlikely(!PageUptodate(page))) {
2251 err = -EIO;
2252 goto fail;
4375a336 2253 }
eb47b800 2254 }
eb47b800 2255 return 0;
9ba69cf9 2256
3aab8f82 2257fail:
86531d6b 2258 f2fs_put_page(page, 1);
3aab8f82 2259 f2fs_write_failed(mapping, pos + len);
a2e2e76b 2260 if (drop_atomic)
57864ae5 2261 drop_inmem_pages_all(sbi);
3aab8f82 2262 return err;
eb47b800
JK		 2263}
2264
a1dd3c13
JK		 2265static int f2fs_write_end(struct file *file,
2266 struct address_space *mapping,
2267 loff_t pos, unsigned len, unsigned copied,
2268 struct page *page, void *fsdata)
2269{
2270 struct inode *inode = page->mapping->host;
2271
dfb2bf38
CY		 2272 trace_f2fs_write_end(inode, pos, len, copied);
2273
649d7df2
JK		 2274 /*
2275 * This should be come from len == PAGE_SIZE, and we expect copied
2276 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2277 * let generic_perform_write() try to copy data again through copied=0.
2278 */
2279 if (!PageUptodate(page)) {
746e2403 2280 if (unlikely(copied != len))
649d7df2
JK		 2281 copied = 0;
2282 else
2283 SetPageUptodate(page);
2284 }
2285 if (!copied)
2286 goto unlock_out;
2287
34ba94ba 2288 set_page_dirty(page);
a1dd3c13 2289
fc9581c8
JK		 2290 if (pos + copied > i_size_read(inode))
2291 f2fs_i_size_write(inode, pos + copied);
649d7df2 2292unlock_out:
3024c9a1 2293 f2fs_put_page(page, 1);
d0239e1b 2294 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
a1dd3c13
JK		 2295 return copied;
2296}
2297
6f673763
OS		 2298static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2299 loff_t offset)
944fcfc1
JK		 2300{
2301 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
944fcfc1 2302
944fcfc1
JK		 2303 if (offset & blocksize_mask)
2304 return -EINVAL;
2305
5b46f25d
AV		 2306 if (iov_iter_alignment(iter) & blocksize_mask)
2307 return -EINVAL;
2308
944fcfc1
JK		 2309 return 0;
2310}
2311
c8b8e32d 2312static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
eb47b800 2313{
b439b103 2314 struct address_space *mapping = iocb->ki_filp->f_mapping;
3aab8f82 2315 struct inode *inode = mapping->host;
0cdd3195 2316 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3aab8f82 2317 size_t count = iov_iter_count(iter);
c8b8e32d 2318 loff_t offset = iocb->ki_pos;
82e0a5aa 2319 int rw = iov_iter_rw(iter);
3aab8f82 2320 int err;
0cdd3195 2321 enum rw_hint hint = iocb->ki_hint;
63189b78 2322 int whint_mode = F2FS_OPTION(sbi).whint_mode;
944fcfc1 2323
b439b103 2324 err = check_direct_IO(inode, iter, offset);
b9d777b8
JK		 2325 if (err)
2326 return err;
9ffe0fb5 2327
b91050a8 2328 if (f2fs_force_buffered_io(inode, rw))
36abef4e 2329 return 0;
fcc85a4d 2330
5302fb00 2331 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
70407fad 2332
0cdd3195
HL		 2333 if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
2334 iocb->ki_hint = WRITE_LIFE_NOT_SET;
2335
b91050a8
HL		 2336 if (!down_read_trylock(&F2FS_I(inode)->dio_rwsem[rw])) {
2337 if (iocb->ki_flags & IOCB_NOWAIT) {
2338 iocb->ki_hint = hint;
2339 err = -EAGAIN;
2340 goto out;
2341 }
2342 down_read(&F2FS_I(inode)->dio_rwsem[rw]);
2343 }
2344
c8b8e32d 2345 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
82e0a5aa
CY		 2346 up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2347
2348 if (rw == WRITE) {
0cdd3195
HL		 2349 if (whint_mode == WHINT_MODE_OFF)
2350 iocb->ki_hint = hint;
b0af6d49
CY		 2351 if (err > 0) {
2352 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2353 err);
91942321 2354 set_inode_flag(inode, FI_UPDATE_WRITE);
b0af6d49 2355 } else if (err < 0) {
6bfc4919 2356 f2fs_write_failed(mapping, offset + count);
b0af6d49 2357 }
6bfc4919 2358 }
70407fad 2359
b91050a8 2360out:
5302fb00 2361 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
70407fad 2362
3aab8f82 2363 return err;
eb47b800
JK		 2364}
2365
487261f3
CY		 2366void f2fs_invalidate_page(struct page *page, unsigned int offset,
2367 unsigned int length)
eb47b800
JK		 2368{
2369 struct inode *inode = page->mapping->host;
487261f3 2370 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
a7ffdbe2 2371
487261f3 2372 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
09cbfeaf 2373 (offset % PAGE_SIZE || length != PAGE_SIZE))
a7ffdbe2
JK		 2374 return;
2375
487261f3 2376 if (PageDirty(page)) {
933439c8 2377 if (inode->i_ino == F2FS_META_INO(sbi)) {
487261f3 2378 dec_page_count(sbi, F2FS_DIRTY_META);
933439c8 2379 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
487261f3 2380 dec_page_count(sbi, F2FS_DIRTY_NODES);
933439c8 2381 } else {
487261f3 2382 inode_dec_dirty_pages(inode);
933439c8
CY		 2383 remove_dirty_inode(inode);
2384 }
487261f3 2385 }
decd36b6
CY		 2386
2387 /* This is atomic written page, keep Private */
2388 if (IS_ATOMIC_WRITTEN_PAGE(page))
8c242db9 2389 return drop_inmem_page(inode, page);
decd36b6 2390
23dc974e 2391 set_page_private(page, 0);
eb47b800
JK		 2392 ClearPagePrivate(page);
2393}
2394
487261f3 2395int f2fs_release_page(struct page *page, gfp_t wait)
eb47b800 2396{
f68daeeb
JK		 2397 /* If this is dirty page, keep PagePrivate */
2398 if (PageDirty(page))
2399 return 0;
2400
decd36b6
CY		 2401 /* This is atomic written page, keep Private */
2402 if (IS_ATOMIC_WRITTEN_PAGE(page))
2403 return 0;
2404
23dc974e 2405 set_page_private(page, 0);
eb47b800 2406 ClearPagePrivate(page);
c3850aa1 2407 return 1;
eb47b800
JK		 2408}
2409
fe76b796
JK		 2410/*
2411 * This was copied from __set_page_dirty_buffers which gives higher performance
2412 * in very high speed storages. (e.g., pmem)
2413 */
2414void f2fs_set_page_dirty_nobuffers(struct page *page)
2415{
2416 struct address_space *mapping = page->mapping;
2417 unsigned long flags;
2418
2419 if (unlikely(!mapping))
2420 return;
2421
2422 spin_lock(&mapping->private_lock);
2423 lock_page_memcg(page);
2424 SetPageDirty(page);
2425 spin_unlock(&mapping->private_lock);
2426
b93b0163 2427 xa_lock_irqsave(&mapping->i_pages, flags);
fe76b796
JK		 2428 WARN_ON_ONCE(!PageUptodate(page));
2429 account_page_dirtied(page, mapping);
b93b0163 2430 radix_tree_tag_set(&mapping->i_pages,
fe76b796 2431 page_index(page), PAGECACHE_TAG_DIRTY);
b93b0163 2432 xa_unlock_irqrestore(&mapping->i_pages, flags);
fe76b796
JK		 2433 unlock_page_memcg(page);
2434
2435 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2436 return;
2437}
2438
eb47b800
JK		 2439static int f2fs_set_data_page_dirty(struct page *page)
2440{
2441 struct address_space *mapping = page->mapping;
2442 struct inode *inode = mapping->host;
2443
26c6b887
JK		 2444 trace_f2fs_set_page_dirty(page, DATA);
2445
237c0790
JK		 2446 if (!PageUptodate(page))
2447 SetPageUptodate(page);
34ba94ba 2448
5fe45743 2449 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
decd36b6
CY		 2450 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2451 register_inmem_page(inode, page);
2452 return 1;
2453 }
2454 /*
2455 * Previously, this page has been registered, we just
2456 * return here.
2457 */
2458 return 0;
34ba94ba
JK		 2459 }
2460
eb47b800 2461 if (!PageDirty(page)) {
fe76b796 2462 f2fs_set_page_dirty_nobuffers(page);
a7ffdbe2 2463 update_dirty_page(inode, page);
eb47b800
JK		 2464 return 1;
2465 }
2466 return 0;
2467}
2468
c01e54b7
JK		 2469static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2470{
454ae7e5
CY		 2471 struct inode *inode = mapping->host;
2472
1d373a0e
JK		 2473 if (f2fs_has_inline_data(inode))
2474 return 0;
2475
2476 /* make sure allocating whole blocks */
2477 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2478 filemap_write_and_wait(mapping);
2479
e2b4e2bc 2480 return generic_block_bmap(mapping, block, get_data_block_bmap);
429511cd
CY		 2481}
2482
5b7a487c
WG		 2483#ifdef CONFIG_MIGRATION
2484#include <linux/migrate.h>
2485
2486int f2fs_migrate_page(struct address_space *mapping,
2487 struct page *newpage, struct page *page, enum migrate_mode mode)
2488{
2489 int rc, extra_count;
2490 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2491 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2492
2493 BUG_ON(PageWriteback(page));
2494
2495 /* migrating an atomic written page is safe with the inmem_lock hold */
ff1048e7
JK		 2496 if (atomic_written) {
2497 if (mode != MIGRATE_SYNC)
2498 return -EBUSY;
2499 if (!mutex_trylock(&fi->inmem_lock))
2500 return -EAGAIN;
2501 }
5b7a487c
WG		 2502
2503 /*
2504 * A reference is expected if PagePrivate set when move mapping,
2505 * however F2FS breaks this for maintaining dirty page counts when
2506 * truncating pages. So here adjusting the 'extra_count' make it work.
2507 */
2508 extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2509 rc = migrate_page_move_mapping(mapping, newpage,
2510 page, NULL, mode, extra_count);
2511 if (rc != MIGRATEPAGE_SUCCESS) {
2512 if (atomic_written)
2513 mutex_unlock(&fi->inmem_lock);
2514 return rc;
2515 }
2516
2517 if (atomic_written) {
2518 struct inmem_pages *cur;
2519 list_for_each_entry(cur, &fi->inmem_pages, list)
2520 if (cur->page == page) {
2521 cur->page = newpage;
2522 break;
2523 }
2524 mutex_unlock(&fi->inmem_lock);
2525 put_page(page);
2526 get_page(newpage);
2527 }
2528
2529 if (PagePrivate(page))
2530 SetPagePrivate(newpage);
2531 set_page_private(newpage, page_private(page));
2532
2916ecc0
JG		 2533 if (mode != MIGRATE_SYNC_NO_COPY)
2534 migrate_page_copy(newpage, page);
2535 else
2536 migrate_page_states(newpage, page);
5b7a487c
WG		 2537
2538 return MIGRATEPAGE_SUCCESS;
2539}
2540#endif
2541
eb47b800
JK		 2542const struct address_space_operations f2fs_dblock_aops = {
2543 .readpage = f2fs_read_data_page,
2544 .readpages = f2fs_read_data_pages,
2545 .writepage = f2fs_write_data_page,
2546 .writepages = f2fs_write_data_pages,
2547 .write_begin = f2fs_write_begin,
a1dd3c13 2548 .write_end = f2fs_write_end,
eb47b800 2549 .set_page_dirty = f2fs_set_data_page_dirty,
487261f3
CY		 2550 .invalidatepage = f2fs_invalidate_page,
2551 .releasepage = f2fs_release_page,
eb47b800 2552 .direct_IO = f2fs_direct_IO,
c01e54b7 2553 .bmap = f2fs_bmap,
5b7a487c
WG		 2554#ifdef CONFIG_MIGRATION
2555 .migratepage = f2fs_migrate_page,
2556#endif
eb47b800 2557};
Linux 6.x block layer and io_uring tree(s)