fs/f2fs/data.c

   1 /*
   2  * fs/f2fs/data.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/buffer_head.h>
  14 #include <linux/mpage.h>
  15 #include <linux/writeback.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/pagevec.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/bio.h>
  20 #include <linux/prefetch.h>
  21 #include <linux/uio.h>
  22 #include <linux/mm.h>
  23 #include <linux/memcontrol.h>
  24 #include <linux/cleancache.h>
  25 #include <linux/sched/signal.h>
  26
  27 #include "f2fs.h"
  28 #include "node.h"
  29 #include "segment.h"
  30 #include "trace.h"
  31 #include <trace/events/f2fs.h>
  32
  33 static bool __is_cp_guaranteed(struct page *page)
  34 {
  35         struct address_space *mapping = page->mapping;
  36         struct inode *inode;
  37         struct f2fs_sb_info *sbi;
  38
  39         if (!mapping)
  40                 return false;
  41
  42         inode = mapping->host;
  43         sbi = F2FS_I_SB(inode);
  44
  45         if (inode->i_ino == F2FS_META_INO(sbi) ||
  46                         inode->i_ino ==  F2FS_NODE_INO(sbi) ||
  47                         S_ISDIR(inode->i_mode) ||
  48                         is_cold_data(page))
  49                 return true;
  50         return false;
  51 }
  52
  53 static void f2fs_read_end_io(struct bio *bio)
  54 {
  55         struct bio_vec *bvec;
  56         int i;
  57
  58 #ifdef CONFIG_F2FS_FAULT_INJECTION
  59         if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) {
  60                 f2fs_show_injection_info(FAULT_IO);
  61                 bio->bi_status = BLK_STS_IOERR;
  62         }
  63 #endif
  64
  65         if (f2fs_bio_encrypted(bio)) {
  66                 if (bio->bi_status) {
  67                         fscrypt_release_ctx(bio->bi_private);
  68                 } else {
  69                         fscrypt_decrypt_bio_pages(bio->bi_private, bio);
  70                         return;
  71                 }
  72         }
  73
  74         bio_for_each_segment_all(bvec, bio, i) {
  75                 struct page *page = bvec->bv_page;
  76
  77                 if (!bio->bi_status) {
  78                         if (!PageUptodate(page))
  79                                 SetPageUptodate(page);
  80                 } else {
  81                         ClearPageUptodate(page);
  82                         SetPageError(page);
  83                 }
  84                 unlock_page(page);
  85         }
  86         bio_put(bio);
  87 }
  88
  89 static void f2fs_write_end_io(struct bio *bio)
  90 {
  91         struct f2fs_sb_info *sbi = bio->bi_private;
  92         struct bio_vec *bvec;
  93         int i;
  94
  95         bio_for_each_segment_all(bvec, bio, i) {
  96                 struct page *page = bvec->bv_page;
  97                 enum count_type type = WB_DATA_TYPE(page);
  98
  99                 if (IS_DUMMY_WRITTEN_PAGE(page)) {
 100                         set_page_private(page, (unsigned long)NULL);
 101                         ClearPagePrivate(page);
 102                         unlock_page(page);
 103                         mempool_free(page, sbi->write_io_dummy);
 104
 105                         if (unlikely(bio->bi_status))
 106                                 f2fs_stop_checkpoint(sbi, true);
 107                         continue;
 108                 }
 109
 110                 fscrypt_pullback_bio_page(&page, true);
 111
 112                 if (unlikely(bio->bi_status)) {
 113                         mapping_set_error(page->mapping, -EIO);
 114                         f2fs_stop_checkpoint(sbi, true);
 115                 }
 116                 dec_page_count(sbi, type);
 117                 clear_cold_data(page);
 118                 end_page_writeback(page);
 119         }
 120         if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
 121                                 wq_has_sleeper(&sbi->cp_wait))
 122                 wake_up(&sbi->cp_wait);
 123
 124         bio_put(bio);
 125 }
 126
 127 /*
 128  * Return true, if pre_bio's bdev is same as its target device.
 129  */
 130 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
 131                                 block_t blk_addr, struct bio *bio)
 132 {
 133         struct block_device *bdev = sbi->sb->s_bdev;
 134         int i;
 135
 136         for (i = 0; i < sbi->s_ndevs; i++) {
 137                 if (FDEV(i).start_blk <= blk_addr &&
 138                                         FDEV(i).end_blk >= blk_addr) {
 139                         blk_addr -= FDEV(i).start_blk;
 140                         bdev = FDEV(i).bdev;
 141                         break;
 142                 }
 143         }
 144         if (bio) {
 145                 bio_set_dev(bio, bdev);
 146                 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
 147         }
 148         return bdev;
 149 }
 150
 151 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
 152 {
 153         int i;
 154
 155         for (i = 0; i < sbi->s_ndevs; i++)
 156                 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
 157                         return i;
 158         return 0;
 159 }
 160
 161 static bool __same_bdev(struct f2fs_sb_info *sbi,
 162                                 block_t blk_addr, struct bio *bio)
 163 {
 164         struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
 165         return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
 166 }
 167
 168 /*
 169  * Low-level block read/write IO operations.
 170  */
 171 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
 172                                 int npages, bool is_read)
 173 {
 174         struct bio *bio;
 175
 176         bio = f2fs_bio_alloc(npages);
 177
 178         f2fs_target_device(sbi, blk_addr, bio);
 179         bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
 180         bio->bi_private = is_read ? NULL : sbi;
 181
 182         return bio;
 183 }
 184
 185 static inline void __submit_bio(struct f2fs_sb_info *sbi,
 186                                 struct bio *bio, enum page_type type)
 187 {
 188         if (!is_read_io(bio_op(bio))) {
 189                 unsigned int start;
 190
 191                 if (f2fs_sb_mounted_blkzoned(sbi->sb) &&
 192                         current->plug && (type == DATA || type == NODE))
 193                         blk_finish_plug(current->plug);
 194
 195                 if (type != DATA && type != NODE)
 196                         goto submit_io;
 197
 198                 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
 199                 start %= F2FS_IO_SIZE(sbi);
 200
 201                 if (start == 0)
 202                         goto submit_io;
 203
 204                 /* fill dummy pages */
 205                 for (; start < F2FS_IO_SIZE(sbi); start++) {
 206                         struct page *page =
 207                                 mempool_alloc(sbi->write_io_dummy,
 208                                         GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
 209                         f2fs_bug_on(sbi, !page);
 210
 211                         SetPagePrivate(page);
 212                         set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
 213                         lock_page(page);
 214                         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
 215                                 f2fs_bug_on(sbi, 1);
 216                 }
 217                 /*
 218                  * In the NODE case, we lose next block address chain. So, we
 219                  * need to do checkpoint in f2fs_sync_file.
 220                  */
 221                 if (type == NODE)
 222                         set_sbi_flag(sbi, SBI_NEED_CP);
 223         }
 224 submit_io:
 225         if (is_read_io(bio_op(bio)))
 226                 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
 227         else
 228                 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
 229         submit_bio(bio);
 230 }
 231
 232 static void __submit_merged_bio(struct f2fs_bio_info *io)
 233 {
 234         struct f2fs_io_info *fio = &io->fio;
 235
 236         if (!io->bio)
 237                 return;
 238
 239         bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
 240
 241         if (is_read_io(fio->op))
 242                 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
 243         else
 244                 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
 245
 246         __submit_bio(io->sbi, io->bio, fio->type);
 247         io->bio = NULL;
 248 }
 249
 250 static bool __has_merged_page(struct f2fs_bio_info *io,
 251                                 struct inode *inode, nid_t ino, pgoff_t idx)
 252 {
 253         struct bio_vec *bvec;
 254         struct page *target;
 255         int i;
 256
 257         if (!io->bio)
 258                 return false;
 259
 260         if (!inode && !ino)
 261                 return true;
 262
 263         bio_for_each_segment_all(bvec, io->bio, i) {
 264
 265                 if (bvec->bv_page->mapping)
 266                         target = bvec->bv_page;
 267                 else
 268                         target = fscrypt_control_page(bvec->bv_page);
 269
 270                 if (idx != target->index)
 271                         continue;
 272
 273                 if (inode && inode == target->mapping->host)
 274                         return true;
 275                 if (ino && ino == ino_of_node(target))
 276                         return true;
 277         }
 278
 279         return false;
 280 }
 281
 282 static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
 283                                 nid_t ino, pgoff_t idx, enum page_type type)
 284 {
 285         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 286         enum temp_type temp;
 287         struct f2fs_bio_info *io;
 288         bool ret = false;
 289
 290         for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
 291                 io = sbi->write_io[btype] + temp;
 292
 293                 down_read(&io->io_rwsem);
 294                 ret = __has_merged_page(io, inode, ino, idx);
 295                 up_read(&io->io_rwsem);
 296
 297                 /* TODO: use HOT temp only for meta pages now. */
 298                 if (ret || btype == META)
 299                         break;
 300         }
 301         return ret;
 302 }
 303
 304 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
 305                                 enum page_type type, enum temp_type temp)
 306 {
 307         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 308         struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
 309
 310         down_write(&io->io_rwsem);
 311
 312         /* change META to META_FLUSH in the checkpoint procedure */
 313         if (type >= META_FLUSH) {
 314                 io->fio.type = META_FLUSH;
 315                 io->fio.op = REQ_OP_WRITE;
 316                 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
 317                 if (!test_opt(sbi, NOBARRIER))
 318                         io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
 319         }
 320         __submit_merged_bio(io);
 321         up_write(&io->io_rwsem);
 322 }
 323
 324 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
 325                                 struct inode *inode, nid_t ino, pgoff_t idx,
 326                                 enum page_type type, bool force)
 327 {
 328         enum temp_type temp;
 329
 330         if (!force && !has_merged_page(sbi, inode, ino, idx, type))
 331                 return;
 332
 333         for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
 334
 335                 __f2fs_submit_merged_write(sbi, type, temp);
 336
 337                 /* TODO: use HOT temp only for meta pages now. */
 338                 if (type >= META)
 339                         break;
 340         }
 341 }
 342
 343 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
 344 {
 345         __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
 346 }
 347
 348 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
 349                                 struct inode *inode, nid_t ino, pgoff_t idx,
 350                                 enum page_type type)
 351 {
 352         __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
 353 }
 354
 355 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
 356 {
 357         f2fs_submit_merged_write(sbi, DATA);
 358         f2fs_submit_merged_write(sbi, NODE);
 359         f2fs_submit_merged_write(sbi, META);
 360 }
 361
 362 /*
 363  * Fill the locked page with data located in the block address.
 364  * A caller needs to unlock the page on failure.
 365  */
 366 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
 367 {
 368         struct bio *bio;
 369         struct page *page = fio->encrypted_page ?
 370                         fio->encrypted_page : fio->page;
 371
 372         trace_f2fs_submit_page_bio(page, fio);
 373         f2fs_trace_ios(fio, 0);
 374
 375         /* Allocate a new bio */
 376         bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->op));
 377
 378         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 379                 bio_put(bio);
 380                 return -EFAULT;
 381         }
 382         bio_set_op_attrs(bio, fio->op, fio->op_flags);
 383
 384         __submit_bio(fio->sbi, bio, fio->type);
 385
 386         if (!is_read_io(fio->op))
 387                 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
 388         return 0;
 389 }
 390
 391 int f2fs_submit_page_write(struct f2fs_io_info *fio)
 392 {
 393         struct f2fs_sb_info *sbi = fio->sbi;
 394         enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
 395         struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
 396         struct page *bio_page;
 397         int err = 0;
 398
 399         f2fs_bug_on(sbi, is_read_io(fio->op));
 400
 401         down_write(&io->io_rwsem);
 402 next:
 403         if (fio->in_list) {
 404                 spin_lock(&io->io_lock);
 405                 if (list_empty(&io->io_list)) {
 406                         spin_unlock(&io->io_lock);
 407                         goto out_fail;
 408                 }
 409                 fio = list_first_entry(&io->io_list,
 410                                                 struct f2fs_io_info, list);
 411                 list_del(&fio->list);
 412                 spin_unlock(&io->io_lock);
 413         }
 414
 415         if (fio->old_blkaddr != NEW_ADDR)
 416                 verify_block_addr(sbi, fio->old_blkaddr);
 417         verify_block_addr(sbi, fio->new_blkaddr);
 418
 419         bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
 420
 421         /* set submitted = 1 as a return value */
 422         fio->submitted = 1;
 423
 424         inc_page_count(sbi, WB_DATA_TYPE(bio_page));
 425
 426         if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
 427             (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
 428                         !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
 429                 __submit_merged_bio(io);
 430 alloc_new:
 431         if (io->bio == NULL) {
 432                 if ((fio->type == DATA || fio->type == NODE) &&
 433                                 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
 434                         err = -EAGAIN;
 435                         dec_page_count(sbi, WB_DATA_TYPE(bio_page));
 436                         goto out_fail;
 437                 }
 438                 io->bio = __bio_alloc(sbi, fio->new_blkaddr,
 439                                                 BIO_MAX_PAGES, false);
 440                 io->fio = *fio;
 441         }
 442
 443         if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
 444                 __submit_merged_bio(io);
 445                 goto alloc_new;
 446         }
 447
 448         io->last_block_in_bio = fio->new_blkaddr;
 449         f2fs_trace_ios(fio, 0);
 450
 451         trace_f2fs_submit_page_write(fio->page, fio);
 452
 453         if (fio->in_list)
 454                 goto next;
 455 out_fail:
 456         up_write(&io->io_rwsem);
 457         return err;
 458 }
 459
 460 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
 461                                                          unsigned nr_pages)
 462 {
 463         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 464         struct fscrypt_ctx *ctx = NULL;
 465         struct bio *bio;
 466
 467         if (f2fs_encrypted_file(inode)) {
 468                 ctx = fscrypt_get_ctx(inode, GFP_NOFS);
 469                 if (IS_ERR(ctx))
 470                         return ERR_CAST(ctx);
 471
 472                 /* wait the page to be moved by cleaning */
 473                 f2fs_wait_on_block_writeback(sbi, blkaddr);
 474         }
 475
 476         bio = bio_alloc(GFP_KERNEL, min_t(int, nr_pages, BIO_MAX_PAGES));
 477         if (!bio) {
 478                 if (ctx)
 479                         fscrypt_release_ctx(ctx);
 480                 return ERR_PTR(-ENOMEM);
 481         }
 482         f2fs_target_device(sbi, blkaddr, bio);
 483         bio->bi_end_io = f2fs_read_end_io;
 484         bio->bi_private = ctx;
 485         bio_set_op_attrs(bio, REQ_OP_READ, 0);
 486
 487         return bio;
 488 }
 489
 490 /* This can handle encryption stuffs */
 491 static int f2fs_submit_page_read(struct inode *inode, struct page *page,
 492                                                         block_t blkaddr)
 493 {
 494         struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1);
 495
 496         if (IS_ERR(bio))
 497                 return PTR_ERR(bio);
 498
 499         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 500                 bio_put(bio);
 501                 return -EFAULT;
 502         }
 503         __submit_bio(F2FS_I_SB(inode), bio, DATA);
 504         return 0;
 505 }
 506
 507 static void __set_data_blkaddr(struct dnode_of_data *dn)
 508 {
 509         struct f2fs_node *rn = F2FS_NODE(dn->node_page);
 510         __le32 *addr_array;
 511         int base = 0;
 512
 513         if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
 514                 base = get_extra_isize(dn->inode);
 515
 516         /* Get physical address of data block */
 517         addr_array = blkaddr_in_node(rn);
 518         addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
 519 }
 520
 521 /*
 522  * Lock ordering for the change of data block address:
 523  * ->data_page
 524  *  ->node_page
 525  *    update block addresses in the node page
 526  */
 527 void set_data_blkaddr(struct dnode_of_data *dn)
 528 {
 529         f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
 530         __set_data_blkaddr(dn);
 531         if (set_page_dirty(dn->node_page))
 532                 dn->node_changed = true;
 533 }
 534
 535 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
 536 {
 537         dn->data_blkaddr = blkaddr;
 538         set_data_blkaddr(dn);
 539         f2fs_update_extent_cache(dn);
 540 }
 541
 542 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
 543 int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
 544 {
 545         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 546         int err;
 547
 548         if (!count)
 549                 return 0;
 550
 551         if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
 552                 return -EPERM;
 553         if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
 554                 return err;
 555
 556         trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
 557                                                 dn->ofs_in_node, count);
 558
 559         f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
 560
 561         for (; count > 0; dn->ofs_in_node++) {
 562                 block_t blkaddr = datablock_addr(dn->inode,
 563                                         dn->node_page, dn->ofs_in_node);
 564                 if (blkaddr == NULL_ADDR) {
 565                         dn->data_blkaddr = NEW_ADDR;
 566                         __set_data_blkaddr(dn);
 567                         count--;
 568                 }
 569         }
 570
 571         if (set_page_dirty(dn->node_page))
 572                 dn->node_changed = true;
 573         return 0;
 574 }
 575
 576 /* Should keep dn->ofs_in_node unchanged */
 577 int reserve_new_block(struct dnode_of_data *dn)
 578 {
 579         unsigned int ofs_in_node = dn->ofs_in_node;
 580         int ret;
 581
 582         ret = reserve_new_blocks(dn, 1);
 583         dn->ofs_in_node = ofs_in_node;
 584         return ret;
 585 }
 586
 587 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
 588 {
 589         bool need_put = dn->inode_page ? false : true;
 590         int err;
 591
 592         err = get_dnode_of_data(dn, index, ALLOC_NODE);
 593         if (err)
 594                 return err;
 595
 596         if (dn->data_blkaddr == NULL_ADDR)
 597                 err = reserve_new_block(dn);
 598         if (err || need_put)
 599                 f2fs_put_dnode(dn);
 600         return err;
 601 }
 602
 603 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
 604 {
 605         struct extent_info ei  = {0,0,0};
 606         struct inode *inode = dn->inode;
 607
 608         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 609                 dn->data_blkaddr = ei.blk + index - ei.fofs;
 610                 return 0;
 611         }
 612
 613         return f2fs_reserve_block(dn, index);
 614 }
 615
 616 struct page *get_read_data_page(struct inode *inode, pgoff_t index,
 617                                                 int op_flags, bool for_write)
 618 {
 619         struct address_space *mapping = inode->i_mapping;
 620         struct dnode_of_data dn;
 621         struct page *page;
 622         struct extent_info ei = {0,0,0};
 623         int err;
 624
 625         page = f2fs_grab_cache_page(mapping, index, for_write);
 626         if (!page)
 627                 return ERR_PTR(-ENOMEM);
 628
 629         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 630                 dn.data_blkaddr = ei.blk + index - ei.fofs;
 631                 goto got_it;
 632         }
 633
 634         set_new_dnode(&dn, inode, NULL, NULL, 0);
 635         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
 636         if (err)
 637                 goto put_err;
 638         f2fs_put_dnode(&dn);
 639
 640         if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
 641                 err = -ENOENT;
 642                 goto put_err;
 643         }
 644 got_it:
 645         if (PageUptodate(page)) {
 646                 unlock_page(page);
 647                 return page;
 648         }
 649
 650         /*
 651          * A new dentry page is allocated but not able to be written, since its
 652          * new inode page couldn't be allocated due to -ENOSPC.
 653          * In such the case, its blkaddr can be remained as NEW_ADDR.
 654          * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
 655          */
 656         if (dn.data_blkaddr == NEW_ADDR) {
 657                 zero_user_segment(page, 0, PAGE_SIZE);
 658                 if (!PageUptodate(page))
 659                         SetPageUptodate(page);
 660                 unlock_page(page);
 661                 return page;
 662         }
 663
 664         err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
 665         if (err)
 666                 goto put_err;
 667         return page;
 668
 669 put_err:
 670         f2fs_put_page(page, 1);
 671         return ERR_PTR(err);
 672 }
 673
 674 struct page *find_data_page(struct inode *inode, pgoff_t index)
 675 {
 676         struct address_space *mapping = inode->i_mapping;
 677         struct page *page;
 678
 679         page = find_get_page(mapping, index);
 680         if (page && PageUptodate(page))
 681                 return page;
 682         f2fs_put_page(page, 0);
 683
 684         page = get_read_data_page(inode, index, 0, false);
 685         if (IS_ERR(page))
 686                 return page;
 687
 688         if (PageUptodate(page))
 689                 return page;
 690
 691         wait_on_page_locked(page);
 692         if (unlikely(!PageUptodate(page))) {
 693                 f2fs_put_page(page, 0);
 694                 return ERR_PTR(-EIO);
 695         }
 696         return page;
 697 }
 698
 699 /*
 700  * If it tries to access a hole, return an error.
 701  * Because, the callers, functions in dir.c and GC, should be able to know
 702  * whether this page exists or not.
 703  */
 704 struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
 705                                                         bool for_write)
 706 {
 707         struct address_space *mapping = inode->i_mapping;
 708         struct page *page;
 709 repeat:
 710         page = get_read_data_page(inode, index, 0, for_write);
 711         if (IS_ERR(page))
 712                 return page;
 713
 714         /* wait for read completion */
 715         lock_page(page);
 716         if (unlikely(page->mapping != mapping)) {
 717                 f2fs_put_page(page, 1);
 718                 goto repeat;
 719         }
 720         if (unlikely(!PageUptodate(page))) {
 721                 f2fs_put_page(page, 1);
 722                 return ERR_PTR(-EIO);
 723         }
 724         return page;
 725 }
 726
 727 /*
 728  * Caller ensures that this data page is never allocated.
 729  * A new zero-filled data page is allocated in the page cache.
 730  *
 731  * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 732  * f2fs_unlock_op().
 733  * Note that, ipage is set only by make_empty_dir, and if any error occur,
 734  * ipage should be released by this function.
 735  */
 736 struct page *get_new_data_page(struct inode *inode,
 737                 struct page *ipage, pgoff_t index, bool new_i_size)
 738 {
 739         struct address_space *mapping = inode->i_mapping;
 740         struct page *page;
 741         struct dnode_of_data dn;
 742         int err;
 743
 744         page = f2fs_grab_cache_page(mapping, index, true);
 745         if (!page) {
 746                 /*
 747                  * before exiting, we should make sure ipage will be released
 748                  * if any error occur.
 749                  */
 750                 f2fs_put_page(ipage, 1);
 751                 return ERR_PTR(-ENOMEM);
 752         }
 753
 754         set_new_dnode(&dn, inode, ipage, NULL, 0);
 755         err = f2fs_reserve_block(&dn, index);
 756         if (err) {
 757                 f2fs_put_page(page, 1);
 758                 return ERR_PTR(err);
 759         }
 760         if (!ipage)
 761                 f2fs_put_dnode(&dn);
 762
 763         if (PageUptodate(page))
 764                 goto got_it;
 765
 766         if (dn.data_blkaddr == NEW_ADDR) {
 767                 zero_user_segment(page, 0, PAGE_SIZE);
 768                 if (!PageUptodate(page))
 769                         SetPageUptodate(page);
 770         } else {
 771                 f2fs_put_page(page, 1);
 772
 773                 /* if ipage exists, blkaddr should be NEW_ADDR */
 774                 f2fs_bug_on(F2FS_I_SB(inode), ipage);
 775                 page = get_lock_data_page(inode, index, true);
 776                 if (IS_ERR(page))
 777                         return page;
 778         }
 779 got_it:
 780         if (new_i_size && i_size_read(inode) <
 781                                 ((loff_t)(index + 1) << PAGE_SHIFT))
 782                 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
 783         return page;
 784 }
 785
 786 static int __allocate_data_block(struct dnode_of_data *dn)
 787 {
 788         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 789         struct f2fs_summary sum;
 790         struct node_info ni;
 791         pgoff_t fofs;
 792         blkcnt_t count = 1;
 793         int err;
 794
 795         if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
 796                 return -EPERM;
 797
 798         dn->data_blkaddr = datablock_addr(dn->inode,
 799                                 dn->node_page, dn->ofs_in_node);
 800         if (dn->data_blkaddr == NEW_ADDR)
 801                 goto alloc;
 802
 803         if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
 804                 return err;
 805
 806 alloc:
 807         get_node_info(sbi, dn->nid, &ni);
 808         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 809
 810         allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
 811                                         &sum, CURSEG_WARM_DATA, NULL, false);
 812         set_data_blkaddr(dn);
 813
 814         /* update i_size */
 815         fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
 816                                                         dn->ofs_in_node;
 817         if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
 818                 f2fs_i_size_write(dn->inode,
 819                                 ((loff_t)(fofs + 1) << PAGE_SHIFT));
 820         return 0;
 821 }
 822
 823 static inline bool __force_buffered_io(struct inode *inode, int rw)
 824 {
 825         return (f2fs_encrypted_file(inode) ||
 826                         (rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) ||
 827                         F2FS_I_SB(inode)->s_ndevs);
 828 }
 829
 830 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
 831 {
 832         struct inode *inode = file_inode(iocb->ki_filp);
 833         struct f2fs_map_blocks map;
 834         int err = 0;
 835
 836         /* convert inline data for Direct I/O*/
 837         if (iocb->ki_flags & IOCB_DIRECT) {
 838                 err = f2fs_convert_inline_inode(inode);
 839                 if (err)
 840                         return err;
 841         }
 842
 843         if (is_inode_flag_set(inode, FI_NO_PREALLOC))
 844                 return 0;
 845
 846         map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
 847         map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
 848         if (map.m_len > map.m_lblk)
 849                 map.m_len -= map.m_lblk;
 850         else
 851                 map.m_len = 0;
 852
 853         map.m_next_pgofs = NULL;
 854
 855         if (iocb->ki_flags & IOCB_DIRECT)
 856                 return f2fs_map_blocks(inode, &map, 1,
 857                         __force_buffered_io(inode, WRITE) ?
 858                                 F2FS_GET_BLOCK_PRE_AIO :
 859                                 F2FS_GET_BLOCK_PRE_DIO);
 860         if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
 861                 err = f2fs_convert_inline_inode(inode);
 862                 if (err)
 863                         return err;
 864         }
 865         if (!f2fs_has_inline_data(inode))
 866                 return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
 867         return err;
 868 }
 869
 870 static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
 871 {
 872         if (flag == F2FS_GET_BLOCK_PRE_AIO) {
 873                 if (lock)
 874                         down_read(&sbi->node_change);
 875                 else
 876                         up_read(&sbi->node_change);
 877         } else {
 878                 if (lock)
 879                         f2fs_lock_op(sbi);
 880                 else
 881                         f2fs_unlock_op(sbi);
 882         }
 883 }
 884
 885 /*
 886  * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
 887  * f2fs_map_blocks structure.
 888  * If original data blocks are allocated, then give them to blockdev.
 889  * Otherwise,
 890  *     a. preallocate requested block addresses
 891  *     b. do not use extent cache for better performance
 892  *     c. give the block addresses to blockdev
 893  */
 894 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
 895                                                 int create, int flag)
 896 {
 897         unsigned int maxblocks = map->m_len;
 898         struct dnode_of_data dn;
 899         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 900         int mode = create ? ALLOC_NODE : LOOKUP_NODE;
 901         pgoff_t pgofs, end_offset, end;
 902         int err = 0, ofs = 1;
 903         unsigned int ofs_in_node, last_ofs_in_node;
 904         blkcnt_t prealloc;
 905         struct extent_info ei = {0,0,0};
 906         block_t blkaddr;
 907
 908         if (!maxblocks)
 909                 return 0;
 910
 911         map->m_len = 0;
 912         map->m_flags = 0;
 913
 914         /* it only supports block size == page size */
 915         pgofs = (pgoff_t)map->m_lblk;
 916         end = pgofs + maxblocks;
 917
 918         if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
 919                 map->m_pblk = ei.blk + pgofs - ei.fofs;
 920                 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
 921                 map->m_flags = F2FS_MAP_MAPPED;
 922                 goto out;
 923         }
 924
 925 next_dnode:
 926         if (create)
 927                 __do_map_lock(sbi, flag, true);
 928
 929         /* When reading holes, we need its node page */
 930         set_new_dnode(&dn, inode, NULL, NULL, 0);
 931         err = get_dnode_of_data(&dn, pgofs, mode);
 932         if (err) {
 933                 if (flag == F2FS_GET_BLOCK_BMAP)
 934                         map->m_pblk = 0;
 935                 if (err == -ENOENT) {
 936                         err = 0;
 937                         if (map->m_next_pgofs)
 938                                 *map->m_next_pgofs =
 939                                         get_next_page_offset(&dn, pgofs);
 940                 }
 941                 goto unlock_out;
 942         }
 943
 944         prealloc = 0;
 945         last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
 946         end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
 947
 948 next_block:
 949         blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
 950
 951         if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
 952                 if (create) {
 953                         if (unlikely(f2fs_cp_error(sbi))) {
 954                                 err = -EIO;
 955                                 goto sync_out;
 956                         }
 957                         if (flag == F2FS_GET_BLOCK_PRE_AIO) {
 958                                 if (blkaddr == NULL_ADDR) {
 959                                         prealloc++;
 960                                         last_ofs_in_node = dn.ofs_in_node;
 961                                 }
 962                         } else {
 963                                 err = __allocate_data_block(&dn);
 964                                 if (!err)
 965                                         set_inode_flag(inode, FI_APPEND_WRITE);
 966                         }
 967                         if (err)
 968                                 goto sync_out;
 969                         map->m_flags |= F2FS_MAP_NEW;
 970                         blkaddr = dn.data_blkaddr;
 971                 } else {
 972                         if (flag == F2FS_GET_BLOCK_BMAP) {
 973                                 map->m_pblk = 0;
 974                                 goto sync_out;
 975                         }
 976                         if (flag == F2FS_GET_BLOCK_FIEMAP &&
 977                                                 blkaddr == NULL_ADDR) {
 978                                 if (map->m_next_pgofs)
 979                                         *map->m_next_pgofs = pgofs + 1;
 980                         }
 981                         if (flag != F2FS_GET_BLOCK_FIEMAP ||
 982                                                 blkaddr != NEW_ADDR)
 983                                 goto sync_out;
 984                 }
 985         }
 986
 987         if (flag == F2FS_GET_BLOCK_PRE_AIO)
 988                 goto skip;
 989
 990         if (map->m_len == 0) {
 991                 /* preallocated unwritten block should be mapped for fiemap. */
 992                 if (blkaddr == NEW_ADDR)
 993                         map->m_flags |= F2FS_MAP_UNWRITTEN;
 994                 map->m_flags |= F2FS_MAP_MAPPED;
 995
 996                 map->m_pblk = blkaddr;
 997                 map->m_len = 1;
 998         } else if ((map->m_pblk != NEW_ADDR &&
 999                         blkaddr == (map->m_pblk + ofs)) ||
1000                         (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1001                         flag == F2FS_GET_BLOCK_PRE_DIO) {
1002                 ofs++;
1003                 map->m_len++;
1004         } else {
1005                 goto sync_out;
1006         }
1007
1008 skip:
1009         dn.ofs_in_node++;
1010         pgofs++;
1011
1012         /* preallocate blocks in batch for one dnode page */
1013         if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1014                         (pgofs == end || dn.ofs_in_node == end_offset)) {
1015
1016                 dn.ofs_in_node = ofs_in_node;
1017                 err = reserve_new_blocks(&dn, prealloc);
1018                 if (err)
1019                         goto sync_out;
1020
1021                 map->m_len += dn.ofs_in_node - ofs_in_node;
1022                 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1023                         err = -ENOSPC;
1024                         goto sync_out;
1025                 }
1026                 dn.ofs_in_node = end_offset;
1027         }
1028
1029         if (pgofs >= end)
1030                 goto sync_out;
1031         else if (dn.ofs_in_node < end_offset)
1032                 goto next_block;
1033
1034         f2fs_put_dnode(&dn);
1035
1036         if (create) {
1037                 __do_map_lock(sbi, flag, false);
1038                 f2fs_balance_fs(sbi, dn.node_changed);
1039         }
1040         goto next_dnode;
1041
1042 sync_out:
1043         f2fs_put_dnode(&dn);
1044 unlock_out:
1045         if (create) {
1046                 __do_map_lock(sbi, flag, false);
1047                 f2fs_balance_fs(sbi, dn.node_changed);
1048         }
1049 out:
1050         trace_f2fs_map_blocks(inode, map, err);
1051         return err;
1052 }
1053
1054 static int __get_data_block(struct inode *inode, sector_t iblock,
1055                         struct buffer_head *bh, int create, int flag,
1056                         pgoff_t *next_pgofs)
1057 {
1058         struct f2fs_map_blocks map;
1059         int err;
1060
1061         map.m_lblk = iblock;
1062         map.m_len = bh->b_size >> inode->i_blkbits;
1063         map.m_next_pgofs = next_pgofs;
1064
1065         err = f2fs_map_blocks(inode, &map, create, flag);
1066         if (!err) {
1067                 map_bh(bh, inode->i_sb, map.m_pblk);
1068                 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1069                 bh->b_size = (u64)map.m_len << inode->i_blkbits;
1070         }
1071         return err;
1072 }
1073
1074 static int get_data_block(struct inode *inode, sector_t iblock,
1075                         struct buffer_head *bh_result, int create, int flag,
1076                         pgoff_t *next_pgofs)
1077 {
1078         return __get_data_block(inode, iblock, bh_result, create,
1079                                                         flag, next_pgofs);
1080 }
1081
1082 static int get_data_block_dio(struct inode *inode, sector_t iblock,
1083                         struct buffer_head *bh_result, int create)
1084 {
1085         return __get_data_block(inode, iblock, bh_result, create,
1086                                                 F2FS_GET_BLOCK_DEFAULT, NULL);
1087 }
1088
1089 static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1090                         struct buffer_head *bh_result, int create)
1091 {
1092         /* Block number less than F2FS MAX BLOCKS */
1093         if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1094                 return -EFBIG;
1095
1096         return __get_data_block(inode, iblock, bh_result, create,
1097                                                 F2FS_GET_BLOCK_BMAP, NULL);
1098 }
1099
1100 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1101 {
1102         return (offset >> inode->i_blkbits);
1103 }
1104
1105 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1106 {
1107         return (blk << inode->i_blkbits);
1108 }
1109
1110 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1111                 u64 start, u64 len)
1112 {
1113         struct buffer_head map_bh;
1114         sector_t start_blk, last_blk;
1115         pgoff_t next_pgofs;
1116         u64 logical = 0, phys = 0, size = 0;
1117         u32 flags = 0;
1118         int ret = 0;
1119
1120         ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
1121         if (ret)
1122                 return ret;
1123
1124         if (f2fs_has_inline_data(inode)) {
1125                 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1126                 if (ret != -EAGAIN)
1127                         return ret;
1128         }
1129
1130         inode_lock(inode);
1131
1132         if (logical_to_blk(inode, len) == 0)
1133                 len = blk_to_logical(inode, 1);
1134
1135         start_blk = logical_to_blk(inode, start);
1136         last_blk = logical_to_blk(inode, start + len - 1);
1137
1138 next:
1139         memset(&map_bh, 0, sizeof(struct buffer_head));
1140         map_bh.b_size = len;
1141
1142         ret = get_data_block(inode, start_blk, &map_bh, 0,
1143                                         F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1144         if (ret)
1145                 goto out;
1146
1147         /* HOLE */
1148         if (!buffer_mapped(&map_bh)) {
1149                 start_blk = next_pgofs;
1150
1151                 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1152                                         F2FS_I_SB(inode)->max_file_blocks))
1153                         goto prep_next;
1154
1155                 flags |= FIEMAP_EXTENT_LAST;
1156         }
1157
1158         if (size) {
1159                 if (f2fs_encrypted_inode(inode))
1160                         flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1161
1162                 ret = fiemap_fill_next_extent(fieinfo, logical,
1163                                 phys, size, flags);
1164         }
1165
1166         if (start_blk > last_blk || ret)
1167                 goto out;
1168
1169         logical = blk_to_logical(inode, start_blk);
1170         phys = blk_to_logical(inode, map_bh.b_blocknr);
1171         size = map_bh.b_size;
1172         flags = 0;
1173         if (buffer_unwritten(&map_bh))
1174                 flags = FIEMAP_EXTENT_UNWRITTEN;
1175
1176         start_blk += logical_to_blk(inode, size);
1177
1178 prep_next:
1179         cond_resched();
1180         if (fatal_signal_pending(current))
1181                 ret = -EINTR;
1182         else
1183                 goto next;
1184 out:
1185         if (ret == 1)
1186                 ret = 0;
1187
1188         inode_unlock(inode);
1189         return ret;
1190 }
1191
1192 /*
1193  * This function was originally taken from fs/mpage.c, and customized for f2fs.
1194  * Major change was from block_size == page_size in f2fs by default.
1195  */
1196 static int f2fs_mpage_readpages(struct address_space *mapping,
1197                         struct list_head *pages, struct page *page,
1198                         unsigned nr_pages)
1199 {
1200         struct bio *bio = NULL;
1201         unsigned page_idx;
1202         sector_t last_block_in_bio = 0;
1203         struct inode *inode = mapping->host;
1204         const unsigned blkbits = inode->i_blkbits;
1205         const unsigned blocksize = 1 << blkbits;
1206         sector_t block_in_file;
1207         sector_t last_block;
1208         sector_t last_block_in_file;
1209         sector_t block_nr;
1210         struct f2fs_map_blocks map;
1211
1212         map.m_pblk = 0;
1213         map.m_lblk = 0;
1214         map.m_len = 0;
1215         map.m_flags = 0;
1216         map.m_next_pgofs = NULL;
1217
1218         for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
1219
1220                 if (pages) {
1221                         page = list_last_entry(pages, struct page, lru);
1222
1223                         prefetchw(&page->flags);
1224                         list_del(&page->lru);
1225                         if (add_to_page_cache_lru(page, mapping,
1226                                                   page->index,
1227                                                   readahead_gfp_mask(mapping)))
1228                                 goto next_page;
1229                 }
1230
1231                 block_in_file = (sector_t)page->index;
1232                 last_block = block_in_file + nr_pages;
1233                 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1234                                                                 blkbits;
1235                 if (last_block > last_block_in_file)
1236                         last_block = last_block_in_file;
1237
1238                 /*
1239                  * Map blocks using the previous result first.
1240                  */
1241                 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1242                                 block_in_file > map.m_lblk &&
1243                                 block_in_file < (map.m_lblk + map.m_len))
1244                         goto got_it;
1245
1246                 /*
1247                  * Then do more f2fs_map_blocks() calls until we are
1248                  * done with this page.
1249                  */
1250                 map.m_flags = 0;
1251
1252                 if (block_in_file < last_block) {
1253                         map.m_lblk = block_in_file;
1254                         map.m_len = last_block - block_in_file;
1255
1256                         if (f2fs_map_blocks(inode, &map, 0,
1257                                                 F2FS_GET_BLOCK_DEFAULT))
1258                                 goto set_error_page;
1259                 }
1260 got_it:
1261                 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1262                         block_nr = map.m_pblk + block_in_file - map.m_lblk;
1263                         SetPageMappedToDisk(page);
1264
1265                         if (!PageUptodate(page) && !cleancache_get_page(page)) {
1266                                 SetPageUptodate(page);
1267                                 goto confused;
1268                         }
1269                 } else {
1270                         zero_user_segment(page, 0, PAGE_SIZE);
1271                         if (!PageUptodate(page))
1272                                 SetPageUptodate(page);
1273                         unlock_page(page);
1274                         goto next_page;
1275                 }
1276
1277                 /*
1278                  * This page will go to BIO.  Do we need to send this
1279                  * BIO off first?
1280                  */
1281                 if (bio && (last_block_in_bio != block_nr - 1 ||
1282                         !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
1283 submit_and_realloc:
1284                         __submit_bio(F2FS_I_SB(inode), bio, DATA);
1285                         bio = NULL;
1286                 }
1287                 if (bio == NULL) {
1288                         bio = f2fs_grab_read_bio(inode, block_nr, nr_pages);
1289                         if (IS_ERR(bio)) {
1290                                 bio = NULL;
1291                                 goto set_error_page;
1292                         }
1293                 }
1294
1295                 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1296                         goto submit_and_realloc;
1297
1298                 last_block_in_bio = block_nr;
1299                 goto next_page;
1300 set_error_page:
1301                 SetPageError(page);
1302                 zero_user_segment(page, 0, PAGE_SIZE);
1303                 unlock_page(page);
1304                 goto next_page;
1305 confused:
1306                 if (bio) {
1307                         __submit_bio(F2FS_I_SB(inode), bio, DATA);
1308                         bio = NULL;
1309                 }
1310                 unlock_page(page);
1311 next_page:
1312                 if (pages)
1313                         put_page(page);
1314         }
1315         BUG_ON(pages && !list_empty(pages));
1316         if (bio)
1317                 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1318         return 0;
1319 }
1320
1321 static int f2fs_read_data_page(struct file *file, struct page *page)
1322 {
1323         struct inode *inode = page->mapping->host;
1324         int ret = -EAGAIN;
1325
1326         trace_f2fs_readpage(page, DATA);
1327
1328         /* If the file has inline data, try to read it directly */
1329         if (f2fs_has_inline_data(inode))
1330                 ret = f2fs_read_inline_data(inode, page);
1331         if (ret == -EAGAIN)
1332                 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1333         return ret;
1334 }
1335
1336 static int f2fs_read_data_pages(struct file *file,
1337                         struct address_space *mapping,
1338                         struct list_head *pages, unsigned nr_pages)
1339 {
1340         struct inode *inode = mapping->host;
1341         struct page *page = list_last_entry(pages, struct page, lru);
1342
1343         trace_f2fs_readpages(inode, page, nr_pages);
1344
1345         /* If the file has inline data, skip readpages */
1346         if (f2fs_has_inline_data(inode))
1347                 return 0;
1348
1349         return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1350 }
1351
1352 static int encrypt_one_page(struct f2fs_io_info *fio)
1353 {
1354         struct inode *inode = fio->page->mapping->host;
1355         gfp_t gfp_flags = GFP_NOFS;
1356
1357         if (!f2fs_encrypted_file(inode))
1358                 return 0;
1359
1360         /* wait for GCed encrypted page writeback */
1361         f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
1362
1363 retry_encrypt:
1364         fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1365                         PAGE_SIZE, 0, fio->page->index, gfp_flags);
1366         if (!IS_ERR(fio->encrypted_page))
1367                 return 0;
1368
1369         /* flush pending IOs and wait for a while in the ENOMEM case */
1370         if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1371                 f2fs_flush_merged_writes(fio->sbi);
1372                 congestion_wait(BLK_RW_ASYNC, HZ/50);
1373                 gfp_flags |= __GFP_NOFAIL;
1374                 goto retry_encrypt;
1375         }
1376         return PTR_ERR(fio->encrypted_page);
1377 }
1378
1379 static inline bool need_inplace_update(struct f2fs_io_info *fio)
1380 {
1381         struct inode *inode = fio->page->mapping->host;
1382
1383         if (S_ISDIR(inode->i_mode) || f2fs_is_atomic_file(inode))
1384                 return false;
1385         if (is_cold_data(fio->page))
1386                 return false;
1387         if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1388                 return false;
1389
1390         return need_inplace_update_policy(inode, fio);
1391 }
1392
1393 static inline bool valid_ipu_blkaddr(struct f2fs_io_info *fio)
1394 {
1395         if (fio->old_blkaddr == NEW_ADDR)
1396                 return false;
1397         if (fio->old_blkaddr == NULL_ADDR)
1398                 return false;
1399         return true;
1400 }
1401
1402 int do_write_data_page(struct f2fs_io_info *fio)
1403 {
1404         struct page *page = fio->page;
1405         struct inode *inode = page->mapping->host;
1406         struct dnode_of_data dn;
1407         struct extent_info ei = {0,0,0};
1408         bool ipu_force = false;
1409         int err = 0;
1410
1411         set_new_dnode(&dn, inode, NULL, NULL, 0);
1412         if (need_inplace_update(fio) &&
1413                         f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1414                 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
1415
1416                 if (valid_ipu_blkaddr(fio)) {
1417                         ipu_force = true;
1418                         fio->need_lock = LOCK_DONE;
1419                         goto got_it;
1420                 }
1421         }
1422
1423         /* Deadlock due to between page->lock and f2fs_lock_op */
1424         if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1425                 return -EAGAIN;
1426
1427         err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1428         if (err)
1429                 goto out;
1430
1431         fio->old_blkaddr = dn.data_blkaddr;
1432
1433         /* This page is already truncated */
1434         if (fio->old_blkaddr == NULL_ADDR) {
1435                 ClearPageUptodate(page);
1436                 goto out_writepage;
1437         }
1438 got_it:
1439         /*
1440          * If current allocation needs SSR,
1441          * it had better in-place writes for updated data.
1442          */
1443         if (ipu_force || (valid_ipu_blkaddr(fio) && need_inplace_update(fio))) {
1444                 err = encrypt_one_page(fio);
1445                 if (err)
1446                         goto out_writepage;
1447
1448                 set_page_writeback(page);
1449                 f2fs_put_dnode(&dn);
1450                 if (fio->need_lock == LOCK_REQ)
1451                         f2fs_unlock_op(fio->sbi);
1452                 err = rewrite_data_page(fio);
1453                 trace_f2fs_do_write_data_page(fio->page, IPU);
1454                 set_inode_flag(inode, FI_UPDATE_WRITE);
1455                 return err;
1456         }
1457
1458         if (fio->need_lock == LOCK_RETRY) {
1459                 if (!f2fs_trylock_op(fio->sbi)) {
1460                         err = -EAGAIN;
1461                         goto out_writepage;
1462                 }
1463                 fio->need_lock = LOCK_REQ;
1464         }
1465
1466         err = encrypt_one_page(fio);
1467         if (err)
1468                 goto out_writepage;
1469
1470         set_page_writeback(page);
1471
1472         /* LFS mode write path */
1473         write_data_page(&dn, fio);
1474         trace_f2fs_do_write_data_page(page, OPU);
1475         set_inode_flag(inode, FI_APPEND_WRITE);
1476         if (page->index == 0)
1477                 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1478 out_writepage:
1479         f2fs_put_dnode(&dn);
1480 out:
1481         if (fio->need_lock == LOCK_REQ)
1482                 f2fs_unlock_op(fio->sbi);
1483         return err;
1484 }
1485
1486 static int __write_data_page(struct page *page, bool *submitted,
1487                                 struct writeback_control *wbc,
1488                                 enum iostat_type io_type)
1489 {
1490         struct inode *inode = page->mapping->host;
1491         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1492         loff_t i_size = i_size_read(inode);
1493         const pgoff_t end_index = ((unsigned long long) i_size)
1494                                                         >> PAGE_SHIFT;
1495         loff_t psize = (page->index + 1) << PAGE_SHIFT;
1496         unsigned offset = 0;
1497         bool need_balance_fs = false;
1498         int err = 0;
1499         struct f2fs_io_info fio = {
1500                 .sbi = sbi,
1501                 .type = DATA,
1502                 .op = REQ_OP_WRITE,
1503                 .op_flags = wbc_to_write_flags(wbc),
1504                 .old_blkaddr = NULL_ADDR,
1505                 .page = page,
1506                 .encrypted_page = NULL,
1507                 .submitted = false,
1508                 .need_lock = LOCK_RETRY,
1509                 .io_type = io_type,
1510         };
1511
1512         trace_f2fs_writepage(page, DATA);
1513
1514         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1515                 goto redirty_out;
1516
1517         if (page->index < end_index)
1518                 goto write;
1519
1520         /*
1521          * If the offset is out-of-range of file size,
1522          * this page does not have to be written to disk.
1523          */
1524         offset = i_size & (PAGE_SIZE - 1);
1525         if ((page->index >= end_index + 1) || !offset)
1526                 goto out;
1527
1528         zero_user_segment(page, offset, PAGE_SIZE);
1529 write:
1530         if (f2fs_is_drop_cache(inode))
1531                 goto out;
1532         /* we should not write 0'th page having journal header */
1533         if (f2fs_is_volatile_file(inode) && (!page->index ||
1534                         (!wbc->for_reclaim &&
1535                         available_free_memory(sbi, BASE_CHECK))))
1536                 goto redirty_out;
1537
1538         /* we should bypass data pages to proceed the kworkder jobs */
1539         if (unlikely(f2fs_cp_error(sbi))) {
1540                 mapping_set_error(page->mapping, -EIO);
1541                 goto out;
1542         }
1543
1544         /* Dentry blocks are controlled by checkpoint */
1545         if (S_ISDIR(inode->i_mode)) {
1546                 fio.need_lock = LOCK_DONE;
1547                 err = do_write_data_page(&fio);
1548                 goto done;
1549         }
1550
1551         if (!wbc->for_reclaim)
1552                 need_balance_fs = true;
1553         else if (has_not_enough_free_secs(sbi, 0, 0))
1554                 goto redirty_out;
1555         else
1556                 set_inode_flag(inode, FI_HOT_DATA);
1557
1558         err = -EAGAIN;
1559         if (f2fs_has_inline_data(inode)) {
1560                 err = f2fs_write_inline_data(inode, page);
1561                 if (!err)
1562                         goto out;
1563         }
1564
1565         if (err == -EAGAIN) {
1566                 err = do_write_data_page(&fio);
1567                 if (err == -EAGAIN) {
1568                         fio.need_lock = LOCK_REQ;
1569                         err = do_write_data_page(&fio);
1570                 }
1571         }
1572         if (F2FS_I(inode)->last_disk_size < psize)
1573                 F2FS_I(inode)->last_disk_size = psize;
1574
1575 done:
1576         if (err && err != -ENOENT)
1577                 goto redirty_out;
1578
1579 out:
1580         inode_dec_dirty_pages(inode);
1581         if (err)
1582                 ClearPageUptodate(page);
1583
1584         if (wbc->for_reclaim) {
1585                 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
1586                 clear_inode_flag(inode, FI_HOT_DATA);
1587                 remove_dirty_inode(inode);
1588                 submitted = NULL;
1589         }
1590
1591         unlock_page(page);
1592         if (!S_ISDIR(inode->i_mode))
1593                 f2fs_balance_fs(sbi, need_balance_fs);
1594
1595         if (unlikely(f2fs_cp_error(sbi))) {
1596                 f2fs_submit_merged_write(sbi, DATA);
1597                 submitted = NULL;
1598         }
1599
1600         if (submitted)
1601                 *submitted = fio.submitted;
1602
1603         return 0;
1604
1605 redirty_out:
1606         redirty_page_for_writepage(wbc, page);
1607         if (!err)
1608                 return AOP_WRITEPAGE_ACTIVATE;
1609         unlock_page(page);
1610         return err;
1611 }
1612
1613 static int f2fs_write_data_page(struct page *page,
1614                                         struct writeback_control *wbc)
1615 {
1616         return __write_data_page(page, NULL, wbc, FS_DATA_IO);
1617 }
1618
1619 /*
1620  * This function was copied from write_cche_pages from mm/page-writeback.c.
1621  * The major change is making write step of cold data page separately from
1622  * warm/hot data page.
1623  */
1624 static int f2fs_write_cache_pages(struct address_space *mapping,
1625                                         struct writeback_control *wbc,
1626                                         enum iostat_type io_type)
1627 {
1628         int ret = 0;
1629         int done = 0;
1630         struct pagevec pvec;
1631         int nr_pages;
1632         pgoff_t uninitialized_var(writeback_index);
1633         pgoff_t index;
1634         pgoff_t end;            /* Inclusive */
1635         pgoff_t done_index;
1636         pgoff_t last_idx = ULONG_MAX;
1637         int cycled;
1638         int range_whole = 0;
1639         int tag;
1640
1641         pagevec_init(&pvec, 0);
1642
1643         if (get_dirty_pages(mapping->host) <=
1644                                 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
1645                 set_inode_flag(mapping->host, FI_HOT_DATA);
1646         else
1647                 clear_inode_flag(mapping->host, FI_HOT_DATA);
1648
1649         if (wbc->range_cyclic) {
1650                 writeback_index = mapping->writeback_index; /* prev offset */
1651                 index = writeback_index;
1652                 if (index == 0)
1653                         cycled = 1;
1654                 else
1655                         cycled = 0;
1656                 end = -1;
1657         } else {
1658                 index = wbc->range_start >> PAGE_SHIFT;
1659                 end = wbc->range_end >> PAGE_SHIFT;
1660                 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1661                         range_whole = 1;
1662                 cycled = 1; /* ignore range_cyclic tests */
1663         }
1664         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1665                 tag = PAGECACHE_TAG_TOWRITE;
1666         else
1667                 tag = PAGECACHE_TAG_DIRTY;
1668 retry:
1669         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1670                 tag_pages_for_writeback(mapping, index, end);
1671         done_index = index;
1672         while (!done && (index <= end)) {
1673                 int i;
1674
1675                 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1676                               min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
1677                 if (nr_pages == 0)
1678                         break;
1679
1680                 for (i = 0; i < nr_pages; i++) {
1681                         struct page *page = pvec.pages[i];
1682                         bool submitted = false;
1683
1684                         if (page->index > end) {
1685                                 done = 1;
1686                                 break;
1687                         }
1688
1689                         done_index = page->index;
1690 retry_write:
1691                         lock_page(page);
1692
1693                         if (unlikely(page->mapping != mapping)) {
1694 continue_unlock:
1695                                 unlock_page(page);
1696                                 continue;
1697                         }
1698
1699                         if (!PageDirty(page)) {
1700                                 /* someone wrote it for us */
1701                                 goto continue_unlock;
1702                         }
1703
1704                         if (PageWriteback(page)) {
1705                                 if (wbc->sync_mode != WB_SYNC_NONE)
1706                                         f2fs_wait_on_page_writeback(page,
1707                                                                 DATA, true);
1708                                 else
1709                                         goto continue_unlock;
1710                         }
1711
1712                         BUG_ON(PageWriteback(page));
1713                         if (!clear_page_dirty_for_io(page))
1714                                 goto continue_unlock;
1715
1716                         ret = __write_data_page(page, &submitted, wbc, io_type);
1717                         if (unlikely(ret)) {
1718                                 /*
1719                                  * keep nr_to_write, since vfs uses this to
1720                                  * get # of written pages.
1721                                  */
1722                                 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1723                                         unlock_page(page);
1724                                         ret = 0;
1725                                         continue;
1726                                 } else if (ret == -EAGAIN) {
1727                                         ret = 0;
1728                                         if (wbc->sync_mode == WB_SYNC_ALL) {
1729                                                 cond_resched();
1730                                                 congestion_wait(BLK_RW_ASYNC,
1731                                                                         HZ/50);
1732                                                 goto retry_write;
1733                                         }
1734                                         continue;
1735                                 }
1736                                 done_index = page->index + 1;
1737                                 done = 1;
1738                                 break;
1739                         } else if (submitted) {
1740                                 last_idx = page->index;
1741                         }
1742
1743                         /* give a priority to WB_SYNC threads */
1744                         if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
1745                                         --wbc->nr_to_write <= 0) &&
1746                                         wbc->sync_mode == WB_SYNC_NONE) {
1747                                 done = 1;
1748                                 break;
1749                         }
1750                 }
1751                 pagevec_release(&pvec);
1752                 cond_resched();
1753         }
1754
1755         if (!cycled && !done) {
1756                 cycled = 1;
1757                 index = 0;
1758                 end = writeback_index - 1;
1759                 goto retry;
1760         }
1761         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1762                 mapping->writeback_index = done_index;
1763
1764         if (last_idx != ULONG_MAX)
1765                 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
1766                                                 0, last_idx, DATA);
1767
1768         return ret;
1769 }
1770
1771 int __f2fs_write_data_pages(struct address_space *mapping,
1772                                                 struct writeback_control *wbc,
1773                                                 enum iostat_type io_type)
1774 {
1775         struct inode *inode = mapping->host;
1776         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1777         struct blk_plug plug;
1778         int ret;
1779
1780         /* deal with chardevs and other special file */
1781         if (!mapping->a_ops->writepage)
1782                 return 0;
1783
1784         /* skip writing if there is no dirty page in this inode */
1785         if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
1786                 return 0;
1787
1788         /* during POR, we don't need to trigger writepage at all. */
1789         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1790                 goto skip_write;
1791
1792         if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
1793                         get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
1794                         available_free_memory(sbi, DIRTY_DENTS))
1795                 goto skip_write;
1796
1797         /* skip writing during file defragment */
1798         if (is_inode_flag_set(inode, FI_DO_DEFRAG))
1799                 goto skip_write;
1800
1801         trace_f2fs_writepages(mapping->host, wbc, DATA);
1802
1803         /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
1804         if (wbc->sync_mode == WB_SYNC_ALL)
1805                 atomic_inc(&sbi->wb_sync_req);
1806         else if (atomic_read(&sbi->wb_sync_req))
1807                 goto skip_write;
1808
1809         blk_start_plug(&plug);
1810         ret = f2fs_write_cache_pages(mapping, wbc, io_type);
1811         blk_finish_plug(&plug);
1812
1813         if (wbc->sync_mode == WB_SYNC_ALL)
1814                 atomic_dec(&sbi->wb_sync_req);
1815         /*
1816          * if some pages were truncated, we cannot guarantee its mapping->host
1817          * to detect pending bios.
1818          */
1819
1820         remove_dirty_inode(inode);
1821         return ret;
1822
1823 skip_write:
1824         wbc->pages_skipped += get_dirty_pages(inode);
1825         trace_f2fs_writepages(mapping->host, wbc, DATA);
1826         return 0;
1827 }
1828
1829 static int f2fs_write_data_pages(struct address_space *mapping,
1830                             struct writeback_control *wbc)
1831 {
1832         struct inode *inode = mapping->host;
1833
1834         return __f2fs_write_data_pages(mapping, wbc,
1835                         F2FS_I(inode)->cp_task == current ?
1836                         FS_CP_DATA_IO : FS_DATA_IO);
1837 }
1838
1839 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
1840 {
1841         struct inode *inode = mapping->host;
1842         loff_t i_size = i_size_read(inode);
1843
1844         if (to > i_size) {
1845                 down_write(&F2FS_I(inode)->i_mmap_sem);
1846                 truncate_pagecache(inode, i_size);
1847                 truncate_blocks(inode, i_size, true);
1848                 up_write(&F2FS_I(inode)->i_mmap_sem);
1849         }
1850 }
1851
1852 static int prepare_write_begin(struct f2fs_sb_info *sbi,
1853                         struct page *page, loff_t pos, unsigned len,
1854                         block_t *blk_addr, bool *node_changed)
1855 {
1856         struct inode *inode = page->mapping->host;
1857         pgoff_t index = page->index;
1858         struct dnode_of_data dn;
1859         struct page *ipage;
1860         bool locked = false;
1861         struct extent_info ei = {0,0,0};
1862         int err = 0;
1863
1864         /*
1865          * we already allocated all the blocks, so we don't need to get
1866          * the block addresses when there is no need to fill the page.
1867          */
1868         if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
1869                         !is_inode_flag_set(inode, FI_NO_PREALLOC))
1870                 return 0;
1871
1872         if (f2fs_has_inline_data(inode) ||
1873                         (pos & PAGE_MASK) >= i_size_read(inode)) {
1874                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
1875                 locked = true;
1876         }
1877 restart:
1878         /* check inline_data */
1879         ipage = get_node_page(sbi, inode->i_ino);
1880         if (IS_ERR(ipage)) {
1881                 err = PTR_ERR(ipage);
1882                 goto unlock_out;
1883         }
1884
1885         set_new_dnode(&dn, inode, ipage, ipage, 0);
1886
1887         if (f2fs_has_inline_data(inode)) {
1888                 if (pos + len <= MAX_INLINE_DATA(inode)) {
1889                         read_inline_data(page, ipage);
1890                         set_inode_flag(inode, FI_DATA_EXIST);
1891                         if (inode->i_nlink)
1892                                 set_inline_node(ipage);
1893                 } else {
1894                         err = f2fs_convert_inline_page(&dn, page);
1895                         if (err)
1896                                 goto out;
1897                         if (dn.data_blkaddr == NULL_ADDR)
1898                                 err = f2fs_get_block(&dn, index);
1899                 }
1900         } else if (locked) {
1901                 err = f2fs_get_block(&dn, index);
1902         } else {
1903                 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1904                         dn.data_blkaddr = ei.blk + index - ei.fofs;
1905                 } else {
1906                         /* hole case */
1907                         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
1908                         if (err || dn.data_blkaddr == NULL_ADDR) {
1909                                 f2fs_put_dnode(&dn);
1910                                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
1911                                                                 true);
1912                                 locked = true;
1913                                 goto restart;
1914                         }
1915                 }
1916         }
1917
1918         /* convert_inline_page can make node_changed */
1919         *blk_addr = dn.data_blkaddr;
1920         *node_changed = dn.node_changed;
1921 out:
1922         f2fs_put_dnode(&dn);
1923 unlock_out:
1924         if (locked)
1925                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
1926         return err;
1927 }
1928
1929 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
1930                 loff_t pos, unsigned len, unsigned flags,
1931                 struct page **pagep, void **fsdata)
1932 {
1933         struct inode *inode = mapping->host;
1934         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1935         struct page *page = NULL;
1936         pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
1937         bool need_balance = false;
1938         block_t blkaddr = NULL_ADDR;
1939         int err = 0;
1940
1941         trace_f2fs_write_begin(inode, pos, len, flags);
1942
1943         /*
1944          * We should check this at this moment to avoid deadlock on inode page
1945          * and #0 page. The locking rule for inline_data conversion should be:
1946          * lock_page(page #0) -> lock_page(inode_page)
1947          */
1948         if (index != 0) {
1949                 err = f2fs_convert_inline_inode(inode);
1950                 if (err)
1951                         goto fail;
1952         }
1953 repeat:
1954         /*
1955          * Do not use grab_cache_page_write_begin() to avoid deadlock due to
1956          * wait_for_stable_page. Will wait that below with our IO control.
1957          */
1958         page = pagecache_get_page(mapping, index,
1959                                 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
1960         if (!page) {
1961                 err = -ENOMEM;
1962                 goto fail;
1963         }
1964
1965         *pagep = page;
1966
1967         err = prepare_write_begin(sbi, page, pos, len,
1968                                         &blkaddr, &need_balance);
1969         if (err)
1970                 goto fail;
1971
1972         if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
1973                 unlock_page(page);
1974                 f2fs_balance_fs(sbi, true);
1975                 lock_page(page);
1976                 if (page->mapping != mapping) {
1977                         /* The page got truncated from under us */
1978                         f2fs_put_page(page, 1);
1979                         goto repeat;
1980                 }
1981         }
1982
1983         f2fs_wait_on_page_writeback(page, DATA, false);
1984
1985         /* wait for GCed encrypted page writeback */
1986         if (f2fs_encrypted_file(inode))
1987                 f2fs_wait_on_block_writeback(sbi, blkaddr);
1988
1989         if (len == PAGE_SIZE || PageUptodate(page))
1990                 return 0;
1991
1992         if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
1993                 zero_user_segment(page, len, PAGE_SIZE);
1994                 return 0;
1995         }
1996
1997         if (blkaddr == NEW_ADDR) {
1998                 zero_user_segment(page, 0, PAGE_SIZE);
1999                 SetPageUptodate(page);
2000         } else {
2001                 err = f2fs_submit_page_read(inode, page, blkaddr);
2002                 if (err)
2003                         goto fail;
2004
2005                 lock_page(page);
2006                 if (unlikely(page->mapping != mapping)) {
2007                         f2fs_put_page(page, 1);
2008                         goto repeat;
2009                 }
2010                 if (unlikely(!PageUptodate(page))) {
2011                         err = -EIO;
2012                         goto fail;
2013                 }
2014         }
2015         return 0;
2016
2017 fail:
2018         f2fs_put_page(page, 1);
2019         f2fs_write_failed(mapping, pos + len);
2020         return err;
2021 }
2022
2023 static int f2fs_write_end(struct file *file,
2024                         struct address_space *mapping,
2025                         loff_t pos, unsigned len, unsigned copied,
2026                         struct page *page, void *fsdata)
2027 {
2028         struct inode *inode = page->mapping->host;
2029
2030         trace_f2fs_write_end(inode, pos, len, copied);
2031
2032         /*
2033          * This should be come from len == PAGE_SIZE, and we expect copied
2034          * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2035          * let generic_perform_write() try to copy data again through copied=0.
2036          */
2037         if (!PageUptodate(page)) {
2038                 if (unlikely(copied != len))
2039                         copied = 0;
2040                 else
2041                         SetPageUptodate(page);
2042         }
2043         if (!copied)
2044                 goto unlock_out;
2045
2046         set_page_dirty(page);
2047
2048         if (pos + copied > i_size_read(inode))
2049                 f2fs_i_size_write(inode, pos + copied);
2050 unlock_out:
2051         f2fs_put_page(page, 1);
2052         f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
2053         return copied;
2054 }
2055
2056 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2057                            loff_t offset)
2058 {
2059         unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
2060
2061         if (offset & blocksize_mask)
2062                 return -EINVAL;
2063
2064         if (iov_iter_alignment(iter) & blocksize_mask)
2065                 return -EINVAL;
2066
2067         return 0;
2068 }
2069
2070 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2071 {
2072         struct address_space *mapping = iocb->ki_filp->f_mapping;
2073         struct inode *inode = mapping->host;
2074         size_t count = iov_iter_count(iter);
2075         loff_t offset = iocb->ki_pos;
2076         int rw = iov_iter_rw(iter);
2077         int err;
2078
2079         err = check_direct_IO(inode, iter, offset);
2080         if (err)
2081                 return err;
2082
2083         if (__force_buffered_io(inode, rw))
2084                 return 0;
2085
2086         trace_f2fs_direct_IO_enter(inode, offset, count, rw);
2087
2088         down_read(&F2FS_I(inode)->dio_rwsem[rw]);
2089         err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
2090         up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2091
2092         if (rw == WRITE) {
2093                 if (err > 0) {
2094                         f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2095                                                                         err);
2096                         set_inode_flag(inode, FI_UPDATE_WRITE);
2097                 } else if (err < 0) {
2098                         f2fs_write_failed(mapping, offset + count);
2099                 }
2100         }
2101
2102         trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
2103
2104         return err;
2105 }
2106
2107 void f2fs_invalidate_page(struct page *page, unsigned int offset,
2108                                                         unsigned int length)
2109 {
2110         struct inode *inode = page->mapping->host;
2111         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2112
2113         if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
2114                 (offset % PAGE_SIZE || length != PAGE_SIZE))
2115                 return;
2116
2117         if (PageDirty(page)) {
2118                 if (inode->i_ino == F2FS_META_INO(sbi)) {
2119                         dec_page_count(sbi, F2FS_DIRTY_META);
2120                 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
2121                         dec_page_count(sbi, F2FS_DIRTY_NODES);
2122                 } else {
2123                         inode_dec_dirty_pages(inode);
2124                         remove_dirty_inode(inode);
2125                 }
2126         }
2127
2128         /* This is atomic written page, keep Private */
2129         if (IS_ATOMIC_WRITTEN_PAGE(page))
2130                 return drop_inmem_page(inode, page);
2131
2132         set_page_private(page, 0);
2133         ClearPagePrivate(page);
2134 }
2135
2136 int f2fs_release_page(struct page *page, gfp_t wait)
2137 {
2138         /* If this is dirty page, keep PagePrivate */
2139         if (PageDirty(page))
2140                 return 0;
2141
2142         /* This is atomic written page, keep Private */
2143         if (IS_ATOMIC_WRITTEN_PAGE(page))
2144                 return 0;
2145
2146         set_page_private(page, 0);
2147         ClearPagePrivate(page);
2148         return 1;
2149 }
2150
2151 /*
2152  * This was copied from __set_page_dirty_buffers which gives higher performance
2153  * in very high speed storages. (e.g., pmem)
2154  */
2155 void f2fs_set_page_dirty_nobuffers(struct page *page)
2156 {
2157         struct address_space *mapping = page->mapping;
2158         unsigned long flags;
2159
2160         if (unlikely(!mapping))
2161                 return;
2162
2163         spin_lock(&mapping->private_lock);
2164         lock_page_memcg(page);
2165         SetPageDirty(page);
2166         spin_unlock(&mapping->private_lock);
2167
2168         spin_lock_irqsave(&mapping->tree_lock, flags);
2169         WARN_ON_ONCE(!PageUptodate(page));
2170         account_page_dirtied(page, mapping);
2171         radix_tree_tag_set(&mapping->page_tree,
2172                         page_index(page), PAGECACHE_TAG_DIRTY);
2173         spin_unlock_irqrestore(&mapping->tree_lock, flags);
2174         unlock_page_memcg(page);
2175
2176         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2177         return;
2178 }
2179
2180 static int f2fs_set_data_page_dirty(struct page *page)
2181 {
2182         struct address_space *mapping = page->mapping;
2183         struct inode *inode = mapping->host;
2184
2185         trace_f2fs_set_page_dirty(page, DATA);
2186
2187         if (!PageUptodate(page))
2188                 SetPageUptodate(page);
2189
2190         if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
2191                 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2192                         register_inmem_page(inode, page);
2193                         return 1;
2194                 }
2195                 /*
2196                  * Previously, this page has been registered, we just
2197                  * return here.
2198                  */
2199                 return 0;
2200         }
2201
2202         if (!PageDirty(page)) {
2203                 f2fs_set_page_dirty_nobuffers(page);
2204                 update_dirty_page(inode, page);
2205                 return 1;
2206         }
2207         return 0;
2208 }
2209
2210 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2211 {
2212         struct inode *inode = mapping->host;
2213
2214         if (f2fs_has_inline_data(inode))
2215                 return 0;
2216
2217         /* make sure allocating whole blocks */
2218         if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2219                 filemap_write_and_wait(mapping);
2220
2221         return generic_block_bmap(mapping, block, get_data_block_bmap);
2222 }
2223
2224 #ifdef CONFIG_MIGRATION
2225 #include <linux/migrate.h>
2226
2227 int f2fs_migrate_page(struct address_space *mapping,
2228                 struct page *newpage, struct page *page, enum migrate_mode mode)
2229 {
2230         int rc, extra_count;
2231         struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2232         bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2233
2234         BUG_ON(PageWriteback(page));
2235
2236         /* migrating an atomic written page is safe with the inmem_lock hold */
2237         if (atomic_written) {
2238                 if (mode != MIGRATE_SYNC)
2239                         return -EBUSY;
2240                 if (!mutex_trylock(&fi->inmem_lock))
2241                         return -EAGAIN;
2242         }
2243
2244         /*
2245          * A reference is expected if PagePrivate set when move mapping,
2246          * however F2FS breaks this for maintaining dirty page counts when
2247          * truncating pages. So here adjusting the 'extra_count' make it work.
2248          */
2249         extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2250         rc = migrate_page_move_mapping(mapping, newpage,
2251                                 page, NULL, mode, extra_count);
2252         if (rc != MIGRATEPAGE_SUCCESS) {
2253                 if (atomic_written)
2254                         mutex_unlock(&fi->inmem_lock);
2255                 return rc;
2256         }
2257
2258         if (atomic_written) {
2259                 struct inmem_pages *cur;
2260                 list_for_each_entry(cur, &fi->inmem_pages, list)
2261                         if (cur->page == page) {
2262                                 cur->page = newpage;
2263                                 break;
2264                         }
2265                 mutex_unlock(&fi->inmem_lock);
2266                 put_page(page);
2267                 get_page(newpage);
2268         }
2269
2270         if (PagePrivate(page))
2271                 SetPagePrivate(newpage);
2272         set_page_private(newpage, page_private(page));
2273
2274         if (mode != MIGRATE_SYNC_NO_COPY)
2275                 migrate_page_copy(newpage, page);
2276         else
2277                 migrate_page_states(newpage, page);
2278
2279         return MIGRATEPAGE_SUCCESS;
2280 }
2281 #endif
2282
2283 const struct address_space_operations f2fs_dblock_aops = {
2284         .readpage       = f2fs_read_data_page,
2285         .readpages      = f2fs_read_data_pages,
2286         .writepage      = f2fs_write_data_page,
2287         .writepages     = f2fs_write_data_pages,
2288         .write_begin    = f2fs_write_begin,
2289         .write_end      = f2fs_write_end,
2290         .set_page_dirty = f2fs_set_data_page_dirty,
2291         .invalidatepage = f2fs_invalidate_page,
2292         .releasepage    = f2fs_release_page,
2293         .direct_IO      = f2fs_direct_IO,
2294         .bmap           = f2fs_bmap,
2295 #ifdef CONFIG_MIGRATION
2296         .migratepage    = f2fs_migrate_page,
2297 #endif
2298 };