fs/f2fs/segment.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/segment.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/f2fs_fs.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/prefetch.h>
  13 #include <linux/kthread.h>
  14 #include <linux/swap.h>
  15 #include <linux/timer.h>
  16 #include <linux/freezer.h>
  17 #include <linux/sched/signal.h>
  18
  19 #include "f2fs.h"
  20 #include "segment.h"
  21 #include "node.h"
  22 #include "gc.h"
  23 #include "trace.h"
  24 #include <trace/events/f2fs.h>
  25
  26 #define __reverse_ffz(x) __reverse_ffs(~(x))
  27
  28 static struct kmem_cache *discard_entry_slab;
  29 static struct kmem_cache *discard_cmd_slab;
  30 static struct kmem_cache *sit_entry_set_slab;
  31 static struct kmem_cache *inmem_entry_slab;
  32
  33 static unsigned long __reverse_ulong(unsigned char *str)
  34 {
  35         unsigned long tmp = 0;
  36         int shift = 24, idx = 0;
  37
  38 #if BITS_PER_LONG == 64
  39         shift = 56;
  40 #endif
  41         while (shift >= 0) {
  42                 tmp |= (unsigned long)str[idx++] << shift;
  43                 shift -= BITS_PER_BYTE;
  44         }
  45         return tmp;
  46 }
  47
  48 /*
  49  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  50  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  51  */
  52 static inline unsigned long __reverse_ffs(unsigned long word)
  53 {
  54         int num = 0;
  55
  56 #if BITS_PER_LONG == 64
  57         if ((word & 0xffffffff00000000UL) == 0)
  58                 num += 32;
  59         else
  60                 word >>= 32;
  61 #endif
  62         if ((word & 0xffff0000) == 0)
  63                 num += 16;
  64         else
  65                 word >>= 16;
  66
  67         if ((word & 0xff00) == 0)
  68                 num += 8;
  69         else
  70                 word >>= 8;
  71
  72         if ((word & 0xf0) == 0)
  73                 num += 4;
  74         else
  75                 word >>= 4;
  76
  77         if ((word & 0xc) == 0)
  78                 num += 2;
  79         else
  80                 word >>= 2;
  81
  82         if ((word & 0x2) == 0)
  83                 num += 1;
  84         return num;
  85 }
  86
  87 /*
  88  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  89  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  90  * @size must be integral times of unsigned long.
  91  * Example:
  92  *                             MSB <--> LSB
  93  *   f2fs_set_bit(0, bitmap) => 1000 0000
  94  *   f2fs_set_bit(7, bitmap) => 0000 0001
  95  */
  96 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  97                         unsigned long size, unsigned long offset)
  98 {
  99         const unsigned long *p = addr + BIT_WORD(offset);
 100         unsigned long result = size;
 101         unsigned long tmp;
 102
 103         if (offset >= size)
 104                 return size;
 105
 106         size -= (offset & ~(BITS_PER_LONG - 1));
 107         offset %= BITS_PER_LONG;
 108
 109         while (1) {
 110                 if (*p == 0)
 111                         goto pass;
 112
 113                 tmp = __reverse_ulong((unsigned char *)p);
 114
 115                 tmp &= ~0UL >> offset;
 116                 if (size < BITS_PER_LONG)
 117                         tmp &= (~0UL << (BITS_PER_LONG - size));
 118                 if (tmp)
 119                         goto found;
 120 pass:
 121                 if (size <= BITS_PER_LONG)
 122                         break;
 123                 size -= BITS_PER_LONG;
 124                 offset = 0;
 125                 p++;
 126         }
 127         return result;
 128 found:
 129         return result - size + __reverse_ffs(tmp);
 130 }
 131
 132 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 133                         unsigned long size, unsigned long offset)
 134 {
 135         const unsigned long *p = addr + BIT_WORD(offset);
 136         unsigned long result = size;
 137         unsigned long tmp;
 138
 139         if (offset >= size)
 140                 return size;
 141
 142         size -= (offset & ~(BITS_PER_LONG - 1));
 143         offset %= BITS_PER_LONG;
 144
 145         while (1) {
 146                 if (*p == ~0UL)
 147                         goto pass;
 148
 149                 tmp = __reverse_ulong((unsigned char *)p);
 150
 151                 if (offset)
 152                         tmp |= ~0UL << (BITS_PER_LONG - offset);
 153                 if (size < BITS_PER_LONG)
 154                         tmp |= ~0UL >> size;
 155                 if (tmp != ~0UL)
 156                         goto found;
 157 pass:
 158                 if (size <= BITS_PER_LONG)
 159                         break;
 160                 size -= BITS_PER_LONG;
 161                 offset = 0;
 162                 p++;
 163         }
 164         return result;
 165 found:
 166         return result - size + __reverse_ffz(tmp);
 167 }
 168
 169 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
 170 {
 171         int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 172         int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
 173         int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
 174
 175         if (test_opt(sbi, LFS))
 176                 return false;
 177         if (sbi->gc_mode == GC_URGENT)
 178                 return true;
 179         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
 180                 return true;
 181
 182         return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
 183                         SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
 184 }
 185
 186 void f2fs_register_inmem_page(struct inode *inode, struct page *page)
 187 {
 188         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 189         struct f2fs_inode_info *fi = F2FS_I(inode);
 190         struct inmem_pages *new;
 191
 192         f2fs_trace_pid(page);
 193
 194         set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
 195         SetPagePrivate(page);
 196
 197         new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
 198
 199         /* add atomic page indices to the list */
 200         new->page = page;
 201         INIT_LIST_HEAD(&new->list);
 202
 203         /* increase reference count with clean state */
 204         mutex_lock(&fi->inmem_lock);
 205         get_page(page);
 206         list_add_tail(&new->list, &fi->inmem_pages);
 207         spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
 208         if (list_empty(&fi->inmem_ilist))
 209                 list_add_tail(&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE]);
 210         spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
 211         inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
 212         mutex_unlock(&fi->inmem_lock);
 213
 214         trace_f2fs_register_inmem_page(page, INMEM);
 215 }
 216
 217 static int __revoke_inmem_pages(struct inode *inode,
 218                                 struct list_head *head, bool drop, bool recover)
 219 {
 220         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 221         struct inmem_pages *cur, *tmp;
 222         int err = 0;
 223
 224         list_for_each_entry_safe(cur, tmp, head, list) {
 225                 struct page *page = cur->page;
 226
 227                 if (drop)
 228                         trace_f2fs_commit_inmem_page(page, INMEM_DROP);
 229
 230                 lock_page(page);
 231
 232                 f2fs_wait_on_page_writeback(page, DATA, true, true);
 233
 234                 if (recover) {
 235                         struct dnode_of_data dn;
 236                         struct node_info ni;
 237
 238                         trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
 239 retry:
 240                         set_new_dnode(&dn, inode, NULL, NULL, 0);
 241                         err = f2fs_get_dnode_of_data(&dn, page->index,
 242                                                                 LOOKUP_NODE);
 243                         if (err) {
 244                                 if (err == -ENOMEM) {
 245                                         congestion_wait(BLK_RW_ASYNC, HZ/50);
 246                                         cond_resched();
 247                                         goto retry;
 248                                 }
 249                                 err = -EAGAIN;
 250                                 goto next;
 251                         }
 252
 253                         err = f2fs_get_node_info(sbi, dn.nid, &ni);
 254                         if (err) {
 255                                 f2fs_put_dnode(&dn);
 256                                 return err;
 257                         }
 258
 259                         if (cur->old_addr == NEW_ADDR) {
 260                                 f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
 261                                 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
 262                         } else
 263                                 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
 264                                         cur->old_addr, ni.version, true, true);
 265                         f2fs_put_dnode(&dn);
 266                 }
 267 next:
 268                 /* we don't need to invalidate this in the sccessful status */
 269                 if (drop || recover) {
 270                         ClearPageUptodate(page);
 271                         clear_cold_data(page);
 272                 }
 273                 set_page_private(page, 0);
 274                 ClearPagePrivate(page);
 275                 f2fs_put_page(page, 1);
 276
 277                 list_del(&cur->list);
 278                 kmem_cache_free(inmem_entry_slab, cur);
 279                 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
 280         }
 281         return err;
 282 }
 283
 284 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
 285 {
 286         struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
 287         struct inode *inode;
 288         struct f2fs_inode_info *fi;
 289 next:
 290         spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
 291         if (list_empty(head)) {
 292                 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
 293                 return;
 294         }
 295         fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
 296         inode = igrab(&fi->vfs_inode);
 297         spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
 298
 299         if (inode) {
 300                 if (gc_failure) {
 301                         if (fi->i_gc_failures[GC_FAILURE_ATOMIC])
 302                                 goto drop;
 303                         goto skip;
 304                 }
 305 drop:
 306                 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
 307                 f2fs_drop_inmem_pages(inode);
 308                 iput(inode);
 309         }
 310 skip:
 311         congestion_wait(BLK_RW_ASYNC, HZ/50);
 312         cond_resched();
 313         goto next;
 314 }
 315
 316 void f2fs_drop_inmem_pages(struct inode *inode)
 317 {
 318         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 319         struct f2fs_inode_info *fi = F2FS_I(inode);
 320
 321         mutex_lock(&fi->inmem_lock);
 322         __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
 323         spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
 324         if (!list_empty(&fi->inmem_ilist))
 325                 list_del_init(&fi->inmem_ilist);
 326         spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
 327         mutex_unlock(&fi->inmem_lock);
 328
 329         clear_inode_flag(inode, FI_ATOMIC_FILE);
 330         fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
 331         stat_dec_atomic_write(inode);
 332 }
 333
 334 void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
 335 {
 336         struct f2fs_inode_info *fi = F2FS_I(inode);
 337         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 338         struct list_head *head = &fi->inmem_pages;
 339         struct inmem_pages *cur = NULL;
 340
 341         f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page));
 342
 343         mutex_lock(&fi->inmem_lock);
 344         list_for_each_entry(cur, head, list) {
 345                 if (cur->page == page)
 346                         break;
 347         }
 348
 349         f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
 350         list_del(&cur->list);
 351         mutex_unlock(&fi->inmem_lock);
 352
 353         dec_page_count(sbi, F2FS_INMEM_PAGES);
 354         kmem_cache_free(inmem_entry_slab, cur);
 355
 356         ClearPageUptodate(page);
 357         set_page_private(page, 0);
 358         ClearPagePrivate(page);
 359         f2fs_put_page(page, 0);
 360
 361         trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
 362 }
 363
 364 static int __f2fs_commit_inmem_pages(struct inode *inode)
 365 {
 366         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 367         struct f2fs_inode_info *fi = F2FS_I(inode);
 368         struct inmem_pages *cur, *tmp;
 369         struct f2fs_io_info fio = {
 370                 .sbi = sbi,
 371                 .ino = inode->i_ino,
 372                 .type = DATA,
 373                 .op = REQ_OP_WRITE,
 374                 .op_flags = REQ_SYNC | REQ_PRIO,
 375                 .io_type = FS_DATA_IO,
 376         };
 377         struct list_head revoke_list;
 378         bool submit_bio = false;
 379         int err = 0;
 380
 381         INIT_LIST_HEAD(&revoke_list);
 382
 383         list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
 384                 struct page *page = cur->page;
 385
 386                 lock_page(page);
 387                 if (page->mapping == inode->i_mapping) {
 388                         trace_f2fs_commit_inmem_page(page, INMEM);
 389
 390                         f2fs_wait_on_page_writeback(page, DATA, true, true);
 391
 392                         set_page_dirty(page);
 393                         if (clear_page_dirty_for_io(page)) {
 394                                 inode_dec_dirty_pages(inode);
 395                                 f2fs_remove_dirty_inode(inode);
 396                         }
 397 retry:
 398                         fio.page = page;
 399                         fio.old_blkaddr = NULL_ADDR;
 400                         fio.encrypted_page = NULL;
 401                         fio.need_lock = LOCK_DONE;
 402                         err = f2fs_do_write_data_page(&fio);
 403                         if (err) {
 404                                 if (err == -ENOMEM) {
 405                                         congestion_wait(BLK_RW_ASYNC, HZ/50);
 406                                         cond_resched();
 407                                         goto retry;
 408                                 }
 409                                 unlock_page(page);
 410                                 break;
 411                         }
 412                         /* record old blkaddr for revoking */
 413                         cur->old_addr = fio.old_blkaddr;
 414                         submit_bio = true;
 415                 }
 416                 unlock_page(page);
 417                 list_move_tail(&cur->list, &revoke_list);
 418         }
 419
 420         if (submit_bio)
 421                 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
 422
 423         if (err) {
 424                 /*
 425                  * try to revoke all committed pages, but still we could fail
 426                  * due to no memory or other reason, if that happened, EAGAIN
 427                  * will be returned, which means in such case, transaction is
 428                  * already not integrity, caller should use journal to do the
 429                  * recovery or rewrite & commit last transaction. For other
 430                  * error number, revoking was done by filesystem itself.
 431                  */
 432                 err = __revoke_inmem_pages(inode, &revoke_list, false, true);
 433
 434                 /* drop all uncommitted pages */
 435                 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
 436         } else {
 437                 __revoke_inmem_pages(inode, &revoke_list, false, false);
 438         }
 439
 440         return err;
 441 }
 442
 443 int f2fs_commit_inmem_pages(struct inode *inode)
 444 {
 445         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 446         struct f2fs_inode_info *fi = F2FS_I(inode);
 447         int err;
 448
 449         f2fs_balance_fs(sbi, true);
 450
 451         down_write(&fi->i_gc_rwsem[WRITE]);
 452
 453         f2fs_lock_op(sbi);
 454         set_inode_flag(inode, FI_ATOMIC_COMMIT);
 455
 456         mutex_lock(&fi->inmem_lock);
 457         err = __f2fs_commit_inmem_pages(inode);
 458
 459         spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
 460         if (!list_empty(&fi->inmem_ilist))
 461                 list_del_init(&fi->inmem_ilist);
 462         spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
 463         mutex_unlock(&fi->inmem_lock);
 464
 465         clear_inode_flag(inode, FI_ATOMIC_COMMIT);
 466
 467         f2fs_unlock_op(sbi);
 468         up_write(&fi->i_gc_rwsem[WRITE]);
 469
 470         return err;
 471 }
 472
 473 /*
 474  * This function balances dirty node and dentry pages.
 475  * In addition, it controls garbage collection.
 476  */
 477 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 478 {
 479         if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
 480                 f2fs_show_injection_info(FAULT_CHECKPOINT);
 481                 f2fs_stop_checkpoint(sbi, false);
 482         }
 483
 484         /* balance_fs_bg is able to be pending */
 485         if (need && excess_cached_nats(sbi))
 486                 f2fs_balance_fs_bg(sbi);
 487
 488         if (f2fs_is_checkpoint_ready(sbi))
 489                 return;
 490
 491         /*
 492          * We should do GC or end up with checkpoint, if there are so many dirty
 493          * dir/node pages without enough free segments.
 494          */
 495         if (has_not_enough_free_secs(sbi, 0, 0)) {
 496                 mutex_lock(&sbi->gc_mutex);
 497                 f2fs_gc(sbi, false, false, NULL_SEGNO);
 498         }
 499 }
 500
 501 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
 502 {
 503         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 504                 return;
 505
 506         /* try to shrink extent cache when there is no enough memory */
 507         if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
 508                 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
 509
 510         /* check the # of cached NAT entries */
 511         if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
 512                 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
 513
 514         if (!f2fs_available_free_memory(sbi, FREE_NIDS))
 515                 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
 516         else
 517                 f2fs_build_free_nids(sbi, false, false);
 518
 519         if (!is_idle(sbi, REQ_TIME) &&
 520                 (!excess_dirty_nats(sbi) && !excess_dirty_nodes(sbi)))
 521                 return;
 522
 523         /* checkpoint is the only way to shrink partial cached entries */
 524         if (!f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
 525                         !f2fs_available_free_memory(sbi, INO_ENTRIES) ||
 526                         excess_prefree_segs(sbi) ||
 527                         excess_dirty_nats(sbi) ||
 528                         excess_dirty_nodes(sbi) ||
 529                         f2fs_time_over(sbi, CP_TIME)) {
 530                 if (test_opt(sbi, DATA_FLUSH)) {
 531                         struct blk_plug plug;
 532
 533                         blk_start_plug(&plug);
 534                         f2fs_sync_dirty_inodes(sbi, FILE_INODE);
 535                         blk_finish_plug(&plug);
 536                 }
 537                 f2fs_sync_fs(sbi->sb, true);
 538                 stat_inc_bg_cp_count(sbi->stat_info);
 539         }
 540 }
 541
 542 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
 543                                 struct block_device *bdev)
 544 {
 545         struct bio *bio;
 546         int ret;
 547
 548         bio = f2fs_bio_alloc(sbi, 0, false);
 549         if (!bio)
 550                 return -ENOMEM;
 551
 552         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH;
 553         bio_set_dev(bio, bdev);
 554         ret = submit_bio_wait(bio);
 555         bio_put(bio);
 556
 557         trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
 558                                 test_opt(sbi, FLUSH_MERGE), ret);
 559         return ret;
 560 }
 561
 562 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
 563 {
 564         int ret = 0;
 565         int i;
 566
 567         if (!sbi->s_ndevs)
 568                 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
 569
 570         for (i = 0; i < sbi->s_ndevs; i++) {
 571                 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
 572                         continue;
 573                 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 574                 if (ret)
 575                         break;
 576         }
 577         return ret;
 578 }
 579
 580 static int issue_flush_thread(void *data)
 581 {
 582         struct f2fs_sb_info *sbi = data;
 583         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 584         wait_queue_head_t *q = &fcc->flush_wait_queue;
 585 repeat:
 586         if (kthread_should_stop())
 587                 return 0;
 588
 589         sb_start_intwrite(sbi->sb);
 590
 591         if (!llist_empty(&fcc->issue_list)) {
 592                 struct flush_cmd *cmd, *next;
 593                 int ret;
 594
 595                 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 596                 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 597
 598                 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
 599
 600                 ret = submit_flush_wait(sbi, cmd->ino);
 601                 atomic_inc(&fcc->issued_flush);
 602
 603                 llist_for_each_entry_safe(cmd, next,
 604                                           fcc->dispatch_list, llnode) {
 605                         cmd->ret = ret;
 606                         complete(&cmd->wait);
 607                 }
 608                 fcc->dispatch_list = NULL;
 609         }
 610
 611         sb_end_intwrite(sbi->sb);
 612
 613         wait_event_interruptible(*q,
 614                 kthread_should_stop() || !llist_empty(&fcc->issue_list));
 615         goto repeat;
 616 }
 617
 618 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
 619 {
 620         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 621         struct flush_cmd cmd;
 622         int ret;
 623
 624         if (test_opt(sbi, NOBARRIER))
 625                 return 0;
 626
 627         if (!test_opt(sbi, FLUSH_MERGE)) {
 628                 atomic_inc(&fcc->queued_flush);
 629                 ret = submit_flush_wait(sbi, ino);
 630                 atomic_dec(&fcc->queued_flush);
 631                 atomic_inc(&fcc->issued_flush);
 632                 return ret;
 633         }
 634
 635         if (atomic_inc_return(&fcc->queued_flush) == 1 || sbi->s_ndevs > 1) {
 636                 ret = submit_flush_wait(sbi, ino);
 637                 atomic_dec(&fcc->queued_flush);
 638
 639                 atomic_inc(&fcc->issued_flush);
 640                 return ret;
 641         }
 642
 643         cmd.ino = ino;
 644         init_completion(&cmd.wait);
 645
 646         llist_add(&cmd.llnode, &fcc->issue_list);
 647
 648         /* update issue_list before we wake up issue_flush thread */
 649         smp_mb();
 650
 651         if (waitqueue_active(&fcc->flush_wait_queue))
 652                 wake_up(&fcc->flush_wait_queue);
 653
 654         if (fcc->f2fs_issue_flush) {
 655                 wait_for_completion(&cmd.wait);
 656                 atomic_dec(&fcc->queued_flush);
 657         } else {
 658                 struct llist_node *list;
 659
 660                 list = llist_del_all(&fcc->issue_list);
 661                 if (!list) {
 662                         wait_for_completion(&cmd.wait);
 663                         atomic_dec(&fcc->queued_flush);
 664                 } else {
 665                         struct flush_cmd *tmp, *next;
 666
 667                         ret = submit_flush_wait(sbi, ino);
 668
 669                         llist_for_each_entry_safe(tmp, next, list, llnode) {
 670                                 if (tmp == &cmd) {
 671                                         cmd.ret = ret;
 672                                         atomic_dec(&fcc->queued_flush);
 673                                         continue;
 674                                 }
 675                                 tmp->ret = ret;
 676                                 complete(&tmp->wait);
 677                         }
 678                 }
 679         }
 680
 681         return cmd.ret;
 682 }
 683
 684 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
 685 {
 686         dev_t dev = sbi->sb->s_bdev->bd_dev;
 687         struct flush_cmd_control *fcc;
 688         int err = 0;
 689
 690         if (SM_I(sbi)->fcc_info) {
 691                 fcc = SM_I(sbi)->fcc_info;
 692                 if (fcc->f2fs_issue_flush)
 693                         return err;
 694                 goto init_thread;
 695         }
 696
 697         fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
 698         if (!fcc)
 699                 return -ENOMEM;
 700         atomic_set(&fcc->issued_flush, 0);
 701         atomic_set(&fcc->queued_flush, 0);
 702         init_waitqueue_head(&fcc->flush_wait_queue);
 703         init_llist_head(&fcc->issue_list);
 704         SM_I(sbi)->fcc_info = fcc;
 705         if (!test_opt(sbi, FLUSH_MERGE))
 706                 return err;
 707
 708 init_thread:
 709         fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 710                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 711         if (IS_ERR(fcc->f2fs_issue_flush)) {
 712                 err = PTR_ERR(fcc->f2fs_issue_flush);
 713                 kvfree(fcc);
 714                 SM_I(sbi)->fcc_info = NULL;
 715                 return err;
 716         }
 717
 718         return err;
 719 }
 720
 721 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
 722 {
 723         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 724
 725         if (fcc && fcc->f2fs_issue_flush) {
 726                 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
 727
 728                 fcc->f2fs_issue_flush = NULL;
 729                 kthread_stop(flush_thread);
 730         }
 731         if (free) {
 732                 kvfree(fcc);
 733                 SM_I(sbi)->fcc_info = NULL;
 734         }
 735 }
 736
 737 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
 738 {
 739         int ret = 0, i;
 740
 741         if (!sbi->s_ndevs)
 742                 return 0;
 743
 744         for (i = 1; i < sbi->s_ndevs; i++) {
 745                 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
 746                         continue;
 747                 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 748                 if (ret)
 749                         break;
 750
 751                 spin_lock(&sbi->dev_lock);
 752                 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
 753                 spin_unlock(&sbi->dev_lock);
 754         }
 755
 756         return ret;
 757 }
 758
 759 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 760                 enum dirty_type dirty_type)
 761 {
 762         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 763
 764         /* need not be added */
 765         if (IS_CURSEG(sbi, segno))
 766                 return;
 767
 768         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 769                 dirty_i->nr_dirty[dirty_type]++;
 770
 771         if (dirty_type == DIRTY) {
 772                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 773                 enum dirty_type t = sentry->type;
 774
 775                 if (unlikely(t >= DIRTY)) {
 776                         f2fs_bug_on(sbi, 1);
 777                         return;
 778                 }
 779                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 780                         dirty_i->nr_dirty[t]++;
 781         }
 782 }
 783
 784 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 785                 enum dirty_type dirty_type)
 786 {
 787         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 788
 789         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 790                 dirty_i->nr_dirty[dirty_type]--;
 791
 792         if (dirty_type == DIRTY) {
 793                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 794                 enum dirty_type t = sentry->type;
 795
 796                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 797                         dirty_i->nr_dirty[t]--;
 798
 799                 if (get_valid_blocks(sbi, segno, true) == 0)
 800                         clear_bit(GET_SEC_FROM_SEG(sbi, segno),
 801                                                 dirty_i->victim_secmap);
 802         }
 803 }
 804
 805 /*
 806  * Should not occur error such as -ENOMEM.
 807  * Adding dirty entry into seglist is not critical operation.
 808  * If a given segment is one of current working segments, it won't be added.
 809  */
 810 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 811 {
 812         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 813         unsigned short valid_blocks, ckpt_valid_blocks;
 814
 815         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 816                 return;
 817
 818         mutex_lock(&dirty_i->seglist_lock);
 819
 820         valid_blocks = get_valid_blocks(sbi, segno, false);
 821         ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno);
 822
 823         if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
 824                                 ckpt_valid_blocks == sbi->blocks_per_seg)) {
 825                 __locate_dirty_segment(sbi, segno, PRE);
 826                 __remove_dirty_segment(sbi, segno, DIRTY);
 827         } else if (valid_blocks < sbi->blocks_per_seg) {
 828                 __locate_dirty_segment(sbi, segno, DIRTY);
 829         } else {
 830                 /* Recovery routine with SSR needs this */
 831                 __remove_dirty_segment(sbi, segno, DIRTY);
 832         }
 833
 834         mutex_unlock(&dirty_i->seglist_lock);
 835 }
 836
 837 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
 838 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
 839 {
 840         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 841         unsigned int segno;
 842
 843         mutex_lock(&dirty_i->seglist_lock);
 844         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 845                 if (get_valid_blocks(sbi, segno, false))
 846                         continue;
 847                 if (IS_CURSEG(sbi, segno))
 848                         continue;
 849                 __locate_dirty_segment(sbi, segno, PRE);
 850                 __remove_dirty_segment(sbi, segno, DIRTY);
 851         }
 852         mutex_unlock(&dirty_i->seglist_lock);
 853 }
 854
 855 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi)
 856 {
 857         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 858         block_t ovp = overprovision_segments(sbi) << sbi->log_blocks_per_seg;
 859         block_t holes[2] = {0, 0};      /* DATA and NODE */
 860         struct seg_entry *se;
 861         unsigned int segno;
 862
 863         mutex_lock(&dirty_i->seglist_lock);
 864         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 865                 se = get_seg_entry(sbi, segno);
 866                 if (IS_NODESEG(se->type))
 867                         holes[NODE] += sbi->blocks_per_seg - se->valid_blocks;
 868                 else
 869                         holes[DATA] += sbi->blocks_per_seg - se->valid_blocks;
 870         }
 871         mutex_unlock(&dirty_i->seglist_lock);
 872
 873         if (holes[DATA] > ovp || holes[NODE] > ovp)
 874                 return -EAGAIN;
 875         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
 876                 dirty_segments(sbi) > overprovision_segments(sbi))
 877                 return -EAGAIN;
 878         return 0;
 879 }
 880
 881 /* This is only used by SBI_CP_DISABLED */
 882 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
 883 {
 884         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 885         unsigned int segno = 0;
 886
 887         mutex_lock(&dirty_i->seglist_lock);
 888         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 889                 if (get_valid_blocks(sbi, segno, false))
 890                         continue;
 891                 if (get_ckpt_valid_blocks(sbi, segno))
 892                         continue;
 893                 mutex_unlock(&dirty_i->seglist_lock);
 894                 return segno;
 895         }
 896         mutex_unlock(&dirty_i->seglist_lock);
 897         return NULL_SEGNO;
 898 }
 899
 900 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
 901                 struct block_device *bdev, block_t lstart,
 902                 block_t start, block_t len)
 903 {
 904         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 905         struct list_head *pend_list;
 906         struct discard_cmd *dc;
 907
 908         f2fs_bug_on(sbi, !len);
 909
 910         pend_list = &dcc->pend_list[plist_idx(len)];
 911
 912         dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
 913         INIT_LIST_HEAD(&dc->list);
 914         dc->bdev = bdev;
 915         dc->lstart = lstart;
 916         dc->start = start;
 917         dc->len = len;
 918         dc->ref = 0;
 919         dc->state = D_PREP;
 920         dc->queued = 0;
 921         dc->error = 0;
 922         init_completion(&dc->wait);
 923         list_add_tail(&dc->list, pend_list);
 924         spin_lock_init(&dc->lock);
 925         dc->bio_ref = 0;
 926         atomic_inc(&dcc->discard_cmd_cnt);
 927         dcc->undiscard_blks += len;
 928
 929         return dc;
 930 }
 931
 932 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
 933                                 struct block_device *bdev, block_t lstart,
 934                                 block_t start, block_t len,
 935                                 struct rb_node *parent, struct rb_node **p,
 936                                 bool leftmost)
 937 {
 938         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 939         struct discard_cmd *dc;
 940
 941         dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
 942
 943         rb_link_node(&dc->rb_node, parent, p);
 944         rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
 945
 946         return dc;
 947 }
 948
 949 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
 950                                                         struct discard_cmd *dc)
 951 {
 952         if (dc->state == D_DONE)
 953                 atomic_sub(dc->queued, &dcc->queued_discard);
 954
 955         list_del(&dc->list);
 956         rb_erase_cached(&dc->rb_node, &dcc->root);
 957         dcc->undiscard_blks -= dc->len;
 958
 959         kmem_cache_free(discard_cmd_slab, dc);
 960
 961         atomic_dec(&dcc->discard_cmd_cnt);
 962 }
 963
 964 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
 965                                                         struct discard_cmd *dc)
 966 {
 967         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 968         unsigned long flags;
 969
 970         trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
 971
 972         spin_lock_irqsave(&dc->lock, flags);
 973         if (dc->bio_ref) {
 974                 spin_unlock_irqrestore(&dc->lock, flags);
 975                 return;
 976         }
 977         spin_unlock_irqrestore(&dc->lock, flags);
 978
 979         f2fs_bug_on(sbi, dc->ref);
 980
 981         if (dc->error == -EOPNOTSUPP)
 982                 dc->error = 0;
 983
 984         if (dc->error)
 985                 printk_ratelimited(
 986                         "%sF2FS-fs: Issue discard(%u, %u, %u) failed, ret: %d",
 987                         KERN_INFO, dc->lstart, dc->start, dc->len, dc->error);
 988         __detach_discard_cmd(dcc, dc);
 989 }
 990
 991 static void f2fs_submit_discard_endio(struct bio *bio)
 992 {
 993         struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
 994         unsigned long flags;
 995
 996         dc->error = blk_status_to_errno(bio->bi_status);
 997
 998         spin_lock_irqsave(&dc->lock, flags);
 999         dc->bio_ref--;
1000         if (!dc->bio_ref && dc->state == D_SUBMIT) {
1001                 dc->state = D_DONE;
1002                 complete_all(&dc->wait);
1003         }
1004         spin_unlock_irqrestore(&dc->lock, flags);
1005         bio_put(bio);
1006 }
1007
1008 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1009                                 block_t start, block_t end)
1010 {
1011 #ifdef CONFIG_F2FS_CHECK_FS
1012         struct seg_entry *sentry;
1013         unsigned int segno;
1014         block_t blk = start;
1015         unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1016         unsigned long *map;
1017
1018         while (blk < end) {
1019                 segno = GET_SEGNO(sbi, blk);
1020                 sentry = get_seg_entry(sbi, segno);
1021                 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1022
1023                 if (end < START_BLOCK(sbi, segno + 1))
1024                         size = GET_BLKOFF_FROM_SEG0(sbi, end);
1025                 else
1026                         size = max_blocks;
1027                 map = (unsigned long *)(sentry->cur_valid_map);
1028                 offset = __find_rev_next_bit(map, size, offset);
1029                 f2fs_bug_on(sbi, offset != size);
1030                 blk = START_BLOCK(sbi, segno + 1);
1031         }
1032 #endif
1033 }
1034
1035 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1036                                 struct discard_policy *dpolicy,
1037                                 int discard_type, unsigned int granularity)
1038 {
1039         /* common policy */
1040         dpolicy->type = discard_type;
1041         dpolicy->sync = true;
1042         dpolicy->ordered = false;
1043         dpolicy->granularity = granularity;
1044
1045         dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
1046         dpolicy->io_aware_gran = MAX_PLIST_NUM;
1047         dpolicy->timeout = 0;
1048
1049         if (discard_type == DPOLICY_BG) {
1050                 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1051                 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1052                 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1053                 dpolicy->io_aware = true;
1054                 dpolicy->sync = false;
1055                 dpolicy->ordered = true;
1056                 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1057                         dpolicy->granularity = 1;
1058                         dpolicy->max_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1059                 }
1060         } else if (discard_type == DPOLICY_FORCE) {
1061                 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1062                 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1063                 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1064                 dpolicy->io_aware = false;
1065         } else if (discard_type == DPOLICY_FSTRIM) {
1066                 dpolicy->io_aware = false;
1067         } else if (discard_type == DPOLICY_UMOUNT) {
1068                 dpolicy->max_requests = UINT_MAX;
1069                 dpolicy->io_aware = false;
1070                 /* we need to issue all to keep CP_TRIMMED_FLAG */
1071                 dpolicy->granularity = 1;
1072         }
1073 }
1074
1075 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1076                                 struct block_device *bdev, block_t lstart,
1077                                 block_t start, block_t len);
1078 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1079 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1080                                                 struct discard_policy *dpolicy,
1081                                                 struct discard_cmd *dc,
1082                                                 unsigned int *issued)
1083 {
1084         struct block_device *bdev = dc->bdev;
1085         struct request_queue *q = bdev_get_queue(bdev);
1086         unsigned int max_discard_blocks =
1087                         SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1088         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1089         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1090                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1091         int flag = dpolicy->sync ? REQ_SYNC : 0;
1092         block_t lstart, start, len, total_len;
1093         int err = 0;
1094
1095         if (dc->state != D_PREP)
1096                 return 0;
1097
1098         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1099                 return 0;
1100
1101         trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1102
1103         lstart = dc->lstart;
1104         start = dc->start;
1105         len = dc->len;
1106         total_len = len;
1107
1108         dc->len = 0;
1109
1110         while (total_len && *issued < dpolicy->max_requests && !err) {
1111                 struct bio *bio = NULL;
1112                 unsigned long flags;
1113                 bool last = true;
1114
1115                 if (len > max_discard_blocks) {
1116                         len = max_discard_blocks;
1117                         last = false;
1118                 }
1119
1120                 (*issued)++;
1121                 if (*issued == dpolicy->max_requests)
1122                         last = true;
1123
1124                 dc->len += len;
1125
1126                 if (time_to_inject(sbi, FAULT_DISCARD)) {
1127                         f2fs_show_injection_info(FAULT_DISCARD);
1128                         err = -EIO;
1129                         goto submit;
1130                 }
1131                 err = __blkdev_issue_discard(bdev,
1132                                         SECTOR_FROM_BLOCK(start),
1133                                         SECTOR_FROM_BLOCK(len),
1134                                         GFP_NOFS, 0, &bio);
1135 submit:
1136                 if (err) {
1137                         spin_lock_irqsave(&dc->lock, flags);
1138                         if (dc->state == D_PARTIAL)
1139                                 dc->state = D_SUBMIT;
1140                         spin_unlock_irqrestore(&dc->lock, flags);
1141
1142                         break;
1143                 }
1144
1145                 f2fs_bug_on(sbi, !bio);
1146
1147                 /*
1148                  * should keep before submission to avoid D_DONE
1149                  * right away
1150                  */
1151                 spin_lock_irqsave(&dc->lock, flags);
1152                 if (last)
1153                         dc->state = D_SUBMIT;
1154                 else
1155                         dc->state = D_PARTIAL;
1156                 dc->bio_ref++;
1157                 spin_unlock_irqrestore(&dc->lock, flags);
1158
1159                 atomic_inc(&dcc->queued_discard);
1160                 dc->queued++;
1161                 list_move_tail(&dc->list, wait_list);
1162
1163                 /* sanity check on discard range */
1164                 __check_sit_bitmap(sbi, lstart, lstart + len);
1165
1166                 bio->bi_private = dc;
1167                 bio->bi_end_io = f2fs_submit_discard_endio;
1168                 bio->bi_opf |= flag;
1169                 submit_bio(bio);
1170
1171                 atomic_inc(&dcc->issued_discard);
1172
1173                 f2fs_update_iostat(sbi, FS_DISCARD, 1);
1174
1175                 lstart += len;
1176                 start += len;
1177                 total_len -= len;
1178                 len = total_len;
1179         }
1180
1181         if (!err && len)
1182                 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1183         return err;
1184 }
1185
1186 static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi,
1187                                 struct block_device *bdev, block_t lstart,
1188                                 block_t start, block_t len,
1189                                 struct rb_node **insert_p,
1190                                 struct rb_node *insert_parent)
1191 {
1192         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1193         struct rb_node **p;
1194         struct rb_node *parent = NULL;
1195         struct discard_cmd *dc = NULL;
1196         bool leftmost = true;
1197
1198         if (insert_p && insert_parent) {
1199                 parent = insert_parent;
1200                 p = insert_p;
1201                 goto do_insert;
1202         }
1203
1204         p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1205                                                         lstart, &leftmost);
1206 do_insert:
1207         dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1208                                                                 p, leftmost);
1209         if (!dc)
1210                 return NULL;
1211
1212         return dc;
1213 }
1214
1215 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1216                                                 struct discard_cmd *dc)
1217 {
1218         list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1219 }
1220
1221 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1222                                 struct discard_cmd *dc, block_t blkaddr)
1223 {
1224         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1225         struct discard_info di = dc->di;
1226         bool modified = false;
1227
1228         if (dc->state == D_DONE || dc->len == 1) {
1229                 __remove_discard_cmd(sbi, dc);
1230                 return;
1231         }
1232
1233         dcc->undiscard_blks -= di.len;
1234
1235         if (blkaddr > di.lstart) {
1236                 dc->len = blkaddr - dc->lstart;
1237                 dcc->undiscard_blks += dc->len;
1238                 __relocate_discard_cmd(dcc, dc);
1239                 modified = true;
1240         }
1241
1242         if (blkaddr < di.lstart + di.len - 1) {
1243                 if (modified) {
1244                         __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1245                                         di.start + blkaddr + 1 - di.lstart,
1246                                         di.lstart + di.len - 1 - blkaddr,
1247                                         NULL, NULL);
1248                 } else {
1249                         dc->lstart++;
1250                         dc->len--;
1251                         dc->start++;
1252                         dcc->undiscard_blks += dc->len;
1253                         __relocate_discard_cmd(dcc, dc);
1254                 }
1255         }
1256 }
1257
1258 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1259                                 struct block_device *bdev, block_t lstart,
1260                                 block_t start, block_t len)
1261 {
1262         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1263         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1264         struct discard_cmd *dc;
1265         struct discard_info di = {0};
1266         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1267         struct request_queue *q = bdev_get_queue(bdev);
1268         unsigned int max_discard_blocks =
1269                         SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1270         block_t end = lstart + len;
1271
1272         dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1273                                         NULL, lstart,
1274                                         (struct rb_entry **)&prev_dc,
1275                                         (struct rb_entry **)&next_dc,
1276                                         &insert_p, &insert_parent, true, NULL);
1277         if (dc)
1278                 prev_dc = dc;
1279
1280         if (!prev_dc) {
1281                 di.lstart = lstart;
1282                 di.len = next_dc ? next_dc->lstart - lstart : len;
1283                 di.len = min(di.len, len);
1284                 di.start = start;
1285         }
1286
1287         while (1) {
1288                 struct rb_node *node;
1289                 bool merged = false;
1290                 struct discard_cmd *tdc = NULL;
1291
1292                 if (prev_dc) {
1293                         di.lstart = prev_dc->lstart + prev_dc->len;
1294                         if (di.lstart < lstart)
1295                                 di.lstart = lstart;
1296                         if (di.lstart >= end)
1297                                 break;
1298
1299                         if (!next_dc || next_dc->lstart > end)
1300                                 di.len = end - di.lstart;
1301                         else
1302                                 di.len = next_dc->lstart - di.lstart;
1303                         di.start = start + di.lstart - lstart;
1304                 }
1305
1306                 if (!di.len)
1307                         goto next;
1308
1309                 if (prev_dc && prev_dc->state == D_PREP &&
1310                         prev_dc->bdev == bdev &&
1311                         __is_discard_back_mergeable(&di, &prev_dc->di,
1312                                                         max_discard_blocks)) {
1313                         prev_dc->di.len += di.len;
1314                         dcc->undiscard_blks += di.len;
1315                         __relocate_discard_cmd(dcc, prev_dc);
1316                         di = prev_dc->di;
1317                         tdc = prev_dc;
1318                         merged = true;
1319                 }
1320
1321                 if (next_dc && next_dc->state == D_PREP &&
1322                         next_dc->bdev == bdev &&
1323                         __is_discard_front_mergeable(&di, &next_dc->di,
1324                                                         max_discard_blocks)) {
1325                         next_dc->di.lstart = di.lstart;
1326                         next_dc->di.len += di.len;
1327                         next_dc->di.start = di.start;
1328                         dcc->undiscard_blks += di.len;
1329                         __relocate_discard_cmd(dcc, next_dc);
1330                         if (tdc)
1331                                 __remove_discard_cmd(sbi, tdc);
1332                         merged = true;
1333                 }
1334
1335                 if (!merged) {
1336                         __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1337                                                         di.len, NULL, NULL);
1338                 }
1339  next:
1340                 prev_dc = next_dc;
1341                 if (!prev_dc)
1342                         break;
1343
1344                 node = rb_next(&prev_dc->rb_node);
1345                 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1346         }
1347 }
1348
1349 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1350                 struct block_device *bdev, block_t blkstart, block_t blklen)
1351 {
1352         block_t lblkstart = blkstart;
1353
1354         trace_f2fs_queue_discard(bdev, blkstart, blklen);
1355
1356         if (sbi->s_ndevs) {
1357                 int devi = f2fs_target_device_index(sbi, blkstart);
1358
1359                 blkstart -= FDEV(devi).start_blk;
1360         }
1361         mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1362         __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1363         mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1364         return 0;
1365 }
1366
1367 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1368                                         struct discard_policy *dpolicy)
1369 {
1370         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1371         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1372         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1373         struct discard_cmd *dc;
1374         struct blk_plug plug;
1375         unsigned int pos = dcc->next_pos;
1376         unsigned int issued = 0;
1377         bool io_interrupted = false;
1378
1379         mutex_lock(&dcc->cmd_lock);
1380         dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1381                                         NULL, pos,
1382                                         (struct rb_entry **)&prev_dc,
1383                                         (struct rb_entry **)&next_dc,
1384                                         &insert_p, &insert_parent, true, NULL);
1385         if (!dc)
1386                 dc = next_dc;
1387
1388         blk_start_plug(&plug);
1389
1390         while (dc) {
1391                 struct rb_node *node;
1392                 int err = 0;
1393
1394                 if (dc->state != D_PREP)
1395                         goto next;
1396
1397                 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1398                         io_interrupted = true;
1399                         break;
1400                 }
1401
1402                 dcc->next_pos = dc->lstart + dc->len;
1403                 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1404
1405                 if (issued >= dpolicy->max_requests)
1406                         break;
1407 next:
1408                 node = rb_next(&dc->rb_node);
1409                 if (err)
1410                         __remove_discard_cmd(sbi, dc);
1411                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1412         }
1413
1414         blk_finish_plug(&plug);
1415
1416         if (!dc)
1417                 dcc->next_pos = 0;
1418
1419         mutex_unlock(&dcc->cmd_lock);
1420
1421         if (!issued && io_interrupted)
1422                 issued = -1;
1423
1424         return issued;
1425 }
1426
1427 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1428                                         struct discard_policy *dpolicy)
1429 {
1430         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1431         struct list_head *pend_list;
1432         struct discard_cmd *dc, *tmp;
1433         struct blk_plug plug;
1434         int i, issued = 0;
1435         bool io_interrupted = false;
1436
1437         if (dpolicy->timeout != 0)
1438                 f2fs_update_time(sbi, dpolicy->timeout);
1439
1440         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1441                 if (dpolicy->timeout != 0 &&
1442                                 f2fs_time_over(sbi, dpolicy->timeout))
1443                         break;
1444
1445                 if (i + 1 < dpolicy->granularity)
1446                         break;
1447
1448                 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1449                         return __issue_discard_cmd_orderly(sbi, dpolicy);
1450
1451                 pend_list = &dcc->pend_list[i];
1452
1453                 mutex_lock(&dcc->cmd_lock);
1454                 if (list_empty(pend_list))
1455                         goto next;
1456                 if (unlikely(dcc->rbtree_check))
1457                         f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1458                                                                 &dcc->root));
1459                 blk_start_plug(&plug);
1460                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1461                         f2fs_bug_on(sbi, dc->state != D_PREP);
1462
1463                         if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1464                                                 !is_idle(sbi, DISCARD_TIME)) {
1465                                 io_interrupted = true;
1466                                 break;
1467                         }
1468
1469                         __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1470
1471                         if (issued >= dpolicy->max_requests)
1472                                 break;
1473                 }
1474                 blk_finish_plug(&plug);
1475 next:
1476                 mutex_unlock(&dcc->cmd_lock);
1477
1478                 if (issued >= dpolicy->max_requests || io_interrupted)
1479                         break;
1480         }
1481
1482         if (!issued && io_interrupted)
1483                 issued = -1;
1484
1485         return issued;
1486 }
1487
1488 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1489 {
1490         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1491         struct list_head *pend_list;
1492         struct discard_cmd *dc, *tmp;
1493         int i;
1494         bool dropped = false;
1495
1496         mutex_lock(&dcc->cmd_lock);
1497         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1498                 pend_list = &dcc->pend_list[i];
1499                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1500                         f2fs_bug_on(sbi, dc->state != D_PREP);
1501                         __remove_discard_cmd(sbi, dc);
1502                         dropped = true;
1503                 }
1504         }
1505         mutex_unlock(&dcc->cmd_lock);
1506
1507         return dropped;
1508 }
1509
1510 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1511 {
1512         __drop_discard_cmd(sbi);
1513 }
1514
1515 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1516                                                         struct discard_cmd *dc)
1517 {
1518         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1519         unsigned int len = 0;
1520
1521         wait_for_completion_io(&dc->wait);
1522         mutex_lock(&dcc->cmd_lock);
1523         f2fs_bug_on(sbi, dc->state != D_DONE);
1524         dc->ref--;
1525         if (!dc->ref) {
1526                 if (!dc->error)
1527                         len = dc->len;
1528                 __remove_discard_cmd(sbi, dc);
1529         }
1530         mutex_unlock(&dcc->cmd_lock);
1531
1532         return len;
1533 }
1534
1535 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1536                                                 struct discard_policy *dpolicy,
1537                                                 block_t start, block_t end)
1538 {
1539         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1540         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1541                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1542         struct discard_cmd *dc, *tmp;
1543         bool need_wait;
1544         unsigned int trimmed = 0;
1545
1546 next:
1547         need_wait = false;
1548
1549         mutex_lock(&dcc->cmd_lock);
1550         list_for_each_entry_safe(dc, tmp, wait_list, list) {
1551                 if (dc->lstart + dc->len <= start || end <= dc->lstart)
1552                         continue;
1553                 if (dc->len < dpolicy->granularity)
1554                         continue;
1555                 if (dc->state == D_DONE && !dc->ref) {
1556                         wait_for_completion_io(&dc->wait);
1557                         if (!dc->error)
1558                                 trimmed += dc->len;
1559                         __remove_discard_cmd(sbi, dc);
1560                 } else {
1561                         dc->ref++;
1562                         need_wait = true;
1563                         break;
1564                 }
1565         }
1566         mutex_unlock(&dcc->cmd_lock);
1567
1568         if (need_wait) {
1569                 trimmed += __wait_one_discard_bio(sbi, dc);
1570                 goto next;
1571         }
1572
1573         return trimmed;
1574 }
1575
1576 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1577                                                 struct discard_policy *dpolicy)
1578 {
1579         struct discard_policy dp;
1580         unsigned int discard_blks;
1581
1582         if (dpolicy)
1583                 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1584
1585         /* wait all */
1586         __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1587         discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1588         __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1589         discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1590
1591         return discard_blks;
1592 }
1593
1594 /* This should be covered by global mutex, &sit_i->sentry_lock */
1595 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1596 {
1597         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1598         struct discard_cmd *dc;
1599         bool need_wait = false;
1600
1601         mutex_lock(&dcc->cmd_lock);
1602         dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1603                                                         NULL, blkaddr);
1604         if (dc) {
1605                 if (dc->state == D_PREP) {
1606                         __punch_discard_cmd(sbi, dc, blkaddr);
1607                 } else {
1608                         dc->ref++;
1609                         need_wait = true;
1610                 }
1611         }
1612         mutex_unlock(&dcc->cmd_lock);
1613
1614         if (need_wait)
1615                 __wait_one_discard_bio(sbi, dc);
1616 }
1617
1618 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1619 {
1620         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1621
1622         if (dcc && dcc->f2fs_issue_discard) {
1623                 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1624
1625                 dcc->f2fs_issue_discard = NULL;
1626                 kthread_stop(discard_thread);
1627         }
1628 }
1629
1630 /* This comes from f2fs_put_super */
1631 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1632 {
1633         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1634         struct discard_policy dpolicy;
1635         bool dropped;
1636
1637         __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1638                                         dcc->discard_granularity);
1639         dpolicy.timeout = UMOUNT_DISCARD_TIMEOUT;
1640         __issue_discard_cmd(sbi, &dpolicy);
1641         dropped = __drop_discard_cmd(sbi);
1642
1643         /* just to make sure there is no pending discard commands */
1644         __wait_all_discard_cmd(sbi, NULL);
1645
1646         f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1647         return dropped;
1648 }
1649
1650 static int issue_discard_thread(void *data)
1651 {
1652         struct f2fs_sb_info *sbi = data;
1653         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1654         wait_queue_head_t *q = &dcc->discard_wait_queue;
1655         struct discard_policy dpolicy;
1656         unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
1657         int issued;
1658
1659         set_freezable();
1660
1661         do {
1662                 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1663                                         dcc->discard_granularity);
1664
1665                 wait_event_interruptible_timeout(*q,
1666                                 kthread_should_stop() || freezing(current) ||
1667                                 dcc->discard_wake,
1668                                 msecs_to_jiffies(wait_ms));
1669
1670                 if (dcc->discard_wake)
1671                         dcc->discard_wake = 0;
1672
1673                 /* clean up pending candidates before going to sleep */
1674                 if (atomic_read(&dcc->queued_discard))
1675                         __wait_all_discard_cmd(sbi, NULL);
1676
1677                 if (try_to_freeze())
1678                         continue;
1679                 if (f2fs_readonly(sbi->sb))
1680                         continue;
1681                 if (kthread_should_stop())
1682                         return 0;
1683                 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1684                         wait_ms = dpolicy.max_interval;
1685                         continue;
1686                 }
1687
1688                 if (sbi->gc_mode == GC_URGENT)
1689                         __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1690
1691                 sb_start_intwrite(sbi->sb);
1692
1693                 issued = __issue_discard_cmd(sbi, &dpolicy);
1694                 if (issued > 0) {
1695                         __wait_all_discard_cmd(sbi, &dpolicy);
1696                         wait_ms = dpolicy.min_interval;
1697                 } else if (issued == -1){
1698                         wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1699                         if (!wait_ms)
1700                                 wait_ms = dpolicy.mid_interval;
1701                 } else {
1702                         wait_ms = dpolicy.max_interval;
1703                 }
1704
1705                 sb_end_intwrite(sbi->sb);
1706
1707         } while (!kthread_should_stop());
1708         return 0;
1709 }
1710
1711 #ifdef CONFIG_BLK_DEV_ZONED
1712 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1713                 struct block_device *bdev, block_t blkstart, block_t blklen)
1714 {
1715         sector_t sector, nr_sects;
1716         block_t lblkstart = blkstart;
1717         int devi = 0;
1718
1719         if (sbi->s_ndevs) {
1720                 devi = f2fs_target_device_index(sbi, blkstart);
1721                 blkstart -= FDEV(devi).start_blk;
1722         }
1723
1724         /*
1725          * We need to know the type of the zone: for conventional zones,
1726          * use regular discard if the drive supports it. For sequential
1727          * zones, reset the zone write pointer.
1728          */
1729         switch (get_blkz_type(sbi, bdev, blkstart)) {
1730
1731         case BLK_ZONE_TYPE_CONVENTIONAL:
1732                 if (!blk_queue_discard(bdev_get_queue(bdev)))
1733                         return 0;
1734                 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1735         case BLK_ZONE_TYPE_SEQWRITE_REQ:
1736         case BLK_ZONE_TYPE_SEQWRITE_PREF:
1737                 sector = SECTOR_FROM_BLOCK(blkstart);
1738                 nr_sects = SECTOR_FROM_BLOCK(blklen);
1739
1740                 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1741                                 nr_sects != bdev_zone_sectors(bdev)) {
1742                         f2fs_msg(sbi->sb, KERN_INFO,
1743                                 "(%d) %s: Unaligned discard attempted (block %x + %x)",
1744                                 devi, sbi->s_ndevs ? FDEV(devi).path: "",
1745                                 blkstart, blklen);
1746                         return -EIO;
1747                 }
1748                 trace_f2fs_issue_reset_zone(bdev, blkstart);
1749                 return blkdev_reset_zones(bdev, sector,
1750                                           nr_sects, GFP_NOFS);
1751         default:
1752                 /* Unknown zone type: broken device ? */
1753                 return -EIO;
1754         }
1755 }
1756 #endif
1757
1758 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1759                 struct block_device *bdev, block_t blkstart, block_t blklen)
1760 {
1761 #ifdef CONFIG_BLK_DEV_ZONED
1762         if (f2fs_sb_has_blkzoned(sbi) &&
1763                                 bdev_zoned_model(bdev) != BLK_ZONED_NONE)
1764                 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1765 #endif
1766         return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1767 }
1768
1769 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1770                                 block_t blkstart, block_t blklen)
1771 {
1772         sector_t start = blkstart, len = 0;
1773         struct block_device *bdev;
1774         struct seg_entry *se;
1775         unsigned int offset;
1776         block_t i;
1777         int err = 0;
1778
1779         bdev = f2fs_target_device(sbi, blkstart, NULL);
1780
1781         for (i = blkstart; i < blkstart + blklen; i++, len++) {
1782                 if (i != start) {
1783                         struct block_device *bdev2 =
1784                                 f2fs_target_device(sbi, i, NULL);
1785
1786                         if (bdev2 != bdev) {
1787                                 err = __issue_discard_async(sbi, bdev,
1788                                                 start, len);
1789                                 if (err)
1790                                         return err;
1791                                 bdev = bdev2;
1792                                 start = i;
1793                                 len = 0;
1794                         }
1795                 }
1796
1797                 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1798                 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1799
1800                 if (!f2fs_test_and_set_bit(offset, se->discard_map))
1801                         sbi->discard_blks--;
1802         }
1803
1804         if (len)
1805                 err = __issue_discard_async(sbi, bdev, start, len);
1806         return err;
1807 }
1808
1809 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1810                                                         bool check_only)
1811 {
1812         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1813         int max_blocks = sbi->blocks_per_seg;
1814         struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1815         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1816         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1817         unsigned long *discard_map = (unsigned long *)se->discard_map;
1818         unsigned long *dmap = SIT_I(sbi)->tmp_map;
1819         unsigned int start = 0, end = -1;
1820         bool force = (cpc->reason & CP_DISCARD);
1821         struct discard_entry *de = NULL;
1822         struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1823         int i;
1824
1825         if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))
1826                 return false;
1827
1828         if (!force) {
1829                 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1830                         SM_I(sbi)->dcc_info->nr_discards >=
1831                                 SM_I(sbi)->dcc_info->max_discards)
1832                         return false;
1833         }
1834
1835         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
1836         for (i = 0; i < entries; i++)
1837                 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1838                                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1839
1840         while (force || SM_I(sbi)->dcc_info->nr_discards <=
1841                                 SM_I(sbi)->dcc_info->max_discards) {
1842                 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1843                 if (start >= max_blocks)
1844                         break;
1845
1846                 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1847                 if (force && start && end != max_blocks
1848                                         && (end - start) < cpc->trim_minlen)
1849                         continue;
1850
1851                 if (check_only)
1852                         return true;
1853
1854                 if (!de) {
1855                         de = f2fs_kmem_cache_alloc(discard_entry_slab,
1856                                                                 GFP_F2FS_ZERO);
1857                         de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1858                         list_add_tail(&de->list, head);
1859                 }
1860
1861                 for (i = start; i < end; i++)
1862                         __set_bit_le(i, (void *)de->discard_map);
1863
1864                 SM_I(sbi)->dcc_info->nr_discards += end - start;
1865         }
1866         return false;
1867 }
1868
1869 static void release_discard_addr(struct discard_entry *entry)
1870 {
1871         list_del(&entry->list);
1872         kmem_cache_free(discard_entry_slab, entry);
1873 }
1874
1875 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
1876 {
1877         struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
1878         struct discard_entry *entry, *this;
1879
1880         /* drop caches */
1881         list_for_each_entry_safe(entry, this, head, list)
1882                 release_discard_addr(entry);
1883 }
1884
1885 /*
1886  * Should call f2fs_clear_prefree_segments after checkpoint is done.
1887  */
1888 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
1889 {
1890         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1891         unsigned int segno;
1892
1893         mutex_lock(&dirty_i->seglist_lock);
1894         for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1895                 __set_test_and_free(sbi, segno);
1896         mutex_unlock(&dirty_i->seglist_lock);
1897 }
1898
1899 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
1900                                                 struct cp_control *cpc)
1901 {
1902         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1903         struct list_head *head = &dcc->entry_list;
1904         struct discard_entry *entry, *this;
1905         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1906         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
1907         unsigned int start = 0, end = -1;
1908         unsigned int secno, start_segno;
1909         bool force = (cpc->reason & CP_DISCARD);
1910         bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi);
1911
1912         mutex_lock(&dirty_i->seglist_lock);
1913
1914         while (1) {
1915                 int i;
1916
1917                 if (need_align && end != -1)
1918                         end--;
1919                 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
1920                 if (start >= MAIN_SEGS(sbi))
1921                         break;
1922                 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
1923                                                                 start + 1);
1924
1925                 if (need_align) {
1926                         start = rounddown(start, sbi->segs_per_sec);
1927                         end = roundup(end, sbi->segs_per_sec);
1928                 }
1929
1930                 for (i = start; i < end; i++) {
1931                         if (test_and_clear_bit(i, prefree_map))
1932                                 dirty_i->nr_dirty[PRE]--;
1933                 }
1934
1935                 if (!f2fs_realtime_discard_enable(sbi))
1936                         continue;
1937
1938                 if (force && start >= cpc->trim_start &&
1939                                         (end - 1) <= cpc->trim_end)
1940                                 continue;
1941
1942                 if (!test_opt(sbi, LFS) || !__is_large_section(sbi)) {
1943                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
1944                                 (end - start) << sbi->log_blocks_per_seg);
1945                         continue;
1946                 }
1947 next:
1948                 secno = GET_SEC_FROM_SEG(sbi, start);
1949                 start_segno = GET_SEG_FROM_SEC(sbi, secno);
1950                 if (!IS_CURSEC(sbi, secno) &&
1951                         !get_valid_blocks(sbi, start, true))
1952                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
1953                                 sbi->segs_per_sec << sbi->log_blocks_per_seg);
1954
1955                 start = start_segno + sbi->segs_per_sec;
1956                 if (start < end)
1957                         goto next;
1958                 else
1959                         end = start - 1;
1960         }
1961         mutex_unlock(&dirty_i->seglist_lock);
1962
1963         /* send small discards */
1964         list_for_each_entry_safe(entry, this, head, list) {
1965                 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
1966                 bool is_valid = test_bit_le(0, entry->discard_map);
1967
1968 find_next:
1969                 if (is_valid) {
1970                         next_pos = find_next_zero_bit_le(entry->discard_map,
1971                                         sbi->blocks_per_seg, cur_pos);
1972                         len = next_pos - cur_pos;
1973
1974                         if (f2fs_sb_has_blkzoned(sbi) ||
1975                             (force && len < cpc->trim_minlen))
1976                                 goto skip;
1977
1978                         f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
1979                                                                         len);
1980                         total_len += len;
1981                 } else {
1982                         next_pos = find_next_bit_le(entry->discard_map,
1983                                         sbi->blocks_per_seg, cur_pos);
1984                 }
1985 skip:
1986                 cur_pos = next_pos;
1987                 is_valid = !is_valid;
1988
1989                 if (cur_pos < sbi->blocks_per_seg)
1990                         goto find_next;
1991
1992                 release_discard_addr(entry);
1993                 dcc->nr_discards -= total_len;
1994         }
1995
1996         wake_up_discard_thread(sbi, false);
1997 }
1998
1999 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2000 {
2001         dev_t dev = sbi->sb->s_bdev->bd_dev;
2002         struct discard_cmd_control *dcc;
2003         int err = 0, i;
2004
2005         if (SM_I(sbi)->dcc_info) {
2006                 dcc = SM_I(sbi)->dcc_info;
2007                 goto init_thread;
2008         }
2009
2010         dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2011         if (!dcc)
2012                 return -ENOMEM;
2013
2014         dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2015         INIT_LIST_HEAD(&dcc->entry_list);
2016         for (i = 0; i < MAX_PLIST_NUM; i++)
2017                 INIT_LIST_HEAD(&dcc->pend_list[i]);
2018         INIT_LIST_HEAD(&dcc->wait_list);
2019         INIT_LIST_HEAD(&dcc->fstrim_list);
2020         mutex_init(&dcc->cmd_lock);
2021         atomic_set(&dcc->issued_discard, 0);
2022         atomic_set(&dcc->queued_discard, 0);
2023         atomic_set(&dcc->discard_cmd_cnt, 0);
2024         dcc->nr_discards = 0;
2025         dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2026         dcc->undiscard_blks = 0;
2027         dcc->next_pos = 0;
2028         dcc->root = RB_ROOT_CACHED;
2029         dcc->rbtree_check = false;
2030
2031         init_waitqueue_head(&dcc->discard_wait_queue);
2032         SM_I(sbi)->dcc_info = dcc;
2033 init_thread:
2034         dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2035                                 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2036         if (IS_ERR(dcc->f2fs_issue_discard)) {
2037                 err = PTR_ERR(dcc->f2fs_issue_discard);
2038                 kvfree(dcc);
2039                 SM_I(sbi)->dcc_info = NULL;
2040                 return err;
2041         }
2042
2043         return err;
2044 }
2045
2046 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2047 {
2048         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2049
2050         if (!dcc)
2051                 return;
2052
2053         f2fs_stop_discard_thread(sbi);
2054
2055         kvfree(dcc);
2056         SM_I(sbi)->dcc_info = NULL;
2057 }
2058
2059 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2060 {
2061         struct sit_info *sit_i = SIT_I(sbi);
2062
2063         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2064                 sit_i->dirty_sentries++;
2065                 return false;
2066         }
2067
2068         return true;
2069 }
2070
2071 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2072                                         unsigned int segno, int modified)
2073 {
2074         struct seg_entry *se = get_seg_entry(sbi, segno);
2075         se->type = type;
2076         if (modified)
2077                 __mark_sit_entry_dirty(sbi, segno);
2078 }
2079
2080 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2081 {
2082         struct seg_entry *se;
2083         unsigned int segno, offset;
2084         long int new_vblocks;
2085         bool exist;
2086 #ifdef CONFIG_F2FS_CHECK_FS
2087         bool mir_exist;
2088 #endif
2089
2090         segno = GET_SEGNO(sbi, blkaddr);
2091
2092         se = get_seg_entry(sbi, segno);
2093         new_vblocks = se->valid_blocks + del;
2094         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2095
2096         f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
2097                                 (new_vblocks > sbi->blocks_per_seg)));
2098
2099         se->valid_blocks = new_vblocks;
2100         se->mtime = get_mtime(sbi, false);
2101         if (se->mtime > SIT_I(sbi)->max_mtime)
2102                 SIT_I(sbi)->max_mtime = se->mtime;
2103
2104         /* Update valid block bitmap */
2105         if (del > 0) {
2106                 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2107 #ifdef CONFIG_F2FS_CHECK_FS
2108                 mir_exist = f2fs_test_and_set_bit(offset,
2109                                                 se->cur_valid_map_mir);
2110                 if (unlikely(exist != mir_exist)) {
2111                         f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error "
2112                                 "when setting bitmap, blk:%u, old bit:%d",
2113                                 blkaddr, exist);
2114                         f2fs_bug_on(sbi, 1);
2115                 }
2116 #endif
2117                 if (unlikely(exist)) {
2118                         f2fs_msg(sbi->sb, KERN_ERR,
2119                                 "Bitmap was wrongly set, blk:%u", blkaddr);
2120                         f2fs_bug_on(sbi, 1);
2121                         se->valid_blocks--;
2122                         del = 0;
2123                 }
2124
2125                 if (!f2fs_test_and_set_bit(offset, se->discard_map))
2126                         sbi->discard_blks--;
2127
2128                 /* don't overwrite by SSR to keep node chain */
2129                 if (IS_NODESEG(se->type) &&
2130                                 !is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2131                         if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2132                                 se->ckpt_valid_blocks++;
2133                 }
2134         } else {
2135                 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2136 #ifdef CONFIG_F2FS_CHECK_FS
2137                 mir_exist = f2fs_test_and_clear_bit(offset,
2138                                                 se->cur_valid_map_mir);
2139                 if (unlikely(exist != mir_exist)) {
2140                         f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error "
2141                                 "when clearing bitmap, blk:%u, old bit:%d",
2142                                 blkaddr, exist);
2143                         f2fs_bug_on(sbi, 1);
2144                 }
2145 #endif
2146                 if (unlikely(!exist)) {
2147                         f2fs_msg(sbi->sb, KERN_ERR,
2148                                 "Bitmap was wrongly cleared, blk:%u", blkaddr);
2149                         f2fs_bug_on(sbi, 1);
2150                         se->valid_blocks++;
2151                         del = 0;
2152                 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2153                         /*
2154                          * If checkpoints are off, we must not reuse data that
2155                          * was used in the previous checkpoint. If it was used
2156                          * before, we must track that to know how much space we
2157                          * really have.
2158                          */
2159                         if (f2fs_test_bit(offset, se->ckpt_valid_map))
2160                                 sbi->unusable_block_count++;
2161                 }
2162
2163                 if (f2fs_test_and_clear_bit(offset, se->discard_map))
2164                         sbi->discard_blks++;
2165         }
2166         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2167                 se->ckpt_valid_blocks += del;
2168
2169         __mark_sit_entry_dirty(sbi, segno);
2170
2171         /* update total number of valid blocks to be written in ckpt area */
2172         SIT_I(sbi)->written_valid_blocks += del;
2173
2174         if (__is_large_section(sbi))
2175                 get_sec_entry(sbi, segno)->valid_blocks += del;
2176 }
2177
2178 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2179 {
2180         unsigned int segno = GET_SEGNO(sbi, addr);
2181         struct sit_info *sit_i = SIT_I(sbi);
2182
2183         f2fs_bug_on(sbi, addr == NULL_ADDR);
2184         if (addr == NEW_ADDR)
2185                 return;
2186
2187         invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2188
2189         /* add it into sit main buffer */
2190         down_write(&sit_i->sentry_lock);
2191
2192         update_sit_entry(sbi, addr, -1);
2193
2194         /* add it into dirty seglist */
2195         locate_dirty_segment(sbi, segno);
2196
2197         up_write(&sit_i->sentry_lock);
2198 }
2199
2200 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2201 {
2202         struct sit_info *sit_i = SIT_I(sbi);
2203         unsigned int segno, offset;
2204         struct seg_entry *se;
2205         bool is_cp = false;
2206
2207         if (!is_valid_data_blkaddr(sbi, blkaddr))
2208                 return true;
2209
2210         down_read(&sit_i->sentry_lock);
2211
2212         segno = GET_SEGNO(sbi, blkaddr);
2213         se = get_seg_entry(sbi, segno);
2214         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2215
2216         if (f2fs_test_bit(offset, se->ckpt_valid_map))
2217                 is_cp = true;
2218
2219         up_read(&sit_i->sentry_lock);
2220
2221         return is_cp;
2222 }
2223
2224 /*
2225  * This function should be resided under the curseg_mutex lock
2226  */
2227 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2228                                         struct f2fs_summary *sum)
2229 {
2230         struct curseg_info *curseg = CURSEG_I(sbi, type);
2231         void *addr = curseg->sum_blk;
2232         addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2233         memcpy(addr, sum, sizeof(struct f2fs_summary));
2234 }
2235
2236 /*
2237  * Calculate the number of current summary pages for writing
2238  */
2239 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2240 {
2241         int valid_sum_count = 0;
2242         int i, sum_in_page;
2243
2244         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2245                 if (sbi->ckpt->alloc_type[i] == SSR)
2246                         valid_sum_count += sbi->blocks_per_seg;
2247                 else {
2248                         if (for_ra)
2249                                 valid_sum_count += le16_to_cpu(
2250                                         F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2251                         else
2252                                 valid_sum_count += curseg_blkoff(sbi, i);
2253                 }
2254         }
2255
2256         sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2257                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2258         if (valid_sum_count <= sum_in_page)
2259                 return 1;
2260         else if ((valid_sum_count - sum_in_page) <=
2261                 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2262                 return 2;
2263         return 3;
2264 }
2265
2266 /*
2267  * Caller should put this summary page
2268  */
2269 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2270 {
2271         return f2fs_get_meta_page_nofail(sbi, GET_SUM_BLOCK(sbi, segno));
2272 }
2273
2274 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2275                                         void *src, block_t blk_addr)
2276 {
2277         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2278
2279         memcpy(page_address(page), src, PAGE_SIZE);
2280         set_page_dirty(page);
2281         f2fs_put_page(page, 1);
2282 }
2283
2284 static void write_sum_page(struct f2fs_sb_info *sbi,
2285                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
2286 {
2287         f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2288 }
2289
2290 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2291                                                 int type, block_t blk_addr)
2292 {
2293         struct curseg_info *curseg = CURSEG_I(sbi, type);
2294         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2295         struct f2fs_summary_block *src = curseg->sum_blk;
2296         struct f2fs_summary_block *dst;
2297
2298         dst = (struct f2fs_summary_block *)page_address(page);
2299         memset(dst, 0, PAGE_SIZE);
2300
2301         mutex_lock(&curseg->curseg_mutex);
2302
2303         down_read(&curseg->journal_rwsem);
2304         memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2305         up_read(&curseg->journal_rwsem);
2306
2307         memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2308         memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2309
2310         mutex_unlock(&curseg->curseg_mutex);
2311
2312         set_page_dirty(page);
2313         f2fs_put_page(page, 1);
2314 }
2315
2316 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
2317 {
2318         struct curseg_info *curseg = CURSEG_I(sbi, type);
2319         unsigned int segno = curseg->segno + 1;
2320         struct free_segmap_info *free_i = FREE_I(sbi);
2321
2322         if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2323                 return !test_bit(segno, free_i->free_segmap);
2324         return 0;
2325 }
2326
2327 /*
2328  * Find a new segment from the free segments bitmap to right order
2329  * This function should be returned with success, otherwise BUG
2330  */
2331 static void get_new_segment(struct f2fs_sb_info *sbi,
2332                         unsigned int *newseg, bool new_sec, int dir)
2333 {
2334         struct free_segmap_info *free_i = FREE_I(sbi);
2335         unsigned int segno, secno, zoneno;
2336         unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2337         unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2338         unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2339         unsigned int left_start = hint;
2340         bool init = true;
2341         int go_left = 0;
2342         int i;
2343
2344         spin_lock(&free_i->segmap_lock);
2345
2346         if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2347                 segno = find_next_zero_bit(free_i->free_segmap,
2348                         GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2349                 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2350                         goto got_it;
2351         }
2352 find_other_zone:
2353         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2354         if (secno >= MAIN_SECS(sbi)) {
2355                 if (dir == ALLOC_RIGHT) {
2356                         secno = find_next_zero_bit(free_i->free_secmap,
2357                                                         MAIN_SECS(sbi), 0);
2358                         f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2359                 } else {
2360                         go_left = 1;
2361                         left_start = hint - 1;
2362                 }
2363         }
2364         if (go_left == 0)
2365                 goto skip_left;
2366
2367         while (test_bit(left_start, free_i->free_secmap)) {
2368                 if (left_start > 0) {
2369                         left_start--;
2370                         continue;
2371                 }
2372                 left_start = find_next_zero_bit(free_i->free_secmap,
2373                                                         MAIN_SECS(sbi), 0);
2374                 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2375                 break;
2376         }
2377         secno = left_start;
2378 skip_left:
2379         segno = GET_SEG_FROM_SEC(sbi, secno);
2380         zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2381
2382         /* give up on finding another zone */
2383         if (!init)
2384                 goto got_it;
2385         if (sbi->secs_per_zone == 1)
2386                 goto got_it;
2387         if (zoneno == old_zoneno)
2388                 goto got_it;
2389         if (dir == ALLOC_LEFT) {
2390                 if (!go_left && zoneno + 1 >= total_zones)
2391                         goto got_it;
2392                 if (go_left && zoneno == 0)
2393                         goto got_it;
2394         }
2395         for (i = 0; i < NR_CURSEG_TYPE; i++)
2396                 if (CURSEG_I(sbi, i)->zone == zoneno)
2397                         break;
2398
2399         if (i < NR_CURSEG_TYPE) {
2400                 /* zone is in user, try another */
2401                 if (go_left)
2402                         hint = zoneno * sbi->secs_per_zone - 1;
2403                 else if (zoneno + 1 >= total_zones)
2404                         hint = 0;
2405                 else
2406                         hint = (zoneno + 1) * sbi->secs_per_zone;
2407                 init = false;
2408                 goto find_other_zone;
2409         }
2410 got_it:
2411         /* set it as dirty segment in free segmap */
2412         f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2413         __set_inuse(sbi, segno);
2414         *newseg = segno;
2415         spin_unlock(&free_i->segmap_lock);
2416 }
2417
2418 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2419 {
2420         struct curseg_info *curseg = CURSEG_I(sbi, type);
2421         struct summary_footer *sum_footer;
2422
2423         curseg->segno = curseg->next_segno;
2424         curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2425         curseg->next_blkoff = 0;
2426         curseg->next_segno = NULL_SEGNO;
2427
2428         sum_footer = &(curseg->sum_blk->footer);
2429         memset(sum_footer, 0, sizeof(struct summary_footer));
2430         if (IS_DATASEG(type))
2431                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2432         if (IS_NODESEG(type))
2433                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2434         __set_sit_entry_type(sbi, type, curseg->segno, modified);
2435 }
2436
2437 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2438 {
2439         /* if segs_per_sec is large than 1, we need to keep original policy. */
2440         if (__is_large_section(sbi))
2441                 return CURSEG_I(sbi, type)->segno;
2442
2443         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2444                 return 0;
2445
2446         if (test_opt(sbi, NOHEAP) &&
2447                 (type == CURSEG_HOT_DATA || IS_NODESEG(type)))
2448                 return 0;
2449
2450         if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2451                 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2452
2453         /* find segments from 0 to reuse freed segments */
2454         if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2455                 return 0;
2456
2457         return CURSEG_I(sbi, type)->segno;
2458 }
2459
2460 /*
2461  * Allocate a current working segment.
2462  * This function always allocates a free segment in LFS manner.
2463  */
2464 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2465 {
2466         struct curseg_info *curseg = CURSEG_I(sbi, type);
2467         unsigned int segno = curseg->segno;
2468         int dir = ALLOC_LEFT;
2469
2470         write_sum_page(sbi, curseg->sum_blk,
2471                                 GET_SUM_BLOCK(sbi, segno));
2472         if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
2473                 dir = ALLOC_RIGHT;
2474
2475         if (test_opt(sbi, NOHEAP))
2476                 dir = ALLOC_RIGHT;
2477
2478         segno = __get_next_segno(sbi, type);
2479         get_new_segment(sbi, &segno, new_sec, dir);
2480         curseg->next_segno = segno;
2481         reset_curseg(sbi, type, 1);
2482         curseg->alloc_type = LFS;
2483 }
2484
2485 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
2486                         struct curseg_info *seg, block_t start)
2487 {
2488         struct seg_entry *se = get_seg_entry(sbi, seg->segno);
2489         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2490         unsigned long *target_map = SIT_I(sbi)->tmp_map;
2491         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2492         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2493         int i, pos;
2494
2495         for (i = 0; i < entries; i++)
2496                 target_map[i] = ckpt_map[i] | cur_map[i];
2497
2498         pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2499
2500         seg->next_blkoff = pos;
2501 }
2502
2503 /*
2504  * If a segment is written by LFS manner, next block offset is just obtained
2505  * by increasing the current block offset. However, if a segment is written by
2506  * SSR manner, next block offset obtained by calling __next_free_blkoff
2507  */
2508 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2509                                 struct curseg_info *seg)
2510 {
2511         if (seg->alloc_type == SSR)
2512                 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
2513         else
2514                 seg->next_blkoff++;
2515 }
2516
2517 /*
2518  * This function always allocates a used segment(from dirty seglist) by SSR
2519  * manner, so it should recover the existing segment information of valid blocks
2520  */
2521 static void change_curseg(struct f2fs_sb_info *sbi, int type)
2522 {
2523         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2524         struct curseg_info *curseg = CURSEG_I(sbi, type);
2525         unsigned int new_segno = curseg->next_segno;
2526         struct f2fs_summary_block *sum_node;
2527         struct page *sum_page;
2528
2529         write_sum_page(sbi, curseg->sum_blk,
2530                                 GET_SUM_BLOCK(sbi, curseg->segno));
2531         __set_test_and_inuse(sbi, new_segno);
2532
2533         mutex_lock(&dirty_i->seglist_lock);
2534         __remove_dirty_segment(sbi, new_segno, PRE);
2535         __remove_dirty_segment(sbi, new_segno, DIRTY);
2536         mutex_unlock(&dirty_i->seglist_lock);
2537
2538         reset_curseg(sbi, type, 1);
2539         curseg->alloc_type = SSR;
2540         __next_free_blkoff(sbi, curseg, 0);
2541
2542         sum_page = f2fs_get_sum_page(sbi, new_segno);
2543         f2fs_bug_on(sbi, IS_ERR(sum_page));
2544         sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2545         memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2546         f2fs_put_page(sum_page, 1);
2547 }
2548
2549 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
2550 {
2551         struct curseg_info *curseg = CURSEG_I(sbi, type);
2552         const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2553         unsigned segno = NULL_SEGNO;
2554         int i, cnt;
2555         bool reversed = false;
2556
2557         /* f2fs_need_SSR() already forces to do this */
2558         if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) {
2559                 curseg->next_segno = segno;
2560                 return 1;
2561         }
2562
2563         /* For node segments, let's do SSR more intensively */
2564         if (IS_NODESEG(type)) {
2565                 if (type >= CURSEG_WARM_NODE) {
2566                         reversed = true;
2567                         i = CURSEG_COLD_NODE;
2568                 } else {
2569                         i = CURSEG_HOT_NODE;
2570                 }
2571                 cnt = NR_CURSEG_NODE_TYPE;
2572         } else {
2573                 if (type >= CURSEG_WARM_DATA) {
2574                         reversed = true;
2575                         i = CURSEG_COLD_DATA;
2576                 } else {
2577                         i = CURSEG_HOT_DATA;
2578                 }
2579                 cnt = NR_CURSEG_DATA_TYPE;
2580         }
2581
2582         for (; cnt-- > 0; reversed ? i-- : i++) {
2583                 if (i == type)
2584                         continue;
2585                 if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) {
2586                         curseg->next_segno = segno;
2587                         return 1;
2588                 }
2589         }
2590
2591         /* find valid_blocks=0 in dirty list */
2592         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2593                 segno = get_free_segment(sbi);
2594                 if (segno != NULL_SEGNO) {
2595                         curseg->next_segno = segno;
2596                         return 1;
2597                 }
2598         }
2599         return 0;
2600 }
2601
2602 /*
2603  * flush out current segment and replace it with new segment
2604  * This function should be returned with success, otherwise BUG
2605  */
2606 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2607                                                 int type, bool force)
2608 {
2609         struct curseg_info *curseg = CURSEG_I(sbi, type);
2610
2611         if (force)
2612                 new_curseg(sbi, type, true);
2613         else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2614                                         type == CURSEG_WARM_NODE)
2615                 new_curseg(sbi, type, false);
2616         else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type) &&
2617                         likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2618                 new_curseg(sbi, type, false);
2619         else if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type))
2620                 change_curseg(sbi, type);
2621         else
2622                 new_curseg(sbi, type, false);
2623
2624         stat_inc_seg_type(sbi, curseg);
2625 }
2626
2627 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
2628 {
2629         struct curseg_info *curseg;
2630         unsigned int old_segno;
2631         int i;
2632
2633         down_write(&SIT_I(sbi)->sentry_lock);
2634
2635         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2636                 curseg = CURSEG_I(sbi, i);
2637                 old_segno = curseg->segno;
2638                 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
2639                 locate_dirty_segment(sbi, old_segno);
2640         }
2641
2642         up_write(&SIT_I(sbi)->sentry_lock);
2643 }
2644
2645 static const struct segment_allocation default_salloc_ops = {
2646         .allocate_segment = allocate_segment_by_default,
2647 };
2648
2649 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
2650                                                 struct cp_control *cpc)
2651 {
2652         __u64 trim_start = cpc->trim_start;
2653         bool has_candidate = false;
2654
2655         down_write(&SIT_I(sbi)->sentry_lock);
2656         for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
2657                 if (add_discard_addrs(sbi, cpc, true)) {
2658                         has_candidate = true;
2659                         break;
2660                 }
2661         }
2662         up_write(&SIT_I(sbi)->sentry_lock);
2663
2664         cpc->trim_start = trim_start;
2665         return has_candidate;
2666 }
2667
2668 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
2669                                         struct discard_policy *dpolicy,
2670                                         unsigned int start, unsigned int end)
2671 {
2672         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2673         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
2674         struct rb_node **insert_p = NULL, *insert_parent = NULL;
2675         struct discard_cmd *dc;
2676         struct blk_plug plug;
2677         int issued;
2678         unsigned int trimmed = 0;
2679
2680 next:
2681         issued = 0;
2682
2683         mutex_lock(&dcc->cmd_lock);
2684         if (unlikely(dcc->rbtree_check))
2685                 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
2686                                                                 &dcc->root));
2687
2688         dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
2689                                         NULL, start,
2690                                         (struct rb_entry **)&prev_dc,
2691                                         (struct rb_entry **)&next_dc,
2692                                         &insert_p, &insert_parent, true, NULL);
2693         if (!dc)
2694                 dc = next_dc;
2695
2696         blk_start_plug(&plug);
2697
2698         while (dc && dc->lstart <= end) {
2699                 struct rb_node *node;
2700                 int err = 0;
2701
2702                 if (dc->len < dpolicy->granularity)
2703                         goto skip;
2704
2705                 if (dc->state != D_PREP) {
2706                         list_move_tail(&dc->list, &dcc->fstrim_list);
2707                         goto skip;
2708                 }
2709
2710                 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
2711
2712                 if (issued >= dpolicy->max_requests) {
2713                         start = dc->lstart + dc->len;
2714
2715                         if (err)
2716                                 __remove_discard_cmd(sbi, dc);
2717
2718                         blk_finish_plug(&plug);
2719                         mutex_unlock(&dcc->cmd_lock);
2720                         trimmed += __wait_all_discard_cmd(sbi, NULL);
2721                         congestion_wait(BLK_RW_ASYNC, HZ/50);
2722                         goto next;
2723                 }
2724 skip:
2725                 node = rb_next(&dc->rb_node);
2726                 if (err)
2727                         __remove_discard_cmd(sbi, dc);
2728                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
2729
2730                 if (fatal_signal_pending(current))
2731                         break;
2732         }
2733
2734         blk_finish_plug(&plug);
2735         mutex_unlock(&dcc->cmd_lock);
2736
2737         return trimmed;
2738 }
2739
2740 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
2741 {
2742         __u64 start = F2FS_BYTES_TO_BLK(range->start);
2743         __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
2744         unsigned int start_segno, end_segno;
2745         block_t start_block, end_block;
2746         struct cp_control cpc;
2747         struct discard_policy dpolicy;
2748         unsigned long long trimmed = 0;
2749         int err = 0;
2750         bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi);
2751
2752         if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
2753                 return -EINVAL;
2754
2755         if (end < MAIN_BLKADDR(sbi))
2756                 goto out;
2757
2758         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
2759                 f2fs_msg(sbi->sb, KERN_WARNING,
2760                         "Found FS corruption, run fsck to fix.");
2761                 return -EIO;
2762         }
2763
2764         /* start/end segment number in main_area */
2765         start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
2766         end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
2767                                                 GET_SEGNO(sbi, end);
2768         if (need_align) {
2769                 start_segno = rounddown(start_segno, sbi->segs_per_sec);
2770                 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
2771         }
2772
2773         cpc.reason = CP_DISCARD;
2774         cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
2775         cpc.trim_start = start_segno;
2776         cpc.trim_end = end_segno;
2777
2778         if (sbi->discard_blks == 0)
2779                 goto out;
2780
2781         mutex_lock(&sbi->gc_mutex);
2782         err = f2fs_write_checkpoint(sbi, &cpc);
2783         mutex_unlock(&sbi->gc_mutex);
2784         if (err)
2785                 goto out;
2786
2787         /*
2788          * We filed discard candidates, but actually we don't need to wait for
2789          * all of them, since they'll be issued in idle time along with runtime
2790          * discard option. User configuration looks like using runtime discard
2791          * or periodic fstrim instead of it.
2792          */
2793         if (f2fs_realtime_discard_enable(sbi))
2794                 goto out;
2795
2796         start_block = START_BLOCK(sbi, start_segno);
2797         end_block = START_BLOCK(sbi, end_segno + 1);
2798
2799         __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
2800         trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
2801                                         start_block, end_block);
2802
2803         trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
2804                                         start_block, end_block);
2805 out:
2806         if (!err)
2807                 range->len = F2FS_BLK_TO_BYTES(trimmed);
2808         return err;
2809 }
2810
2811 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
2812 {
2813         struct curseg_info *curseg = CURSEG_I(sbi, type);
2814         if (curseg->next_blkoff < sbi->blocks_per_seg)
2815                 return true;
2816         return false;
2817 }
2818
2819 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
2820 {
2821         switch (hint) {
2822         case WRITE_LIFE_SHORT:
2823                 return CURSEG_HOT_DATA;
2824         case WRITE_LIFE_EXTREME:
2825                 return CURSEG_COLD_DATA;
2826         default:
2827                 return CURSEG_WARM_DATA;
2828         }
2829 }
2830
2831 /* This returns write hints for each segment type. This hints will be
2832  * passed down to block layer. There are mapping tables which depend on
2833  * the mount option 'whint_mode'.
2834  *
2835  * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET.
2836  *
2837  * 2) whint_mode=user-based. F2FS tries to pass down hints given by users.
2838  *
2839  * User                  F2FS                     Block
2840  * ----                  ----                     -----
2841  *                       META                     WRITE_LIFE_NOT_SET
2842  *                       HOT_NODE                 "
2843  *                       WARM_NODE                "
2844  *                       COLD_NODE                "
2845  * ioctl(COLD)           COLD_DATA                WRITE_LIFE_EXTREME
2846  * extension list        "                        "
2847  *
2848  * -- buffered io
2849  * WRITE_LIFE_EXTREME    COLD_DATA                WRITE_LIFE_EXTREME
2850  * WRITE_LIFE_SHORT      HOT_DATA                 WRITE_LIFE_SHORT
2851  * WRITE_LIFE_NOT_SET    WARM_DATA                WRITE_LIFE_NOT_SET
2852  * WRITE_LIFE_NONE       "                        "
2853  * WRITE_LIFE_MEDIUM     "                        "
2854  * WRITE_LIFE_LONG       "                        "
2855  *
2856  * -- direct io
2857  * WRITE_LIFE_EXTREME    COLD_DATA                WRITE_LIFE_EXTREME
2858  * WRITE_LIFE_SHORT      HOT_DATA                 WRITE_LIFE_SHORT
2859  * WRITE_LIFE_NOT_SET    WARM_DATA                WRITE_LIFE_NOT_SET
2860  * WRITE_LIFE_NONE       "                        WRITE_LIFE_NONE
2861  * WRITE_LIFE_MEDIUM     "                        WRITE_LIFE_MEDIUM
2862  * WRITE_LIFE_LONG       "                        WRITE_LIFE_LONG
2863  *
2864  * 3) whint_mode=fs-based. F2FS passes down hints with its policy.
2865  *
2866  * User                  F2FS                     Block
2867  * ----                  ----                     -----
2868  *                       META                     WRITE_LIFE_MEDIUM;
2869  *                       HOT_NODE                 WRITE_LIFE_NOT_SET
2870  *                       WARM_NODE                "
2871  *                       COLD_NODE                WRITE_LIFE_NONE
2872  * ioctl(COLD)           COLD_DATA                WRITE_LIFE_EXTREME
2873  * extension list        "                        "
2874  *
2875  * -- buffered io
2876  * WRITE_LIFE_EXTREME    COLD_DATA                WRITE_LIFE_EXTREME
2877  * WRITE_LIFE_SHORT      HOT_DATA                 WRITE_LIFE_SHORT
2878  * WRITE_LIFE_NOT_SET    WARM_DATA                WRITE_LIFE_LONG
2879  * WRITE_LIFE_NONE       "                        "
2880  * WRITE_LIFE_MEDIUM     "                        "
2881  * WRITE_LIFE_LONG       "                        "
2882  *
2883  * -- direct io
2884  * WRITE_LIFE_EXTREME    COLD_DATA                WRITE_LIFE_EXTREME
2885  * WRITE_LIFE_SHORT      HOT_DATA                 WRITE_LIFE_SHORT
2886  * WRITE_LIFE_NOT_SET    WARM_DATA                WRITE_LIFE_NOT_SET
2887  * WRITE_LIFE_NONE       "                        WRITE_LIFE_NONE
2888  * WRITE_LIFE_MEDIUM     "                        WRITE_LIFE_MEDIUM
2889  * WRITE_LIFE_LONG       "                        WRITE_LIFE_LONG
2890  */
2891
2892 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
2893                                 enum page_type type, enum temp_type temp)
2894 {
2895         if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
2896                 if (type == DATA) {
2897                         if (temp == WARM)
2898                                 return WRITE_LIFE_NOT_SET;
2899                         else if (temp == HOT)
2900                                 return WRITE_LIFE_SHORT;
2901                         else if (temp == COLD)
2902                                 return WRITE_LIFE_EXTREME;
2903                 } else {
2904                         return WRITE_LIFE_NOT_SET;
2905                 }
2906         } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
2907                 if (type == DATA) {
2908                         if (temp == WARM)
2909                                 return WRITE_LIFE_LONG;
2910                         else if (temp == HOT)
2911                                 return WRITE_LIFE_SHORT;
2912                         else if (temp == COLD)
2913                                 return WRITE_LIFE_EXTREME;
2914                 } else if (type == NODE) {
2915                         if (temp == WARM || temp == HOT)
2916                                 return WRITE_LIFE_NOT_SET;
2917                         else if (temp == COLD)
2918                                 return WRITE_LIFE_NONE;
2919                 } else if (type == META) {
2920                         return WRITE_LIFE_MEDIUM;
2921                 }
2922         }
2923         return WRITE_LIFE_NOT_SET;
2924 }
2925
2926 static int __get_segment_type_2(struct f2fs_io_info *fio)
2927 {
2928         if (fio->type == DATA)
2929                 return CURSEG_HOT_DATA;
2930         else
2931                 return CURSEG_HOT_NODE;
2932 }
2933
2934 static int __get_segment_type_4(struct f2fs_io_info *fio)
2935 {
2936         if (fio->type == DATA) {
2937                 struct inode *inode = fio->page->mapping->host;
2938
2939                 if (S_ISDIR(inode->i_mode))
2940                         return CURSEG_HOT_DATA;
2941                 else
2942                         return CURSEG_COLD_DATA;
2943         } else {
2944                 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
2945                         return CURSEG_WARM_NODE;
2946                 else
2947                         return CURSEG_COLD_NODE;
2948         }
2949 }
2950
2951 static int __get_segment_type_6(struct f2fs_io_info *fio)
2952 {
2953         if (fio->type == DATA) {
2954                 struct inode *inode = fio->page->mapping->host;
2955
2956                 if (is_cold_data(fio->page) || file_is_cold(inode))
2957                         return CURSEG_COLD_DATA;
2958                 if (file_is_hot(inode) ||
2959                                 is_inode_flag_set(inode, FI_HOT_DATA) ||
2960                                 f2fs_is_atomic_file(inode) ||
2961                                 f2fs_is_volatile_file(inode))
2962                         return CURSEG_HOT_DATA;
2963                 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
2964         } else {
2965                 if (IS_DNODE(fio->page))
2966                         return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
2967                                                 CURSEG_HOT_NODE;
2968                 return CURSEG_COLD_NODE;
2969         }
2970 }
2971
2972 static int __get_segment_type(struct f2fs_io_info *fio)
2973 {
2974         int type = 0;
2975
2976         switch (F2FS_OPTION(fio->sbi).active_logs) {
2977         case 2:
2978                 type = __get_segment_type_2(fio);
2979                 break;
2980         case 4:
2981                 type = __get_segment_type_4(fio);
2982                 break;
2983         case 6:
2984                 type = __get_segment_type_6(fio);
2985                 break;
2986         default:
2987                 f2fs_bug_on(fio->sbi, true);
2988         }
2989
2990         if (IS_HOT(type))
2991                 fio->temp = HOT;
2992         else if (IS_WARM(type))
2993                 fio->temp = WARM;
2994         else
2995                 fio->temp = COLD;
2996         return type;
2997 }
2998
2999 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3000                 block_t old_blkaddr, block_t *new_blkaddr,
3001                 struct f2fs_summary *sum, int type,
3002                 struct f2fs_io_info *fio, bool add_list)
3003 {
3004         struct sit_info *sit_i = SIT_I(sbi);
3005         struct curseg_info *curseg = CURSEG_I(sbi, type);
3006
3007         down_read(&SM_I(sbi)->curseg_lock);
3008
3009         mutex_lock(&curseg->curseg_mutex);
3010         down_write(&sit_i->sentry_lock);
3011
3012         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3013
3014         f2fs_wait_discard_bio(sbi, *new_blkaddr);
3015
3016         /*
3017          * __add_sum_entry should be resided under the curseg_mutex
3018          * because, this function updates a summary entry in the
3019          * current summary block.
3020          */
3021         __add_sum_entry(sbi, type, sum);
3022
3023         __refresh_next_blkoff(sbi, curseg);
3024
3025         stat_inc_block_count(sbi, curseg);
3026
3027         /*
3028          * SIT information should be updated before segment allocation,
3029          * since SSR needs latest valid block information.
3030          */
3031         update_sit_entry(sbi, *new_blkaddr, 1);
3032         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3033                 update_sit_entry(sbi, old_blkaddr, -1);
3034
3035         if (!__has_curseg_space(sbi, type))
3036                 sit_i->s_ops->allocate_segment(sbi, type, false);
3037
3038         /*
3039          * segment dirty status should be updated after segment allocation,
3040          * so we just need to update status only one time after previous
3041          * segment being closed.
3042          */
3043         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3044         locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3045
3046         up_write(&sit_i->sentry_lock);
3047
3048         if (page && IS_NODESEG(type)) {
3049                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3050
3051                 f2fs_inode_chksum_set(sbi, page);
3052         }
3053
3054         if (add_list) {
3055                 struct f2fs_bio_info *io;
3056
3057                 INIT_LIST_HEAD(&fio->list);
3058                 fio->in_list = true;
3059                 fio->retry = false;
3060                 io = sbi->write_io[fio->type] + fio->temp;
3061                 spin_lock(&io->io_lock);
3062                 list_add_tail(&fio->list, &io->io_list);
3063                 spin_unlock(&io->io_lock);
3064         }
3065
3066         mutex_unlock(&curseg->curseg_mutex);
3067
3068         up_read(&SM_I(sbi)->curseg_lock);
3069 }
3070
3071 static void update_device_state(struct f2fs_io_info *fio)
3072 {
3073         struct f2fs_sb_info *sbi = fio->sbi;
3074         unsigned int devidx;
3075
3076         if (!sbi->s_ndevs)
3077                 return;
3078
3079         devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);
3080
3081         /* update device state for fsync */
3082         f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);
3083
3084         /* update device state for checkpoint */
3085         if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3086                 spin_lock(&sbi->dev_lock);
3087                 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3088                 spin_unlock(&sbi->dev_lock);
3089         }
3090 }
3091
3092 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3093 {
3094         int type = __get_segment_type(fio);
3095         bool keep_order = (test_opt(fio->sbi, LFS) && type == CURSEG_COLD_DATA);
3096
3097         if (keep_order)
3098                 down_read(&fio->sbi->io_order_lock);
3099 reallocate:
3100         f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3101                         &fio->new_blkaddr, sum, type, fio, true);
3102         if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
3103                 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3104                                         fio->old_blkaddr, fio->old_blkaddr);
3105
3106         /* writeout dirty page into bdev */
3107         f2fs_submit_page_write(fio);
3108         if (fio->retry) {
3109                 fio->old_blkaddr = fio->new_blkaddr;
3110                 goto reallocate;
3111         }
3112
3113         update_device_state(fio);
3114
3115         if (keep_order)
3116                 up_read(&fio->sbi->io_order_lock);
3117 }
3118
3119 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3120                                         enum iostat_type io_type)
3121 {
3122         struct f2fs_io_info fio = {
3123                 .sbi = sbi,
3124                 .type = META,
3125                 .temp = HOT,
3126                 .op = REQ_OP_WRITE,
3127                 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3128                 .old_blkaddr = page->index,
3129                 .new_blkaddr = page->index,
3130                 .page = page,
3131                 .encrypted_page = NULL,
3132                 .in_list = false,
3133         };
3134
3135         if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3136                 fio.op_flags &= ~REQ_META;
3137
3138         set_page_writeback(page);
3139         ClearPageError(page);
3140         f2fs_submit_page_write(&fio);
3141
3142         stat_inc_meta_count(sbi, page->index);
3143         f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3144 }
3145
3146 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3147 {
3148         struct f2fs_summary sum;
3149
3150         set_summary(&sum, nid, 0, 0);
3151         do_write_page(&sum, fio);
3152
3153         f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3154 }
3155
3156 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3157                                         struct f2fs_io_info *fio)
3158 {
3159         struct f2fs_sb_info *sbi = fio->sbi;
3160         struct f2fs_summary sum;
3161
3162         f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3163         set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3164         do_write_page(&sum, fio);
3165         f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3166
3167         f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3168 }
3169
3170 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3171 {
3172         int err;
3173         struct f2fs_sb_info *sbi = fio->sbi;
3174
3175         fio->new_blkaddr = fio->old_blkaddr;
3176         /* i/o temperature is needed for passing down write hints */
3177         __get_segment_type(fio);
3178
3179         f2fs_bug_on(sbi, !IS_DATASEG(get_seg_entry(sbi,
3180                         GET_SEGNO(sbi, fio->new_blkaddr))->type));
3181
3182         stat_inc_inplace_blocks(fio->sbi);
3183
3184         err = f2fs_submit_page_bio(fio);
3185         if (!err)
3186                 update_device_state(fio);
3187
3188         f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3189
3190         return err;
3191 }
3192
3193 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3194                                                 unsigned int segno)
3195 {
3196         int i;
3197
3198         for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3199                 if (CURSEG_I(sbi, i)->segno == segno)
3200                         break;
3201         }
3202         return i;
3203 }
3204
3205 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3206                                 block_t old_blkaddr, block_t new_blkaddr,
3207                                 bool recover_curseg, bool recover_newaddr)
3208 {
3209         struct sit_info *sit_i = SIT_I(sbi);
3210         struct curseg_info *curseg;
3211         unsigned int segno, old_cursegno;
3212         struct seg_entry *se;
3213         int type;
3214         unsigned short old_blkoff;
3215
3216         segno = GET_SEGNO(sbi, new_blkaddr);
3217         se = get_seg_entry(sbi, segno);
3218         type = se->type;
3219
3220         down_write(&SM_I(sbi)->curseg_lock);
3221
3222         if (!recover_curseg) {
3223                 /* for recovery flow */
3224                 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3225                         if (old_blkaddr == NULL_ADDR)
3226                                 type = CURSEG_COLD_DATA;
3227                         else
3228                                 type = CURSEG_WARM_DATA;
3229                 }
3230         } else {
3231                 if (IS_CURSEG(sbi, segno)) {
3232                         /* se->type is volatile as SSR allocation */
3233                         type = __f2fs_get_curseg(sbi, segno);
3234                         f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3235                 } else {
3236                         type = CURSEG_WARM_DATA;
3237                 }
3238         }
3239
3240         f2fs_bug_on(sbi, !IS_DATASEG(type));
3241         curseg = CURSEG_I(sbi, type);
3242
3243         mutex_lock(&curseg->curseg_mutex);
3244         down_write(&sit_i->sentry_lock);
3245
3246         old_cursegno = curseg->segno;
3247         old_blkoff = curseg->next_blkoff;
3248
3249         /* change the current segment */
3250         if (segno != curseg->segno) {
3251                 curseg->next_segno = segno;
3252                 change_curseg(sbi, type);
3253         }
3254
3255         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3256         __add_sum_entry(sbi, type, sum);
3257
3258         if (!recover_curseg || recover_newaddr)
3259                 update_sit_entry(sbi, new_blkaddr, 1);
3260         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3261                 invalidate_mapping_pages(META_MAPPING(sbi),
3262                                         old_blkaddr, old_blkaddr);
3263                 update_sit_entry(sbi, old_blkaddr, -1);
3264         }
3265
3266         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3267         locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3268
3269         locate_dirty_segment(sbi, old_cursegno);
3270
3271         if (recover_curseg) {
3272                 if (old_cursegno != curseg->segno) {
3273                         curseg->next_segno = old_cursegno;
3274                         change_curseg(sbi, type);
3275                 }
3276                 curseg->next_blkoff = old_blkoff;
3277         }
3278
3279         up_write(&sit_i->sentry_lock);
3280         mutex_unlock(&curseg->curseg_mutex);
3281         up_write(&SM_I(sbi)->curseg_lock);
3282 }
3283
3284 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3285                                 block_t old_addr, block_t new_addr,
3286                                 unsigned char version, bool recover_curseg,
3287                                 bool recover_newaddr)
3288 {
3289         struct f2fs_summary sum;
3290
3291         set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3292
3293         f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3294                                         recover_curseg, recover_newaddr);
3295
3296         f2fs_update_data_blkaddr(dn, new_addr);
3297 }
3298
3299 void f2fs_wait_on_page_writeback(struct page *page,
3300                                 enum page_type type, bool ordered, bool locked)
3301 {
3302         if (PageWriteback(page)) {
3303                 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3304
3305                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3306                 if (ordered) {
3307                         wait_on_page_writeback(page);
3308                         f2fs_bug_on(sbi, locked && PageWriteback(page));
3309                 } else {
3310                         wait_for_stable_page(page);
3311                 }
3312         }
3313 }
3314
3315 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3316 {
3317         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3318         struct page *cpage;
3319
3320         if (!f2fs_post_read_required(inode))
3321                 return;
3322
3323         if (!is_valid_data_blkaddr(sbi, blkaddr))
3324                 return;
3325
3326         cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3327         if (cpage) {
3328                 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3329                 f2fs_put_page(cpage, 1);
3330         }
3331 }
3332
3333 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3334                                                                 block_t len)
3335 {
3336         block_t i;
3337
3338         for (i = 0; i < len; i++)
3339                 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3340 }
3341
3342 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3343 {
3344         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3345         struct curseg_info *seg_i;
3346         unsigned char *kaddr;
3347         struct page *page;
3348         block_t start;
3349         int i, j, offset;
3350
3351         start = start_sum_block(sbi);
3352
3353         page = f2fs_get_meta_page(sbi, start++);
3354         if (IS_ERR(page))
3355                 return PTR_ERR(page);
3356         kaddr = (unsigned char *)page_address(page);
3357
3358         /* Step 1: restore nat cache */
3359         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3360         memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3361
3362         /* Step 2: restore sit cache */
3363         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3364         memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3365         offset = 2 * SUM_JOURNAL_SIZE;
3366
3367         /* Step 3: restore summary entries */
3368         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3369                 unsigned short blk_off;
3370                 unsigned int segno;
3371
3372                 seg_i = CURSEG_I(sbi, i);
3373                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3374                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3375                 seg_i->next_segno = segno;
3376                 reset_curseg(sbi, i, 0);
3377                 seg_i->alloc_type = ckpt->alloc_type[i];
3378                 seg_i->next_blkoff = blk_off;
3379
3380                 if (seg_i->alloc_type == SSR)
3381                         blk_off = sbi->blocks_per_seg;
3382
3383                 for (j = 0; j < blk_off; j++) {
3384                         struct f2fs_summary *s;
3385                         s = (struct f2fs_summary *)(kaddr + offset);
3386                         seg_i->sum_blk->entries[j] = *s;
3387                         offset += SUMMARY_SIZE;
3388                         if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3389                                                 SUM_FOOTER_SIZE)
3390                                 continue;
3391
3392                         f2fs_put_page(page, 1);
3393                         page = NULL;
3394
3395                         page = f2fs_get_meta_page(sbi, start++);
3396                         if (IS_ERR(page))
3397                                 return PTR_ERR(page);
3398                         kaddr = (unsigned char *)page_address(page);
3399                         offset = 0;
3400                 }
3401         }
3402         f2fs_put_page(page, 1);
3403         return 0;
3404 }
3405
3406 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3407 {
3408         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3409         struct f2fs_summary_block *sum;
3410         struct curseg_info *curseg;
3411         struct page *new;
3412         unsigned short blk_off;
3413         unsigned int segno = 0;
3414         block_t blk_addr = 0;
3415         int err = 0;
3416
3417         /* get segment number and block addr */
3418         if (IS_DATASEG(type)) {
3419                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3420                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3421                                                         CURSEG_HOT_DATA]);
3422                 if (__exist_node_summaries(sbi))
3423                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
3424                 else
3425                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3426         } else {
3427                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3428                                                         CURSEG_HOT_NODE]);
3429                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3430                                                         CURSEG_HOT_NODE]);
3431                 if (__exist_node_summaries(sbi))
3432                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3433                                                         type - CURSEG_HOT_NODE);
3434                 else
3435                         blk_addr = GET_SUM_BLOCK(sbi, segno);
3436         }
3437
3438         new = f2fs_get_meta_page(sbi, blk_addr);
3439         if (IS_ERR(new))
3440                 return PTR_ERR(new);
3441         sum = (struct f2fs_summary_block *)page_address(new);
3442
3443         if (IS_NODESEG(type)) {
3444                 if (__exist_node_summaries(sbi)) {
3445                         struct f2fs_summary *ns = &sum->entries[0];
3446                         int i;
3447                         for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3448                                 ns->version = 0;
3449                                 ns->ofs_in_node = 0;
3450                         }
3451                 } else {
3452                         err = f2fs_restore_node_summary(sbi, segno, sum);
3453                         if (err)
3454                                 goto out;
3455                 }
3456         }
3457
3458         /* set uncompleted segment to curseg */
3459         curseg = CURSEG_I(sbi, type);
3460         mutex_lock(&curseg->curseg_mutex);
3461
3462         /* update journal info */
3463         down_write(&curseg->journal_rwsem);
3464         memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3465         up_write(&curseg->journal_rwsem);
3466
3467         memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3468         memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3469         curseg->next_segno = segno;
3470         reset_curseg(sbi, type, 0);
3471         curseg->alloc_type = ckpt->alloc_type[type];
3472         curseg->next_blkoff = blk_off;
3473         mutex_unlock(&curseg->curseg_mutex);
3474 out:
3475         f2fs_put_page(new, 1);
3476         return err;
3477 }
3478
3479 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3480 {
3481         struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3482         struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3483         int type = CURSEG_HOT_DATA;
3484         int err;
3485
3486         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3487                 int npages = f2fs_npages_for_summary_flush(sbi, true);
3488
3489                 if (npages >= 2)
3490                         f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3491                                                         META_CP, true);
3492
3493                 /* restore for compacted data summary */
3494                 err = read_compacted_summaries(sbi);
3495                 if (err)
3496                         return err;
3497                 type = CURSEG_HOT_NODE;
3498         }
3499
3500         if (__exist_node_summaries(sbi))
3501                 f2fs_ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
3502                                         NR_CURSEG_TYPE - type, META_CP, true);
3503
3504         for (; type <= CURSEG_COLD_NODE; type++) {
3505                 err = read_normal_summaries(sbi, type);
3506                 if (err)
3507                         return err;
3508         }
3509
3510         /* sanity check for summary blocks */
3511         if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3512                         sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES)
3513                 return -EINVAL;
3514
3515         return 0;
3516 }
3517
3518 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3519 {
3520         struct page *page;
3521         unsigned char *kaddr;
3522         struct f2fs_summary *summary;
3523         struct curseg_info *seg_i;
3524         int written_size = 0;
3525         int i, j;
3526
3527         page = f2fs_grab_meta_page(sbi, blkaddr++);
3528         kaddr = (unsigned char *)page_address(page);
3529         memset(kaddr, 0, PAGE_SIZE);
3530
3531         /* Step 1: write nat cache */
3532         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3533         memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3534         written_size += SUM_JOURNAL_SIZE;
3535
3536         /* Step 2: write sit cache */
3537         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3538         memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3539         written_size += SUM_JOURNAL_SIZE;
3540
3541         /* Step 3: write summary entries */
3542         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3543                 unsigned short blkoff;
3544                 seg_i = CURSEG_I(sbi, i);
3545                 if (sbi->ckpt->alloc_type[i] == SSR)
3546                         blkoff = sbi->blocks_per_seg;
3547                 else
3548                         blkoff = curseg_blkoff(sbi, i);
3549
3550                 for (j = 0; j < blkoff; j++) {
3551                         if (!page) {
3552                                 page = f2fs_grab_meta_page(sbi, blkaddr++);
3553                                 kaddr = (unsigned char *)page_address(page);
3554                                 memset(kaddr, 0, PAGE_SIZE);
3555                                 written_size = 0;
3556                         }
3557                         summary = (struct f2fs_summary *)(kaddr + written_size);
3558                         *summary = seg_i->sum_blk->entries[j];
3559                         written_size += SUMMARY_SIZE;
3560
3561                         if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
3562                                                         SUM_FOOTER_SIZE)
3563                                 continue;
3564
3565                         set_page_dirty(page);
3566                         f2fs_put_page(page, 1);
3567                         page = NULL;
3568                 }
3569         }
3570         if (page) {
3571                 set_page_dirty(page);
3572                 f2fs_put_page(page, 1);
3573         }
3574 }
3575
3576 static void write_normal_summaries(struct f2fs_sb_info *sbi,
3577                                         block_t blkaddr, int type)
3578 {
3579         int i, end;
3580         if (IS_DATASEG(type))
3581                 end = type + NR_CURSEG_DATA_TYPE;
3582         else
3583                 end = type + NR_CURSEG_NODE_TYPE;
3584
3585         for (i = type; i < end; i++)
3586                 write_current_sum_page(sbi, i, blkaddr + (i - type));
3587 }
3588
3589 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3590 {
3591         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
3592                 write_compacted_summaries(sbi, start_blk);
3593         else
3594                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
3595 }
3596
3597 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3598 {
3599         write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
3600 }
3601
3602 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3603                                         unsigned int val, int alloc)
3604 {
3605         int i;
3606
3607         if (type == NAT_JOURNAL) {
3608                 for (i = 0; i < nats_in_cursum(journal); i++) {
3609                         if (le32_to_cpu(nid_in_journal(journal, i)) == val)
3610                                 return i;
3611                 }
3612                 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
3613                         return update_nats_in_cursum(journal, 1);
3614         } else if (type == SIT_JOURNAL) {
3615                 for (i = 0; i < sits_in_cursum(journal); i++)
3616                         if (le32_to_cpu(segno_in_journal(journal, i)) == val)
3617                                 return i;
3618                 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
3619                         return update_sits_in_cursum(journal, 1);
3620         }
3621         return -1;
3622 }
3623
3624 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
3625                                         unsigned int segno)
3626 {
3627         return f2fs_get_meta_page_nofail(sbi, current_sit_addr(sbi, segno));
3628 }
3629
3630 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
3631                                         unsigned int start)
3632 {
3633         struct sit_info *sit_i = SIT_I(sbi);
3634         struct page *page;
3635         pgoff_t src_off, dst_off;
3636
3637         src_off = current_sit_addr(sbi, start);
3638         dst_off = next_sit_addr(sbi, src_off);
3639
3640         page = f2fs_grab_meta_page(sbi, dst_off);
3641         seg_info_to_sit_page(sbi, page, start);
3642
3643         set_page_dirty(page);
3644         set_to_next_sit(sit_i, start);
3645
3646         return page;
3647 }
3648
3649 static struct sit_entry_set *grab_sit_entry_set(void)
3650 {
3651         struct sit_entry_set *ses =
3652                         f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
3653
3654         ses->entry_cnt = 0;
3655         INIT_LIST_HEAD(&ses->set_list);
3656         return ses;
3657 }
3658
3659 static void release_sit_entry_set(struct sit_entry_set *ses)
3660 {
3661         list_del(&ses->set_list);
3662         kmem_cache_free(sit_entry_set_slab, ses);
3663 }
3664
3665 static void adjust_sit_entry_set(struct sit_entry_set *ses,
3666                                                 struct list_head *head)
3667 {
3668         struct sit_entry_set *next = ses;
3669
3670         if (list_is_last(&ses->set_list, head))
3671                 return;
3672
3673         list_for_each_entry_continue(next, head, set_list)
3674                 if (ses->entry_cnt <= next->entry_cnt)
3675                         break;
3676
3677         list_move_tail(&ses->set_list, &next->set_list);
3678 }
3679
3680 static void add_sit_entry(unsigned int segno, struct list_head *head)
3681 {
3682         struct sit_entry_set *ses;
3683         unsigned int start_segno = START_SEGNO(segno);
3684
3685         list_for_each_entry(ses, head, set_list) {
3686                 if (ses->start_segno == start_segno) {
3687                         ses->entry_cnt++;
3688                         adjust_sit_entry_set(ses, head);
3689                         return;
3690                 }
3691         }
3692
3693         ses = grab_sit_entry_set();
3694
3695         ses->start_segno = start_segno;
3696         ses->entry_cnt++;
3697         list_add(&ses->set_list, head);
3698 }
3699
3700 static void add_sits_in_set(struct f2fs_sb_info *sbi)
3701 {
3702         struct f2fs_sm_info *sm_info = SM_I(sbi);
3703         struct list_head *set_list = &sm_info->sit_entry_set;
3704         unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
3705         unsigned int segno;
3706
3707         for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
3708                 add_sit_entry(segno, set_list);
3709 }
3710
3711 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
3712 {
3713         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
3714         struct f2fs_journal *journal = curseg->journal;
3715         int i;
3716
3717         down_write(&curseg->journal_rwsem);
3718         for (i = 0; i < sits_in_cursum(journal); i++) {
3719                 unsigned int segno;
3720                 bool dirtied;
3721
3722                 segno = le32_to_cpu(segno_in_journal(journal, i));
3723                 dirtied = __mark_sit_entry_dirty(sbi, segno);
3724
3725                 if (!dirtied)
3726                         add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
3727         }
3728         update_sits_in_cursum(journal, -i);
3729         up_write(&curseg->journal_rwsem);
3730 }
3731
3732 /*
3733  * CP calls this function, which flushes SIT entries including sit_journal,
3734  * and moves prefree segs to free segs.
3735  */
3736 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
3737 {
3738         struct sit_info *sit_i = SIT_I(sbi);
3739         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
3740         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
3741         struct f2fs_journal *journal = curseg->journal;
3742         struct sit_entry_set *ses, *tmp;
3743         struct list_head *head = &SM_I(sbi)->sit_entry_set;
3744         bool to_journal = true;
3745         struct seg_entry *se;
3746
3747         down_write(&sit_i->sentry_lock);
3748
3749         if (!sit_i->dirty_sentries)
3750                 goto out;
3751
3752         /*
3753          * add and account sit entries of dirty bitmap in sit entry
3754          * set temporarily
3755          */
3756         add_sits_in_set(sbi);
3757
3758         /*
3759          * if there are no enough space in journal to store dirty sit
3760          * entries, remove all entries from journal and add and account
3761          * them in sit entry set.
3762          */
3763         if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL))
3764                 remove_sits_in_journal(sbi);
3765
3766         /*
3767          * there are two steps to flush sit entries:
3768          * #1, flush sit entries to journal in current cold data summary block.
3769          * #2, flush sit entries to sit page.
3770          */
3771         list_for_each_entry_safe(ses, tmp, head, set_list) {
3772                 struct page *page = NULL;
3773                 struct f2fs_sit_block *raw_sit = NULL;
3774                 unsigned int start_segno = ses->start_segno;
3775                 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
3776                                                 (unsigned long)MAIN_SEGS(sbi));
3777                 unsigned int segno = start_segno;
3778
3779                 if (to_journal &&
3780                         !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
3781                         to_journal = false;
3782
3783                 if (to_journal) {
3784                         down_write(&curseg->journal_rwsem);
3785                 } else {
3786                         page = get_next_sit_page(sbi, start_segno);
3787                         raw_sit = page_address(page);
3788                 }
3789
3790                 /* flush dirty sit entries in region of current sit set */
3791                 for_each_set_bit_from(segno, bitmap, end) {
3792                         int offset, sit_offset;
3793
3794                         se = get_seg_entry(sbi, segno);
3795 #ifdef CONFIG_F2FS_CHECK_FS
3796                         if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
3797                                                 SIT_VBLOCK_MAP_SIZE))
3798                                 f2fs_bug_on(sbi, 1);
3799 #endif
3800
3801                         /* add discard candidates */
3802                         if (!(cpc->reason & CP_DISCARD)) {
3803                                 cpc->trim_start = segno;
3804                                 add_discard_addrs(sbi, cpc, false);
3805                         }
3806
3807                         if (to_journal) {
3808                                 offset = f2fs_lookup_journal_in_cursum(journal,
3809                                                         SIT_JOURNAL, segno, 1);
3810                                 f2fs_bug_on(sbi, offset < 0);
3811                                 segno_in_journal(journal, offset) =
3812                                                         cpu_to_le32(segno);
3813                                 seg_info_to_raw_sit(se,
3814                                         &sit_in_journal(journal, offset));
3815                                 check_block_count(sbi, segno,
3816                                         &sit_in_journal(journal, offset));
3817                         } else {
3818                                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
3819                                 seg_info_to_raw_sit(se,
3820                                                 &raw_sit->entries[sit_offset]);
3821                                 check_block_count(sbi, segno,
3822                                                 &raw_sit->entries[sit_offset]);
3823                         }
3824
3825                         __clear_bit(segno, bitmap);
3826                         sit_i->dirty_sentries--;
3827                         ses->entry_cnt--;
3828                 }
3829
3830                 if (to_journal)
3831                         up_write(&curseg->journal_rwsem);
3832                 else
3833                         f2fs_put_page(page, 1);
3834
3835                 f2fs_bug_on(sbi, ses->entry_cnt);
3836                 release_sit_entry_set(ses);
3837         }
3838
3839         f2fs_bug_on(sbi, !list_empty(head));
3840         f2fs_bug_on(sbi, sit_i->dirty_sentries);
3841 out:
3842         if (cpc->reason & CP_DISCARD) {
3843                 __u64 trim_start = cpc->trim_start;
3844
3845                 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
3846                         add_discard_addrs(sbi, cpc, false);
3847
3848                 cpc->trim_start = trim_start;
3849         }
3850         up_write(&sit_i->sentry_lock);
3851
3852         set_prefree_as_free_segments(sbi);
3853 }
3854
3855 static int build_sit_info(struct f2fs_sb_info *sbi)
3856 {
3857         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
3858         struct sit_info *sit_i;
3859         unsigned int sit_segs, start;
3860         char *src_bitmap;
3861         unsigned int bitmap_size;
3862
3863         /* allocate memory for SIT information */
3864         sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
3865         if (!sit_i)
3866                 return -ENOMEM;
3867
3868         SM_I(sbi)->sit_info = sit_i;
3869
3870         sit_i->sentries =
3871                 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
3872                                               MAIN_SEGS(sbi)),
3873                               GFP_KERNEL);
3874         if (!sit_i->sentries)
3875                 return -ENOMEM;
3876
3877         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
3878         sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, bitmap_size,
3879                                                                 GFP_KERNEL);
3880         if (!sit_i->dirty_sentries_bitmap)
3881                 return -ENOMEM;
3882
3883         for (start = 0; start < MAIN_SEGS(sbi); start++) {
3884                 sit_i->sentries[start].cur_valid_map
3885                         = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
3886                 sit_i->sentries[start].ckpt_valid_map
3887                         = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
3888                 if (!sit_i->sentries[start].cur_valid_map ||
3889                                 !sit_i->sentries[start].ckpt_valid_map)
3890                         return -ENOMEM;
3891
3892 #ifdef CONFIG_F2FS_CHECK_FS
3893                 sit_i->sentries[start].cur_valid_map_mir
3894                         = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
3895                 if (!sit_i->sentries[start].cur_valid_map_mir)
3896                         return -ENOMEM;
3897 #endif
3898
3899                 sit_i->sentries[start].discard_map
3900                         = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE,
3901                                                         GFP_KERNEL);
3902                 if (!sit_i->sentries[start].discard_map)
3903                         return -ENOMEM;
3904         }
3905
3906         sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
3907         if (!sit_i->tmp_map)
3908                 return -ENOMEM;
3909
3910         if (__is_large_section(sbi)) {
3911                 sit_i->sec_entries =
3912                         f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
3913                                                       MAIN_SECS(sbi)),
3914                                       GFP_KERNEL);
3915                 if (!sit_i->sec_entries)
3916                         return -ENOMEM;
3917         }
3918
3919         /* get information related with SIT */
3920         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
3921
3922         /* setup SIT bitmap from ckeckpoint pack */
3923         bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
3924         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
3925
3926         sit_i->sit_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
3927         if (!sit_i->sit_bitmap)
3928                 return -ENOMEM;
3929
3930 #ifdef CONFIG_F2FS_CHECK_FS
3931         sit_i->sit_bitmap_mir = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
3932         if (!sit_i->sit_bitmap_mir)
3933                 return -ENOMEM;
3934 #endif
3935
3936         /* init SIT information */
3937         sit_i->s_ops = &default_salloc_ops;
3938
3939         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
3940         sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
3941         sit_i->written_valid_blocks = 0;
3942         sit_i->bitmap_size = bitmap_size;
3943         sit_i->dirty_sentries = 0;
3944         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
3945         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
3946         sit_i->mounted_time = ktime_get_real_seconds();
3947         init_rwsem(&sit_i->sentry_lock);
3948         return 0;
3949 }
3950
3951 static int build_free_segmap(struct f2fs_sb_info *sbi)
3952 {
3953         struct free_segmap_info *free_i;
3954         unsigned int bitmap_size, sec_bitmap_size;
3955
3956         /* allocate memory for free segmap information */
3957         free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
3958         if (!free_i)
3959                 return -ENOMEM;
3960
3961         SM_I(sbi)->free_info = free_i;
3962
3963         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
3964         free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
3965         if (!free_i->free_segmap)
3966                 return -ENOMEM;
3967
3968         sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
3969         free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
3970         if (!free_i->free_secmap)
3971                 return -ENOMEM;
3972
3973         /* set all segments as dirty temporarily */
3974         memset(free_i->free_segmap, 0xff, bitmap_size);
3975         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
3976
3977         /* init free segmap information */
3978         free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
3979         free_i->free_segments = 0;
3980         free_i->free_sections = 0;
3981         spin_lock_init(&free_i->segmap_lock);
3982         return 0;
3983 }
3984
3985 static int build_curseg(struct f2fs_sb_info *sbi)
3986 {
3987         struct curseg_info *array;
3988         int i;
3989
3990         array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, sizeof(*array)),
3991                              GFP_KERNEL);
3992         if (!array)
3993                 return -ENOMEM;
3994
3995         SM_I(sbi)->curseg_array = array;
3996
3997         for (i = 0; i < NR_CURSEG_TYPE; i++) {
3998                 mutex_init(&array[i].curseg_mutex);
3999                 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4000                 if (!array[i].sum_blk)
4001                         return -ENOMEM;
4002                 init_rwsem(&array[i].journal_rwsem);
4003                 array[i].journal = f2fs_kzalloc(sbi,
4004                                 sizeof(struct f2fs_journal), GFP_KERNEL);
4005                 if (!array[i].journal)
4006                         return -ENOMEM;
4007                 array[i].segno = NULL_SEGNO;
4008                 array[i].next_blkoff = 0;
4009         }
4010         return restore_curseg_summaries(sbi);
4011 }
4012
4013 static int build_sit_entries(struct f2fs_sb_info *sbi)
4014 {
4015         struct sit_info *sit_i = SIT_I(sbi);
4016         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4017         struct f2fs_journal *journal = curseg->journal;
4018         struct seg_entry *se;
4019         struct f2fs_sit_entry sit;
4020         int sit_blk_cnt = SIT_BLK_CNT(sbi);
4021         unsigned int i, start, end;
4022         unsigned int readed, start_blk = 0;
4023         int err = 0;
4024         block_t total_node_blocks = 0;
4025
4026         do {
4027                 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
4028                                                         META_SIT, true);
4029
4030                 start = start_blk * sit_i->sents_per_block;
4031                 end = (start_blk + readed) * sit_i->sents_per_block;
4032
4033                 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4034                         struct f2fs_sit_block *sit_blk;
4035                         struct page *page;
4036
4037                         se = &sit_i->sentries[start];
4038                         page = get_current_sit_page(sbi, start);
4039                         if (IS_ERR(page))
4040                                 return PTR_ERR(page);
4041                         sit_blk = (struct f2fs_sit_block *)page_address(page);
4042                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4043                         f2fs_put_page(page, 1);
4044
4045                         err = check_block_count(sbi, start, &sit);
4046                         if (err)
4047                                 return err;
4048                         seg_info_from_raw_sit(se, &sit);
4049                         if (IS_NODESEG(se->type))
4050                                 total_node_blocks += se->valid_blocks;
4051
4052                         /* build discard map only one time */
4053                         if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4054                                 memset(se->discard_map, 0xff,
4055                                         SIT_VBLOCK_MAP_SIZE);
4056                         } else {
4057                                 memcpy(se->discard_map,
4058                                         se->cur_valid_map,
4059                                         SIT_VBLOCK_MAP_SIZE);
4060                                 sbi->discard_blks +=
4061                                         sbi->blocks_per_seg -
4062                                         se->valid_blocks;
4063                         }
4064
4065                         if (__is_large_section(sbi))
4066                                 get_sec_entry(sbi, start)->valid_blocks +=
4067                                                         se->valid_blocks;
4068                 }
4069                 start_blk += readed;
4070         } while (start_blk < sit_blk_cnt);
4071
4072         down_read(&curseg->journal_rwsem);
4073         for (i = 0; i < sits_in_cursum(journal); i++) {
4074                 unsigned int old_valid_blocks;
4075
4076                 start = le32_to_cpu(segno_in_journal(journal, i));
4077                 if (start >= MAIN_SEGS(sbi)) {
4078                         f2fs_msg(sbi->sb, KERN_ERR,
4079                                         "Wrong journal entry on segno %u",
4080                                         start);
4081                         set_sbi_flag(sbi, SBI_NEED_FSCK);
4082                         err = -EINVAL;
4083                         break;
4084                 }
4085
4086                 se = &sit_i->sentries[start];
4087                 sit = sit_in_journal(journal, i);
4088
4089                 old_valid_blocks = se->valid_blocks;
4090                 if (IS_NODESEG(se->type))
4091                         total_node_blocks -= old_valid_blocks;
4092
4093                 err = check_block_count(sbi, start, &sit);
4094                 if (err)
4095                         break;
4096                 seg_info_from_raw_sit(se, &sit);
4097                 if (IS_NODESEG(se->type))
4098                         total_node_blocks += se->valid_blocks;
4099
4100                 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4101                         memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4102                 } else {
4103                         memcpy(se->discard_map, se->cur_valid_map,
4104                                                 SIT_VBLOCK_MAP_SIZE);
4105                         sbi->discard_blks += old_valid_blocks;
4106                         sbi->discard_blks -= se->valid_blocks;
4107                 }
4108
4109                 if (__is_large_section(sbi)) {
4110                         get_sec_entry(sbi, start)->valid_blocks +=
4111                                                         se->valid_blocks;
4112                         get_sec_entry(sbi, start)->valid_blocks -=
4113                                                         old_valid_blocks;
4114                 }
4115         }
4116         up_read(&curseg->journal_rwsem);
4117
4118         if (!err && total_node_blocks != valid_node_count(sbi)) {
4119                 f2fs_msg(sbi->sb, KERN_ERR,
4120                         "SIT is corrupted node# %u vs %u",
4121                         total_node_blocks, valid_node_count(sbi));
4122                 set_sbi_flag(sbi, SBI_NEED_FSCK);
4123                 err = -EINVAL;
4124         }
4125
4126         return err;
4127 }
4128
4129 static void init_free_segmap(struct f2fs_sb_info *sbi)
4130 {
4131         unsigned int start;
4132         int type;
4133
4134         for (start = 0; start < MAIN_SEGS(sbi); start++) {
4135                 struct seg_entry *sentry = get_seg_entry(sbi, start);
4136                 if (!sentry->valid_blocks)
4137                         __set_free(sbi, start);
4138                 else
4139                         SIT_I(sbi)->written_valid_blocks +=
4140                                                 sentry->valid_blocks;
4141         }
4142
4143         /* set use the current segments */
4144         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4145                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4146                 __set_test_and_inuse(sbi, curseg_t->segno);
4147         }
4148 }
4149
4150 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4151 {
4152         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4153         struct free_segmap_info *free_i = FREE_I(sbi);
4154         unsigned int segno = 0, offset = 0;
4155         unsigned short valid_blocks;
4156
4157         while (1) {
4158                 /* find dirty segment based on free segmap */
4159                 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4160                 if (segno >= MAIN_SEGS(sbi))
4161                         break;
4162                 offset = segno + 1;
4163                 valid_blocks = get_valid_blocks(sbi, segno, false);
4164                 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
4165                         continue;
4166                 if (valid_blocks > sbi->blocks_per_seg) {
4167                         f2fs_bug_on(sbi, 1);
4168                         continue;
4169                 }
4170                 mutex_lock(&dirty_i->seglist_lock);
4171                 __locate_dirty_segment(sbi, segno, DIRTY);
4172                 mutex_unlock(&dirty_i->seglist_lock);
4173         }
4174 }
4175
4176 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4177 {
4178         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4179         unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4180
4181         dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4182         if (!dirty_i->victim_secmap)
4183                 return -ENOMEM;
4184         return 0;
4185 }
4186
4187 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4188 {
4189         struct dirty_seglist_info *dirty_i;
4190         unsigned int bitmap_size, i;
4191
4192         /* allocate memory for dirty segments list information */
4193         dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4194                                                                 GFP_KERNEL);
4195         if (!dirty_i)
4196                 return -ENOMEM;
4197
4198         SM_I(sbi)->dirty_info = dirty_i;
4199         mutex_init(&dirty_i->seglist_lock);
4200
4201         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4202
4203         for (i = 0; i < NR_DIRTY_TYPE; i++) {
4204                 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4205                                                                 GFP_KERNEL);
4206                 if (!dirty_i->dirty_segmap[i])
4207                         return -ENOMEM;
4208         }
4209
4210         init_dirty_segmap(sbi);
4211         return init_victim_secmap(sbi);
4212 }
4213
4214 /*
4215  * Update min, max modified time for cost-benefit GC algorithm
4216  */
4217 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
4218 {
4219         struct sit_info *sit_i = SIT_I(sbi);
4220         unsigned int segno;
4221
4222         down_write(&sit_i->sentry_lock);
4223
4224         sit_i->min_mtime = ULLONG_MAX;
4225
4226         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4227                 unsigned int i;
4228                 unsigned long long mtime = 0;
4229
4230                 for (i = 0; i < sbi->segs_per_sec; i++)
4231                         mtime += get_seg_entry(sbi, segno + i)->mtime;
4232
4233                 mtime = div_u64(mtime, sbi->segs_per_sec);
4234
4235                 if (sit_i->min_mtime > mtime)
4236                         sit_i->min_mtime = mtime;
4237         }
4238         sit_i->max_mtime = get_mtime(sbi, false);
4239         up_write(&sit_i->sentry_lock);
4240 }
4241
4242 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
4243 {
4244         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4245         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
4246         struct f2fs_sm_info *sm_info;
4247         int err;
4248
4249         sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
4250         if (!sm_info)
4251                 return -ENOMEM;
4252
4253         /* init sm info */
4254         sbi->sm_info = sm_info;
4255         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
4256         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
4257         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
4258         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
4259         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
4260         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
4261         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
4262         sm_info->rec_prefree_segments = sm_info->main_segments *
4263                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
4264         if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
4265                 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
4266
4267         if (!test_opt(sbi, LFS))
4268                 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
4269         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
4270         sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
4271         sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;
4272         sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
4273         sm_info->min_ssr_sections = reserved_sections(sbi);
4274
4275         INIT_LIST_HEAD(&sm_info->sit_entry_set);
4276
4277         init_rwsem(&sm_info->curseg_lock);
4278
4279         if (!f2fs_readonly(sbi->sb)) {
4280                 err = f2fs_create_flush_cmd_control(sbi);
4281                 if (err)
4282                         return err;
4283         }
4284
4285         err = create_discard_cmd_control(sbi);
4286         if (err)
4287                 return err;
4288
4289         err = build_sit_info(sbi);
4290         if (err)
4291                 return err;
4292         err = build_free_segmap(sbi);
4293         if (err)
4294                 return err;
4295         err = build_curseg(sbi);
4296         if (err)
4297                 return err;
4298
4299         /* reinit free segmap based on SIT */
4300         err = build_sit_entries(sbi);
4301         if (err)
4302                 return err;
4303
4304         init_free_segmap(sbi);
4305         err = build_dirty_segmap(sbi);
4306         if (err)
4307                 return err;
4308
4309         init_min_max_mtime(sbi);
4310         return 0;
4311 }
4312
4313 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
4314                 enum dirty_type dirty_type)
4315 {
4316         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4317
4318         mutex_lock(&dirty_i->seglist_lock);
4319         kvfree(dirty_i->dirty_segmap[dirty_type]);
4320         dirty_i->nr_dirty[dirty_type] = 0;
4321         mutex_unlock(&dirty_i->seglist_lock);
4322 }
4323
4324 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
4325 {
4326         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4327         kvfree(dirty_i->victim_secmap);
4328 }
4329
4330 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
4331 {
4332         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4333         int i;
4334
4335         if (!dirty_i)
4336                 return;
4337
4338         /* discard pre-free/dirty segments list */
4339         for (i = 0; i < NR_DIRTY_TYPE; i++)
4340                 discard_dirty_segmap(sbi, i);
4341
4342         destroy_victim_secmap(sbi);
4343         SM_I(sbi)->dirty_info = NULL;
4344         kvfree(dirty_i);
4345 }
4346
4347 static void destroy_curseg(struct f2fs_sb_info *sbi)
4348 {
4349         struct curseg_info *array = SM_I(sbi)->curseg_array;
4350         int i;
4351
4352         if (!array)
4353                 return;
4354         SM_I(sbi)->curseg_array = NULL;
4355         for (i = 0; i < NR_CURSEG_TYPE; i++) {
4356                 kvfree(array[i].sum_blk);
4357                 kvfree(array[i].journal);
4358         }
4359         kvfree(array);
4360 }
4361
4362 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
4363 {
4364         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
4365         if (!free_i)
4366                 return;
4367         SM_I(sbi)->free_info = NULL;
4368         kvfree(free_i->free_segmap);
4369         kvfree(free_i->free_secmap);
4370         kvfree(free_i);
4371 }
4372
4373 static void destroy_sit_info(struct f2fs_sb_info *sbi)
4374 {
4375         struct sit_info *sit_i = SIT_I(sbi);
4376         unsigned int start;
4377
4378         if (!sit_i)
4379                 return;
4380
4381         if (sit_i->sentries) {
4382                 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4383                         kvfree(sit_i->sentries[start].cur_valid_map);
4384 #ifdef CONFIG_F2FS_CHECK_FS
4385                         kvfree(sit_i->sentries[start].cur_valid_map_mir);
4386 #endif
4387                         kvfree(sit_i->sentries[start].ckpt_valid_map);
4388                         kvfree(sit_i->sentries[start].discard_map);
4389                 }
4390         }
4391         kvfree(sit_i->tmp_map);
4392
4393         kvfree(sit_i->sentries);
4394         kvfree(sit_i->sec_entries);
4395         kvfree(sit_i->dirty_sentries_bitmap);
4396
4397         SM_I(sbi)->sit_info = NULL;
4398         kvfree(sit_i->sit_bitmap);
4399 #ifdef CONFIG_F2FS_CHECK_FS
4400         kvfree(sit_i->sit_bitmap_mir);
4401 #endif
4402         kvfree(sit_i);
4403 }
4404
4405 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
4406 {
4407         struct f2fs_sm_info *sm_info = SM_I(sbi);
4408
4409         if (!sm_info)
4410                 return;
4411         f2fs_destroy_flush_cmd_control(sbi, true);
4412         destroy_discard_cmd_control(sbi);
4413         destroy_dirty_segmap(sbi);
4414         destroy_curseg(sbi);
4415         destroy_free_segmap(sbi);
4416         destroy_sit_info(sbi);
4417         sbi->sm_info = NULL;
4418         kvfree(sm_info);
4419 }
4420
4421 int __init f2fs_create_segment_manager_caches(void)
4422 {
4423         discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
4424                         sizeof(struct discard_entry));
4425         if (!discard_entry_slab)
4426                 goto fail;
4427
4428         discard_cmd_slab = f2fs_kmem_cache_create("discard_cmd",
4429                         sizeof(struct discard_cmd));
4430         if (!discard_cmd_slab)
4431                 goto destroy_discard_entry;
4432
4433         sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
4434                         sizeof(struct sit_entry_set));
4435         if (!sit_entry_set_slab)
4436                 goto destroy_discard_cmd;
4437
4438         inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
4439                         sizeof(struct inmem_pages));
4440         if (!inmem_entry_slab)
4441                 goto destroy_sit_entry_set;
4442         return 0;
4443
4444 destroy_sit_entry_set:
4445         kmem_cache_destroy(sit_entry_set_slab);
4446 destroy_discard_cmd:
4447         kmem_cache_destroy(discard_cmd_slab);
4448 destroy_discard_entry:
4449         kmem_cache_destroy(discard_entry_slab);
4450 fail:
4451         return -ENOMEM;
4452 }
4453
4454 void f2fs_destroy_segment_manager_caches(void)
4455 {
4456         kmem_cache_destroy(sit_entry_set_slab);
4457         kmem_cache_destroy(discard_cmd_slab);
4458         kmem_cache_destroy(discard_entry_slab);
4459         kmem_cache_destroy(inmem_entry_slab);
4460 }